From 98a33dd9d80731b757d5fe6cf2535bba6cea017d Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Fri, 4 Feb 2022 10:12:15 +0100 Subject: [PATCH 01/19] first import --- apps/dnet-exporter-api/README.md | 6 + apps/dnet-exporter-api/pom.xml | 149 ++++ .../main/java/eu/dnetlib/CacheCustomizer.java | 29 + .../DNetOpenaireExporterApplication.java | 31 + .../DNetOpenaireExporterBeanFactory.java | 141 +++ .../eu/dnetlib/OpenaireExporterConfig.java | 548 ++++++++++++ .../main/java/eu/dnetlib/SwaggerConfig.java | 115 +++ .../common/AbstractExporterController.java | 100 +++ .../openaire/common/ConverterTextArray.java | 58 ++ .../openaire/common/ExporterConstants.java | 47 + .../openaire/common/GenericArrayUserType.java | 106 +++ .../eu/dnetlib/openaire/common/ISClient.java | 60 ++ .../dnetlib/openaire/common/ISClientImpl.java | 413 +++++++++ .../openaire/common/OperationManager.java | 60 ++ .../openaire/common/RFC3339DateFormat.java | 60 ++ .../eu/dnetlib/openaire/common/Utils.java | 22 + .../community/CommunityApiController.java | 304 +++++++ .../openaire/community/CommunityApiCore.java | 386 ++++++++ .../openaire/community/CommunityClient.java | 12 + .../community/CommunityClientImpl.java | 58 ++ .../openaire/community/CommunityCommon.java | 438 +++++++++ .../community/CommunityConstants.java | 58 ++ .../community/CommunityContentprovider.java | 105 +++ .../openaire/community/CommunityDetails.java | 51 ++ .../community/CommunityException.java | 20 + .../community/CommunityMappingUtils.java | 231 +++++ .../community/CommunityNotFoundException.java | 19 + .../CommunityOpenAIRECommunities.java | 42 + .../community/CommunityOrganization.java | 75 ++ .../openaire/community/CommunityProject.java | 86 ++ .../openaire/community/CommunityStatus.java | 17 + .../openaire/community/CommunitySummary.java | 167 ++++ .../CommunityWritableProperties.java | 96 ++ .../community/CommunityZenodoCommunity.java | 47 + .../selectioncriteria/Constraint.java | 44 + .../selectioncriteria/Constraints.java | 23 + .../selectioncriteria/SelectionCriteria.java | 30 + .../eu/dnetlib/openaire/context/Category.java | 67 ++ .../openaire/context/CategorySummary.java | 38 + .../eu/dnetlib/openaire/context/Concept.java | 67 ++ .../openaire/context/ConceptSummary.java | 51 ++ .../eu/dnetlib/openaire/context/Context.java | 91 ++ .../context/ContextApiController.java | 86 ++ .../openaire/context/ContextApiCore.java | 120 +++ .../openaire/context/ContextException.java | 20 + .../openaire/context/ContextMappingUtils.java | 113 +++ .../context/ContextNotFoundException.java | 18 + .../openaire/context/ContextSummary.java | 49 + .../eu/dnetlib/openaire/context/Param.java | 27 + .../openaire/dsm/DsmApiController.java | 498 +++++++++++ .../java/eu/dnetlib/openaire/dsm/DsmCore.java | 579 ++++++++++++ .../dsm/dao/ApiDbEntryRepository.java | 53 ++ .../dsm/dao/CountryTermRepository.java | 15 + .../dao/DatasourceApiDbEntryRepository.java | 14 + .../openaire/dsm/dao/DatasourceDao.java | 58 ++ .../openaire/dsm/dao/DatasourceDaoImpl.java | 262 ++++++ .../dsm/dao/DatasourceDbEntryRepository.java | 75 ++ .../dsm/dao/DatasourceIndexClient.java | 17 + .../dsm/dao/DatasourceIndexClientImpl.java | 207 +++++ .../openaire/dsm/dao/DatasourceSpecs.java | 141 +++ .../openaire/dsm/dao/MongoLoggerClient.java | 14 + .../dsm/dao/MongoLoggerClientImpl.java | 234 +++++ .../openaire/dsm/dao/ObjectStoreClient.java | 9 + .../dsm/dao/ObjectStoreClientImpl.java | 37 + .../openaire/dsm/dao/ResponseUtils.java | 53 ++ .../openaire/dsm/dao/VocabularyClient.java | 14 + .../dsm/dao/VocabularyClientImpl.java | 58 ++ .../dsm/dao/utils/DsmMappingUtils.java | 312 +++++++ .../openaire/dsm/dao/utils/IndexDsInfo.java | 36 + .../dsm/dao/utils/IndexRecordsInfo.java | 49 + .../domain/AggregationHistoryResponse.java | 29 + .../openaire/dsm/domain/ApiDetails.java | 214 +++++ .../dsm/domain/ApiDetailsResponse.java | 24 + .../dsm/domain/ApiIgnoredProperties.java | 72 ++ .../openaire/dsm/domain/ApiParamDetails.java | 24 + .../openaire/dsm/domain/CollectionInfo.java | 25 + .../openaire/dsm/domain/CollectionMode.java | 13 + .../dsm/domain/DatasourceDetailResponse.java | 28 + .../dsm/domain/DatasourceDetails.java | 402 +++++++++ .../dsm/domain/DatasourceDetailsUpdate.java | 240 +++++ .../domain/DatasourceIgnoredProperties.java | 196 ++++ .../openaire/dsm/domain/DatasourceInfo.java | 124 +++ .../dsm/domain/DatasourceResponse.java | 29 + .../dsm/domain/DatasourceSearchResponse.java | 28 + .../dsm/domain/DatasourceSnippet.java | 49 + .../dsm/domain/DatasourceSnippetExtended.java | 170 ++++ .../dsm/domain/DatasourceSnippetResponse.java | 28 + .../openaire/dsm/domain/FilterName.java | 35 + .../openaire/dsm/domain/FilterType.java | 5 + .../dnetlib/openaire/dsm/domain/Header.java | 115 +++ .../dsm/domain/IdentitiesDetails.java | 32 + .../dsm/domain/OrganizationDetails.java | 76 ++ .../domain/OrganizationIgnoredProperties.java | 64 ++ .../dsm/domain/RegisteredDatasourceInfo.java | 95 ++ .../openaire/dsm/domain/RequestFilter.java | 14 + .../openaire/dsm/domain/RequestSort.java | 8 + .../openaire/dsm/domain/RequestSortOrder.java | 8 + .../dnetlib/openaire/dsm/domain/Response.java | 29 + .../openaire/dsm/domain/SimpleResponse.java | 25 + .../dsm/domain/TransformationInfo.java | 15 + .../openaire/dsm/domain/db/ApiDbEntry.java | 28 + .../dsm/domain/db/ApiParamDbEntry.java | 58 ++ .../dsm/domain/db/ApiParamKeyDbEntry.java | 22 + .../openaire/dsm/domain/db/CountryTerm.java | 48 + .../dsm/domain/db/DatasourceApiDbEntry.java | 175 ++++ .../dsm/domain/db/DatasourceDbEntry.java | 30 + .../dsm/domain/db/IdentityDbEntry.java | 20 + .../dsm/domain/db/OrganizationDbEntry.java | 27 + .../dnetlib/openaire/funders/FunderDao.java | 45 + .../openaire/funders/FunderRepository.java | 10 + .../funders/FundersApiController.java | 68 ++ .../openaire/funders/FundersApiException.java | 19 + .../funders/domain/ConversionUtils.java | 88 ++ .../funders/domain/ExtendedFunderDetails.java | 34 + .../funders/domain/FunderDetails.java | 76 ++ .../funders/domain/FundingStream.java | 27 + .../funders/domain/db/FunderDbEntry.java | 116 +++ .../funders/domain/db/FundingPathDbEntry.java | 104 +++ .../dnetlib/openaire/info/InfoController.java | 80 ++ .../eu/dnetlib/openaire/info/JdbcInfoDao.java | 32 + .../openaire/info/JdbcInfoDaoImpl.java | 111 +++ .../openaire/project/ProjectQueryParams.java | 86 ++ .../project/ProjectQueryParamsFactory.java | 47 + .../openaire/project/ProjectsController.java | 229 +++++ .../openaire/project/dao/JdbcApiDao.java | 29 + .../openaire/project/dao/JdbcApiDaoImpl.java | 238 +++++ .../project/dao/ProjectApiRepository.java | 18 + .../project/dao/ProjectTsvRepository.java | 19 + .../openaire/project/dao/ValueCleaner.java | 10 + .../openaire/project/domain/Project.java | 264 ++++++ .../project/domain/db/ProjectApi.java | 152 ++++ .../project/domain/db/ProjectDetails.java | 157 ++++ .../project/domain/db/ProjectTsv.java | 210 +++++ .../openaire/vocabularies/Country.java | 35 + .../dnetlib/openaire/vocabularies/Term.java | 29 + .../openaire/vocabularies/Vocabulary.java | 50 ++ .../src/main/resources/application.properties | 34 + .../src/main/resources/dnet_dsm.data.sql | 176 ++++ .../src/main/resources/dnet_dsm.sql | 510 +++++++++++ .../src/main/resources/dnet_info.sql | 21 + .../openaire/sql/projects_fundings.sql.st | 11 + .../sql/recent_registered_datasources.sql.st | 25 + ...ent_registered_datasources_fromDate.st.sql | 13 + ...tered_datasources_fromDate_typology.st.sql | 14 + .../openaire/st/projects_dspace_header.st | 4 + .../openaire/st/projects_dspace_project.st | 4 + .../openaire/st/projects_dspace_tail.st | 2 + .../dnetlib/openaire/st/projects_eprints.st | 1 + .../xquery/findCommunityContexts.xquery | 3 + .../xquery/findContextProfiles.xquery | 2 + .../xquery/findContextProfilesByType.xquery | 3 + .../openaire/xquery/findFunderContexts.xquery | 3 + .../openaire/xquery/findIndexDsInfo.xquery | 11 + .../openaire/xquery/findObjectStore.xquery | 3 + .../openaire/xquery/findSolrIndexUrl.xquery | 1 + .../openaire/xquery/getRepoProfile.xquery | 3 + .../src/main/resources/global.properties | 105 +++ .../src/main/resources/logback-spring.xml | 13 + .../templates/dnetprofiles/repository.xml | 79 ++ .../community/CommunityApiControllerTest.java | 63 ++ .../SelectionCriteriaTest.java | 77 ++ .../context/ContextMappingUtilsTest.java | 42 + .../dsm/DatasourceApiControllerTest.java | 70 ++ .../funders/FunderContextClientTest.java | 50 ++ .../openaire/info/InfoControllerTest.java | 119 +++ .../project/ProjectQueryParamsTest.java | 41 + .../project/ProjectsControllerTest.java | 131 +++ .../project/domain/ProjectDetailsTest.java | 26 + .../openaire/context/community_test.xml | 838 ++++++++++++++++++ .../dnetlib/openaire/exporter/FCT-project.xml | 12 + .../dnetlib/openaire/exporter/FP7-project.xml | 14 + .../openaire/exporter/MSES-project.xml | 11 + .../dnetlib/openaire/exporter/SFI-project.xml | 11 + .../dnetlib/openaire/exporter/WT-project.xml | 11 + .../eu/dnetlib/openaire/funders/ec-fp7.xml | 438 +++++++++ .../sql/expected_projects_fundings.sql.st | 11 + .../src/test/resources/logback-test.xml | 16 + apps/pom.xml | 1 + 178 files changed, 15766 insertions(+) create mode 100644 apps/dnet-exporter-api/README.md create mode 100644 apps/dnet-exporter-api/pom.xml create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/CacheCustomizer.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/DNetOpenaireExporterApplication.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/DNetOpenaireExporterBeanFactory.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/OpenaireExporterConfig.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/SwaggerConfig.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/AbstractExporterController.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ConverterTextArray.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ExporterConstants.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/GenericArrayUserType.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClient.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClientImpl.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/OperationManager.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/RFC3339DateFormat.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/Utils.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityApiController.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityApiCore.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityClient.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityClientImpl.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityCommon.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityConstants.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityContentprovider.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityDetails.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityException.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityMappingUtils.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityNotFoundException.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityOpenAIRECommunities.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityOrganization.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityProject.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityStatus.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunitySummary.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityWritableProperties.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityZenodoCommunity.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/selectioncriteria/Constraint.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/selectioncriteria/Constraints.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/selectioncriteria/SelectionCriteria.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/Category.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/CategorySummary.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/Concept.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ConceptSummary.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/Context.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextApiController.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextApiCore.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextException.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextMappingUtils.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextNotFoundException.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextSummary.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/Param.java create mode 100755 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmApiController.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmCore.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ApiDbEntryRepository.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/CountryTermRepository.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceApiDbEntryRepository.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDao.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDaoImpl.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDbEntryRepository.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceIndexClient.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceIndexClientImpl.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceSpecs.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/MongoLoggerClient.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/MongoLoggerClientImpl.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ObjectStoreClient.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ObjectStoreClientImpl.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ResponseUtils.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/VocabularyClient.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/VocabularyClientImpl.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/utils/DsmMappingUtils.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/utils/IndexDsInfo.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/utils/IndexRecordsInfo.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/AggregationHistoryResponse.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/ApiDetails.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/ApiDetailsResponse.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/ApiIgnoredProperties.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/ApiParamDetails.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/CollectionInfo.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/CollectionMode.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetailResponse.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetails.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetailsUpdate.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceIgnoredProperties.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceInfo.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceResponse.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceSearchResponse.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceSnippet.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceSnippetExtended.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceSnippetResponse.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/FilterName.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/FilterType.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/Header.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/IdentitiesDetails.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/OrganizationDetails.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/OrganizationIgnoredProperties.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/RegisteredDatasourceInfo.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/RequestFilter.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/RequestSort.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/RequestSortOrder.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/Response.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/SimpleResponse.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/TransformationInfo.java create mode 100755 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/ApiDbEntry.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/ApiParamDbEntry.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/ApiParamKeyDbEntry.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/CountryTerm.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/DatasourceApiDbEntry.java create mode 100755 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/DatasourceDbEntry.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/IdentityDbEntry.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/OrganizationDbEntry.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderDao.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderRepository.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiController.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiException.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/ConversionUtils.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/ExtendedFunderDetails.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/FunderDetails.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/FundingStream.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FundingPathDbEntry.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/info/InfoController.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/info/JdbcInfoDao.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/info/JdbcInfoDaoImpl.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/ProjectQueryParams.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/ProjectQueryParamsFactory.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/ProjectsController.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/dao/JdbcApiDao.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/dao/JdbcApiDaoImpl.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/dao/ProjectApiRepository.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/dao/ProjectTsvRepository.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/dao/ValueCleaner.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/domain/Project.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/domain/db/ProjectApi.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/domain/db/ProjectDetails.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/domain/db/ProjectTsv.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/vocabularies/Country.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/vocabularies/Term.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/vocabularies/Vocabulary.java create mode 100644 apps/dnet-exporter-api/src/main/resources/application.properties create mode 100644 apps/dnet-exporter-api/src/main/resources/dnet_dsm.data.sql create mode 100644 apps/dnet-exporter-api/src/main/resources/dnet_dsm.sql create mode 100644 apps/dnet-exporter-api/src/main/resources/dnet_info.sql create mode 100644 apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/projects_fundings.sql.st create mode 100644 apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources.sql.st create mode 100644 apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate.st.sql create mode 100644 apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate_typology.st.sql create mode 100644 apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/st/projects_dspace_header.st create mode 100644 apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/st/projects_dspace_project.st create mode 100644 apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/st/projects_dspace_tail.st create mode 100644 apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/st/projects_eprints.st create mode 100644 apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findCommunityContexts.xquery create mode 100644 apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findContextProfiles.xquery create mode 100644 apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findContextProfilesByType.xquery create mode 100644 apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findFunderContexts.xquery create mode 100644 apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findIndexDsInfo.xquery create mode 100644 apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findObjectStore.xquery create mode 100644 apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findSolrIndexUrl.xquery create mode 100644 apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/getRepoProfile.xquery create mode 100644 apps/dnet-exporter-api/src/main/resources/global.properties create mode 100644 apps/dnet-exporter-api/src/main/resources/logback-spring.xml create mode 100644 apps/dnet-exporter-api/src/main/resources/templates/dnetprofiles/repository.xml create mode 100644 apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/community/CommunityApiControllerTest.java create mode 100644 apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/community/selectioncriteria/SelectionCriteriaTest.java create mode 100644 apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/context/ContextMappingUtilsTest.java create mode 100644 apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/dsm/DatasourceApiControllerTest.java create mode 100644 apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/funders/FunderContextClientTest.java create mode 100644 apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/info/InfoControllerTest.java create mode 100644 apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/project/ProjectQueryParamsTest.java create mode 100644 apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/project/ProjectsControllerTest.java create mode 100644 apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/project/domain/ProjectDetailsTest.java create mode 100644 apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/context/community_test.xml create mode 100644 apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/exporter/FCT-project.xml create mode 100644 apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/exporter/FP7-project.xml create mode 100644 apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/exporter/MSES-project.xml create mode 100644 apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/exporter/SFI-project.xml create mode 100644 apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/exporter/WT-project.xml create mode 100644 apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/funders/ec-fp7.xml create mode 100644 apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/sql/expected_projects_fundings.sql.st create mode 100644 apps/dnet-exporter-api/src/test/resources/logback-test.xml diff --git a/apps/dnet-exporter-api/README.md b/apps/dnet-exporter-api/README.md new file mode 100644 index 00000000..a0a8d3f5 --- /dev/null +++ b/apps/dnet-exporter-api/README.md @@ -0,0 +1,6 @@ +SpringBoot application implementing OpenAIRE REST API to manage +- Datasources +- Contexts +- Communities +- Funders +- Projects \ No newline at end of file diff --git a/apps/dnet-exporter-api/pom.xml b/apps/dnet-exporter-api/pom.xml new file mode 100644 index 00000000..9d1433d6 --- /dev/null +++ b/apps/dnet-exporter-api/pom.xml @@ -0,0 +1,149 @@ + + + + eu.dnetlib.dhp + apps + 3.2.4-SNAPSHOT + ../ + + + 4.0.0 + dnet-exporter-api + jar + + + + + + org.springframework.boot + spring-boot-starter-jdbc + + + org.springframework.boot + spring-boot-starter-data-jpa + + + org.springframework.boot + spring-boot-starter-cache + + + eu.dnetlib + cnr-rmi-api + [2.0.0,3.0.0) + + + org.apache.cxf + cxf-rt-transports-http + 3.1.5 + + + eu.dnetlib + cnr-service-common + [2.0.0,3.0.0) + + + eu.dnetlib + dnet-openaireplus-mapping-utils + [6.3.0,7.0.0) + + + com.sun.jersey + jersey-client + + + eu.dnetlib + dnet-hadoop-commons + + + + + eu.dnetlib + dnet-objectstore-rmi + [2.0.0,3.0.0) + + + org.apache.solr + solr-solrj + 7.5.0 + + + + org.postgresql + postgresql + + + + org.apache.commons + commons-dbcp2 + + + + org.antlr + stringtemplate + 3.2.1 + + + org.apache.httpcomponents + httpclient + + + org.apache.httpcomponents + httpcore + + + org.apache.httpcomponents + httpmime + + + net.sf.supercsv + super-csv + 2.4.0 + + + com.google.code.gson + gson + + + + com.fasterxml.jackson.datatype + jackson-datatype-joda + + + joda-time + joda-time + 2.8.2 + + + + org.mongodb + mongo-java-driver + 3.4.2 + + + + eu.dnetlib + dnet-datasource-manager-common + [1.1.0,2.0.0) + + + + org.springframework.boot + spring-boot-starter-test + test + + + + + + + org.apache.maven.plugins + maven-help-plugin + + + org.springframework.boot + spring-boot-maven-plugin + + + + + diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/CacheCustomizer.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/CacheCustomizer.java new file mode 100644 index 00000000..cac25806 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/CacheCustomizer.java @@ -0,0 +1,29 @@ +package eu.dnetlib; + +import org.springframework.boot.autoconfigure.cache.CacheManagerCustomizer; +import org.springframework.cache.concurrent.ConcurrentMapCacheManager; +import org.springframework.stereotype.Component; + +import static java.util.Arrays.asList; + +@Component +public class CacheCustomizer implements CacheManagerCustomizer { + + @Override + public void customize(final ConcurrentMapCacheManager cacheManager) { + cacheManager.setCacheNames( + asList( + "fundingpath-ids", + "indexdsinfo-cache", + "objectstoreid-cache", + "context-cache", + "context-cache-funder", + "context-cache-community", + "dsm-aggregationhistory-cache", + "dsm-firstharvestdate-cache", + "vocabularies-cache", + "community-cache", + "info")); + } + +} \ No newline at end of file diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/DNetOpenaireExporterApplication.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/DNetOpenaireExporterApplication.java new file mode 100644 index 00000000..1152802e --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/DNetOpenaireExporterApplication.java @@ -0,0 +1,31 @@ +package eu.dnetlib; + +import java.io.IOException; +import javax.servlet.http.HttpServletResponse; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.solr.SolrAutoConfiguration; +import org.springframework.cache.annotation.EnableCaching; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@EnableCaching +@RestController +@EnableScheduling +@SpringBootApplication +@EnableAutoConfiguration(exclude = { SolrAutoConfiguration.class }) +public class DNetOpenaireExporterApplication { + + @RequestMapping(value = { "/", "/docs" }) + public void index(final HttpServletResponse response) throws IOException { + response.sendRedirect("swagger-ui.html"); + } + + public static void main(String[] args) throws Exception { + SpringApplication.run(DNetOpenaireExporterApplication.class, args); + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/DNetOpenaireExporterBeanFactory.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/DNetOpenaireExporterBeanFactory.java new file mode 100644 index 00000000..32e1940b --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/DNetOpenaireExporterBeanFactory.java @@ -0,0 +1,141 @@ +package eu.dnetlib; + +import com.google.common.collect.Lists; +import com.mongodb.MongoClient; +import com.mongodb.MongoClientOptions; +import com.mongodb.ServerAddress; +import eu.dnetlib.OpenaireExporterConfig.Jdbc; +import eu.dnetlib.data.objectstore.rmi.ObjectStoreService; +import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; +import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService; +import io.micrometer.core.instrument.ImmutableTag; +import io.micrometer.core.instrument.Metrics; +import org.apache.commons.dbcp2.BasicDataSource; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.cxf.endpoint.Client; +import org.apache.cxf.frontend.ClientProxy; +import org.apache.cxf.jaxws.JaxWsProxyFactoryBean; +import org.apache.cxf.transport.http.HTTPConduit; +import org.apache.cxf.transports.http.configuration.HTTPClientPolicy; +import org.apache.maven.model.Model; +import org.apache.maven.model.io.xpp3.MavenXpp3Reader; +import org.codehaus.plexus.util.xml.pull.XmlPullParserException; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.io.ClassPathResource; +import org.springframework.core.io.ResourceLoader; + +import javax.annotation.PostConstruct; +import javax.sql.DataSource; +import java.io.IOException; +import java.io.InputStreamReader; + +/** + * Created by claudio on 07/07/2017. + */ +@Configuration +public class DNetOpenaireExporterBeanFactory { + + private static final Log log = LogFactory.getLog(DNetOpenaireExporterBeanFactory.class); + + @Autowired + private ResourceLoader resourceLoader; + + @Value("pom.xml") + private ClassPathResource pom; + + @PostConstruct + public void init() { + final MavenXpp3Reader reader = new MavenXpp3Reader(); + try { + final Model model = reader.read(new InputStreamReader(pom.getInputStream())); + + log.info(String.format("registering metric for %s", model.getArtifactId())); + Metrics.gauge("micrometer_info", Lists.newArrayList( + new ImmutableTag("component", model.getGroupId()+":"+model.getArtifactId()), + new ImmutableTag("version", model.getVersion()), + new ImmutableTag("scmtag", model.getScm().getTag())), 1); + } catch (IOException | XmlPullParserException e) { + log.error(e); + } + } + + @Autowired + private OpenaireExporterConfig config; + + @Bean + public ISLookUpService getLookUpService() { + return getServiceStub(ISLookUpService.class, config.getIsLookupUrl()); + } + + @Bean + public ObjectStoreService getObjectStoreService() { + return getServiceStub(ObjectStoreService.class, config.getObjectStoreServiceUrl()); + } + + @Bean + public ISRegistryService getRegistryService() { + return getServiceStub(ISRegistryService.class, config.getIsRegistryServiceUrl()); + } + + @SuppressWarnings("unchecked") + private T getServiceStub(final Class clazz, final String endpoint) { + log.info(String.format("Initializing service stub %s, endpoint %s", clazz.toString(),endpoint)); + final JaxWsProxyFactoryBean jaxWsProxyFactory = new JaxWsProxyFactoryBean(); + jaxWsProxyFactory.setServiceClass(clazz); + jaxWsProxyFactory.setAddress(endpoint); + + final T service = (T) jaxWsProxyFactory.create(); + + Client client = ClientProxy.getClient(service); + if (client != null) { + HTTPConduit conduit = (HTTPConduit) client.getConduit(); + HTTPClientPolicy policy = new HTTPClientPolicy(); + + log.info(String.format("setting connectTimeout to %s, receiveTimeout to %s for service %s", + config.getCxfClientConnectTimeout(), + config.getCxfClientReceiveTimeout(), + clazz.getCanonicalName())); + + policy.setConnectionTimeout(config.getCxfClientConnectTimeout()); + policy.setReceiveTimeout(config.getCxfClientReceiveTimeout()); + conduit.setClient(policy); + } + + return service; + } + + @Bean + public DataSource getSqlDataSource() { + final Jdbc jdbc = config.getJdbc(); + return getDatasource( + jdbc.getDriverClassName(), + jdbc.getUrl(), + jdbc.getUser(), + jdbc.getPwd(), + jdbc.getMinIdle(), + jdbc.getMaxRows()); + } + + private BasicDataSource getDatasource(String driverClassName, String jdbcUrl, String jdbcUser, String jdbcPwd, int jdbcMinIdle, int jdbcMaxIdle) { + final BasicDataSource d = new BasicDataSource(); + d.setDriverClassName(driverClassName); + d.setUrl(jdbcUrl); + d.setUsername(jdbcUser); + d.setPassword(jdbcPwd); + d.setMinIdle(jdbcMinIdle); + d.setMaxIdle(jdbcMaxIdle); + return d; + } + + @Bean + public MongoClient getMongoClient() { + return new MongoClient( + new ServerAddress(config.getDatasource().getMongoHost(), config.getDatasource().getMongoPort()), + MongoClientOptions.builder().connectionsPerHost(config.getDatasource().getMongoConnectionsPerHost()).build()); + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/OpenaireExporterConfig.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/OpenaireExporterConfig.java new file mode 100644 index 00000000..87a54dd4 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/OpenaireExporterConfig.java @@ -0,0 +1,548 @@ +package eu.dnetlib; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.PropertySource; +import org.springframework.core.io.ClassPathResource; +import org.springframework.core.io.Resource; + +/** + * Created by Alessia Bardi on 31/03/17. + * + * @author Alessia Bardi, Claudio Atzori + */ +@Configuration +@PropertySource("classpath:global.properties") +@ConfigurationProperties(prefix = "openaire.exporter") +public class OpenaireExporterConfig { + + // ISLOOKUP + private ClassPathResource findSolrIndexUrl; + private ClassPathResource findIndexDsInfo; + private ClassPathResource findObjectStore; + private ClassPathResource findFunderContexts; + private ClassPathResource findCommunityContexts; + private ClassPathResource findContextProfiles; + private ClassPathResource findContextProfilesByType; + private ClassPathResource getRepoProfile; + + @Value("${openaire.exporter.contentLoadQuery}") + private String contentLoadQuery; + + private String isLookupUrl; + private String objectStoreServiceUrl; + private String isRegistryServiceUrl; + + private int requestWorkers = 100; + private int requestTimeout = 10; + + private int cxfClientConnectTimeout = 120; + private int cxfClientReceiveTimeout = 120; + + private Datasource datasource; + private Project project; + private Jdbc jdbc; + private Swagger swaggerDsm; + private Swagger swaggerProjects; + private Swagger swaggerFunders; + private Swagger swaggerCommunities; + private Swagger swaggerContexts; + private Swagger swaggerInfo; + + private Vocabularies vocabularies; + + public static class Datasource { + // MONGODB + private String mongoHost; + private int mongoPort; + private String mongoCollectionName; + private String mongoDbName; + private int mongoConnectionsPerHost; + private int mongoQueryLimit; + + public String getMongoHost() { + return mongoHost; + } + + public void setMongoHost(final String mongoHost) { + this.mongoHost = mongoHost; + } + + public int getMongoPort() { + return mongoPort; + } + + public void setMongoPort(final int mongoPort) { + this.mongoPort = mongoPort; + } + + public String getMongoCollectionName() { + return mongoCollectionName; + } + + public void setMongoCollectionName(final String mongoCollectionName) { + this.mongoCollectionName = mongoCollectionName; + } + + public String getMongoDbName() { + return mongoDbName; + } + + public void setMongoDbName(final String mongoDbName) { + this.mongoDbName = mongoDbName; + } + + public int getMongoConnectionsPerHost() { + return mongoConnectionsPerHost; + } + + public void setMongoConnectionsPerHost(final int mongoConnectionsPerHost) { + this.mongoConnectionsPerHost = mongoConnectionsPerHost; + } + + public int getMongoQueryLimit() { + return mongoQueryLimit; + } + + public void setMongoQueryLimit(final int mongoQueryLimit) { + this.mongoQueryLimit = mongoQueryLimit; + } + } + + public static class Project { + + private int flushSize; + private String tsvFields; + private Resource projectsFundingQueryTemplate; + private Resource dspaceTemplate; + private Resource dspaceHeadTemplate; + private Resource dspaceTailTemplate; + private Resource eprintsTemplate; + + public int getFlushSize() { + return flushSize; + } + + public void setFlushSize(final int flushSize) { + this.flushSize = flushSize; + } + + public String getTsvFields() { + return tsvFields; + } + + public void setTsvFields(final String tsvFields) { + this.tsvFields = tsvFields; + } + + public Resource getProjectsFundingQueryTemplate() { + return projectsFundingQueryTemplate; + } + + public void setProjectsFundingQueryTemplate(final Resource projectsFundingQueryTemplate) { + this.projectsFundingQueryTemplate = projectsFundingQueryTemplate; + } + + public Resource getDspaceTemplate() { + return dspaceTemplate; + } + + public void setDspaceTemplate(final Resource dspaceTemplate) { + this.dspaceTemplate = dspaceTemplate; + } + + public Resource getDspaceHeadTemplate() { + return dspaceHeadTemplate; + } + + public void setDspaceHeadTemplate(final Resource dspaceHeadTemplate) { + this.dspaceHeadTemplate = dspaceHeadTemplate; + } + + public Resource getDspaceTailTemplate() { + return dspaceTailTemplate; + } + + public void setDspaceTailTemplate(final Resource dspaceTailTemplate) { + this.dspaceTailTemplate = dspaceTailTemplate; + } + + public Resource getEprintsTemplate() { + return eprintsTemplate; + } + + public void setEprintsTemplate(final Resource eprintsTemplate) { + this.eprintsTemplate = eprintsTemplate; + } + } + + public static class Jdbc { + + // JDBC + @Value("${spring.datasource.driverClassName}") + private String driverClassName; + + private String url; + private String user; + private String pwd; + private int minIdle; + private int maxidle; + private int maxRows; + + public String getDriverClassName() { + return driverClassName; + } + + public String getUrl() { + return url; + } + + public void setUrl(final String url) { + this.url = url; + } + + public String getUser() { + return user; + } + + public void setUser(final String user) { + this.user = user; + } + + public String getPwd() { + return pwd; + } + + public void setPwd(final String pwd) { + this.pwd = pwd; + } + + public int getMinIdle() { + return minIdle; + } + + public void setMinIdle(final int minIdle) { + this.minIdle = minIdle; + } + + public int getMaxidle() { + return maxidle; + } + + public void setMaxidle(final int maxidle) { + this.maxidle = maxidle; + } + + public int getMaxRows() { + return maxRows; + } + + public void setMaxRows(final int maxRows) { + this.maxRows = maxRows; + } + } + + public static class Swagger { + private String apiTitle; + private String apiDescription; + private String apiLicense; + private String apiLicenseUrl; + private String apiContactName; + private String apiContactUrl; + private String apiContactEmail; + + public String getApiTitle() { + return apiTitle; + } + + public void setApiTitle(final String apiTitle) { + this.apiTitle = apiTitle; + } + + public String getApiDescription() { + return apiDescription; + } + + public void setApiDescription(final String apiDescription) { + this.apiDescription = apiDescription; + } + + public String getApiLicense() { + return apiLicense; + } + + public void setApiLicense(final String apiLicense) { + this.apiLicense = apiLicense; + } + + public String getApiLicenseUrl() { + return apiLicenseUrl; + } + + public void setApiLicenseUrl(final String apiLicenseUrl) { + this.apiLicenseUrl = apiLicenseUrl; + } + + public String getApiContactName() { + return apiContactName; + } + + public void setApiContactName(final String apiContactName) { + this.apiContactName = apiContactName; + } + + public String getApiContactUrl() { + return apiContactUrl; + } + + public void setApiContactUrl(final String apiContactUrl) { + this.apiContactUrl = apiContactUrl; + } + + public String getApiContactEmail() { + return apiContactEmail; + } + + public void setApiContactEmail(final String apiContactEmail) { + this.apiContactEmail = apiContactEmail; + } + } + + public static class Vocabularies { + + private String baseUrl; + + private String countriesEndpoint; + + private String datasourceTypologiesEndpoint; + + public String getCountriesEndpoint() { + return countriesEndpoint; + } + + public void setCountriesEndpoint(final String countriesEndpoint) { + this.countriesEndpoint = countriesEndpoint; + } + + public String getBaseUrl() { + return baseUrl; + } + + public void setBaseUrl(final String baseUrl) { + this.baseUrl = baseUrl; + } + + public String getDatasourceTypologiesEndpoint() { + return datasourceTypologiesEndpoint; + } + + public void setDatasourceTypologiesEndpoint(final String datasourceTypologiesEndpoint) { + this.datasourceTypologiesEndpoint = datasourceTypologiesEndpoint; + } + } + + public ClassPathResource getFindSolrIndexUrl() { + return findSolrIndexUrl; + } + + public void setFindSolrIndexUrl(final ClassPathResource findSolrIndexUrl) { + this.findSolrIndexUrl = findSolrIndexUrl; + } + + public ClassPathResource getFindIndexDsInfo() { + return findIndexDsInfo; + } + + public ClassPathResource getFindObjectStore() { + return findObjectStore; + } + + public void setFindObjectStore(final ClassPathResource findObjectStore) { + this.findObjectStore = findObjectStore; + } + + public void setFindIndexDsInfo(final ClassPathResource findIndexDsInfo) { + this.findIndexDsInfo = findIndexDsInfo; + } + + public ClassPathResource getFindFunderContexts() { + return findFunderContexts; + } + + public void setFindFunderContexts(final ClassPathResource findFunderContexts) { + this.findFunderContexts = findFunderContexts; + } + + public ClassPathResource getFindCommunityContexts() { + return findCommunityContexts; + } + + public ClassPathResource getFindContextProfiles() { + return findContextProfiles; + } + + public ClassPathResource getFindContextProfilesByType() { + return findContextProfilesByType; + } + + public void setFindContextProfiles(final ClassPathResource findContextProfiles) { + this.findContextProfiles = findContextProfiles; + } + + public void setFindContextProfilesByType(ClassPathResource findContextProfilesByType) { + this.findContextProfilesByType = findContextProfilesByType; + } + + public void setFindCommunityContexts(final ClassPathResource findCommunityContexts) { + this.findCommunityContexts = findCommunityContexts; + } + + public ClassPathResource getGetRepoProfile() { + return getRepoProfile; + } + + public void setGetRepoProfile(final ClassPathResource getRepoProfile) { + this.getRepoProfile = getRepoProfile; + } + + public String getContentLoadQuery() { + return contentLoadQuery; + } + + public void setContentLoadQuery(String contentLoadQuery) { + this.contentLoadQuery = contentLoadQuery; + } + + public String getIsLookupUrl() { + return isLookupUrl; + } + + public void setIsLookupUrl(final String isLookupUrl) { + this.isLookupUrl = isLookupUrl; + } + + public String getObjectStoreServiceUrl() { + return objectStoreServiceUrl; + } + + public void setObjectStoreServiceUrl(final String objectStoreServiceUrl) { + this.objectStoreServiceUrl = objectStoreServiceUrl; + } + + public String getIsRegistryServiceUrl() { + return isRegistryServiceUrl; + } + + public void setIsRegistryServiceUrl(final String isRegistryServiceUrl) { + this.isRegistryServiceUrl = isRegistryServiceUrl; + } + + public int getRequestWorkers() { + return requestWorkers; + } + + public void setRequestWorkers(final int requestWorkers) { + this.requestWorkers = requestWorkers; + } + + public int getRequestTimeout() { + return requestTimeout; + } + + public void setRequestTimeout(final int requestTimeout) { + this.requestTimeout = requestTimeout; + } + + public int getCxfClientConnectTimeout() { + return cxfClientConnectTimeout; + } + + public void setCxfClientConnectTimeout(int cxfClientConnectTimeout) { + this.cxfClientConnectTimeout = cxfClientConnectTimeout; + } + + public int getCxfClientReceiveTimeout() { + return cxfClientReceiveTimeout; + } + + public void setCxfClientReceiveTimeout(int cxfClientReceiveTimeout) { + this.cxfClientReceiveTimeout = cxfClientReceiveTimeout; + } + + public Datasource getDatasource() { + return datasource; + } + + public void setDatasource(final Datasource datasource) { + this.datasource = datasource; + } + + public Project getProject() { + return project; + } + + public void setProject(final Project project) { + this.project = project; + } + + public Jdbc getJdbc() { + return jdbc; + } + + public void setJdbc(final Jdbc jdbc) { + this.jdbc = jdbc; + } + + public Swagger getSwaggerDsm() { + return swaggerDsm; + } + + public void setSwaggerDsm(final Swagger swaggerDsm) { + this.swaggerDsm = swaggerDsm; + } + + public Swagger getSwaggerProjects() { + return swaggerProjects; + } + + public void setSwaggerProjects(final Swagger swaggerProjects) { + this.swaggerProjects = swaggerProjects; + } + + public Swagger getSwaggerFunders() { + return swaggerFunders; + } + + public void setSwaggerFunders(final Swagger swaggerFunders) { + this.swaggerFunders = swaggerFunders; + } + + public Swagger getSwaggerCommunities() { + return swaggerCommunities; + } + + public void setSwaggerCommunities(final Swagger swaggerCommunities) { + this.swaggerCommunities = swaggerCommunities; + } + + public Swagger getSwaggerContexts() { + return swaggerContexts; + } + + public void setSwaggerContexts(final Swagger swaggerContexts) { + this.swaggerContexts = swaggerContexts; + } + + public Swagger getSwaggerInfo() { return swaggerInfo; } + + public void setSwaggerInfo(Swagger swaggerInfo) { this.swaggerInfo = swaggerInfo; } + + public Vocabularies getVocabularies() { + return vocabularies; + } + + public void setVocabularies(final Vocabularies vocabularies) { + this.vocabularies = vocabularies; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/SwaggerConfig.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/SwaggerConfig.java new file mode 100644 index 00000000..3276bf1c --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/SwaggerConfig.java @@ -0,0 +1,115 @@ +package eu.dnetlib; + +import eu.dnetlib.OpenaireExporterConfig.Swagger; +import eu.dnetlib.openaire.community.CommunityApiController; +import eu.dnetlib.openaire.context.ContextApiController; +import eu.dnetlib.openaire.dsm.DsmApiController; +import eu.dnetlib.openaire.funders.FundersApiController; +import eu.dnetlib.openaire.info.InfoController; +import eu.dnetlib.openaire.project.ProjectsController; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import springfox.documentation.builders.ApiInfoBuilder; +import springfox.documentation.service.ApiInfo; +import springfox.documentation.service.Contact; +import springfox.documentation.spi.DocumentationType; +import springfox.documentation.spring.web.plugins.Docket; +import springfox.documentation.swagger2.annotations.EnableSwagger2; + +import static springfox.documentation.builders.RequestHandlerSelectors.basePackage; + +@Configuration +@EnableSwagger2 +public class SwaggerConfig { + + private static final Log log = LogFactory.getLog(SwaggerConfig.class); + + public static final String V1 = "1.0.0"; + + @Autowired + private OpenaireExporterConfig config; + + @Bean + public Docket dsm() { + return _docket( + "Datasource Manager", + DsmApiController.class.getPackage().getName(), + config.getSwaggerDsm(), + V1); + } + + @Bean + public Docket projects() { + return _docket( + "OpenAIRE Projects", + ProjectsController.class.getPackage().getName(), + config.getSwaggerProjects(), + V1); + } + + @Bean + public Docket funders() { + return _docket( + "OpenAIRE Funders", + FundersApiController.class.getPackage().getName(), + config.getSwaggerFunders(), + V1); + } + + @Bean + public Docket communities() { + return _docket( + "OpenAIRE Communities", + CommunityApiController.class.getPackage().getName(), + config.getSwaggerCommunities(), + V1); + } + + @Bean + public Docket contexts() { + return _docket( + "OpenAIRE Contexts", + ContextApiController.class.getPackage().getName(), + config.getSwaggerCommunities(), + V1); + } + + @Bean + public Docket info() { + return _docket( + "OpenAIRE Info", + InfoController.class.getPackage().getName(), + config.getSwaggerInfo(), + V1); + } + + private Docket _docket(final String groupName, final String controllerPackage, final Swagger swag, final String version) { + return new Docket(DocumentationType.SWAGGER_2) + .groupName(groupName) + .select() + .apis(basePackage(controllerPackage)) + .build() + .directModelSubstitute(org.joda.time.LocalDate.class, java.sql.Date.class) + .directModelSubstitute(org.joda.time.DateTime.class, java.util.Date.class) + .apiInfo(apiInfo(swag, version)); + } + + private ApiInfo apiInfo(final Swagger swag, final String version) { + return new ApiInfoBuilder() + .title(swag.getApiTitle()) + .description(swag.getApiDescription()) + .license(swag.getApiLicense()) + .licenseUrl(swag.getApiLicenseUrl()) + .termsOfServiceUrl("") + .version(version) + .contact(new Contact( + swag.getApiContactName(), + swag.getApiContactUrl(), + swag.getApiContactEmail())) + .build(); + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/AbstractExporterController.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/AbstractExporterController.java new file mode 100644 index 00000000..c11e15f0 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/AbstractExporterController.java @@ -0,0 +1,100 @@ +package eu.dnetlib.openaire.common; + +import java.util.List; +import java.util.stream.Collectors; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import eu.dnetlib.enabling.datasources.common.DsmException; +import eu.dnetlib.enabling.datasources.common.DsmForbiddenException; +import eu.dnetlib.enabling.datasources.common.DsmNotFoundException; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.MethodArgumentNotValidException; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.ResponseStatus; + +/** + * Created by claudio on 18/07/2017. + */ +public abstract class AbstractExporterController { + + private static final Log log = LogFactory.getLog(AbstractExporterController.class); // NOPMD by marko on 11/24/08 5:02 PM + + @ResponseBody + @ExceptionHandler({DsmException.class}) + @ResponseStatus(value = HttpStatus.INTERNAL_SERVER_ERROR) + public ErrorMessage handleDSMException(final Exception e) { + return _handleError(e); + } + + @ResponseBody + @ExceptionHandler(DsmForbiddenException.class) + @ResponseStatus(value = HttpStatus.FORBIDDEN) + public ErrorMessage handleForbiddenException(final Exception e) { + return _handleError(e); + } + + @ResponseBody + @ExceptionHandler({DsmNotFoundException.class}) + @ResponseStatus(value = HttpStatus.NOT_FOUND) + public ErrorMessage handleNotFoundException(final Exception e) { + return _handleError(e); + } + + @ResponseBody + @ExceptionHandler(MethodArgumentNotValidException.class) + @ResponseStatus(HttpStatus.BAD_REQUEST) + public List processValidationError(final MethodArgumentNotValidException e) { + return e.getBindingResult() + .getFieldErrors().stream() + .map(fe -> new ErrorMessage( + String.format("field '%s'", fe.getField()), + String.format("rejected value '%s'", fe.getRejectedValue()), + fe.getDefaultMessage())) + .collect(Collectors.toList()); + } + + private ErrorMessage _handleError(final Exception e) { + log.error(e); + if (StringUtils.containsIgnoreCase(ExceptionUtils.getRootCauseMessage(e), "Broken pipe")) { + return null; //socket is closed, cannot return any response + } else { + return new ErrorMessage(e); + } + } + + @JsonAutoDetect + public class ErrorMessage { + + private final String message; + private final String details; + private final String stacktrace; + + public ErrorMessage(final Exception e) { + this(e.getMessage(), "", ExceptionUtils.getStackTrace(e)); + } + + public ErrorMessage(final String message, final String details, final String stacktrace) { + this.message = message; + this.details = details; + this.stacktrace = stacktrace; + } + + public String getMessage() { + return this.message; + } + + public String getStacktrace() { + return this.stacktrace; + } + + public String getDetails() { + return details; + } + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ConverterTextArray.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ConverterTextArray.java new file mode 100644 index 00000000..9f7d04e3 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ConverterTextArray.java @@ -0,0 +1,58 @@ +package eu.dnetlib.openaire.common; + +import java.sql.Array; +import java.sql.Connection; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import javax.persistence.AttributeConverter; +import javax.persistence.Converter; +import javax.sql.DataSource; + +import org.springframework.beans.BeansException; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; + +/** + * Created by claudio on 05/07/2017. + */ +@Converter +public class ConverterTextArray implements AttributeConverter, Array>, ApplicationContextAware { + + private ApplicationContext applicationContext; + + @Override + public Array convertToDatabaseColumn(List attribute) { + + final Map datasources = applicationContext.getBeansOfType(DataSource.class); + DataSource source = datasources.values().stream().findFirst().get(); + + try { + Connection conn = source.getConnection(); + return conn.createArrayOf("text", attribute.toArray()); + + } catch (SQLException e) { + e.printStackTrace(); + } + + return null; + + } + + @Override + public List convertToEntityAttribute(Array dbData) { + try { + return Arrays.stream((Object[]) dbData.getArray()).map(d -> (String) d).collect(Collectors.toList()); + } catch (SQLException e) { + e.printStackTrace(); + return null; + } + } + + @Override + public void setApplicationContext(final ApplicationContext applicationContext) throws BeansException { + this.applicationContext = applicationContext; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ExporterConstants.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ExporterConstants.java new file mode 100644 index 00000000..5fd8b6f7 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ExporterConstants.java @@ -0,0 +1,47 @@ +package eu.dnetlib.openaire.common; + +public class ExporterConstants { + + /* + Tags used to group the operations on the swagger UI + */ + public final static String C = "Community"; + public final static String C_CP = "Community content providers"; + public final static String C_PJ = "Community projects"; + public final static String C_ZC = "Community Zenodo Communities"; + public final static String C_O = "Community Organizations"; + + public final static String DS = "Datasource"; + public final static String API = "Interface"; + public final static String R = "Read"; + public final static String W = "Write"; + + public final static String D = "Deprecated"; + public final static String M = "Management"; + + public final static String DSPACE = "DSpace"; + public final static String EPRINT = "EPrints"; + public final static String TSV = "TSV"; + public final static String STREAMING = "Streaming"; + + public static final String OAI = "oai"; + public static final String SET = "set"; + + // Paths used in the repository profile mapping: Datasource + public static final String ADMIN_INFO = "//CONFIGURATION/ADMIN_INFO"; + public static final String ENGLISH_NAME = "//CONFIGURATION/ENGLISH_NAME"; + public static final String OFFICIAL_NAME = "//CONFIGURATION/OFFICIAL_NAME"; + public static final String TIMEZONE = "//CONFIGURATION/LOCATION/TIMEZONE"; + public static final String LATITUDE = "//CONFIGURATION/LOCATION/LATITUDE"; + public static final String LONGITUDE = "//CONFIGURATION/LOCATION/LONGITUDE"; + public static final String PLATFORM = "//CONFIGURATION/TYPOLOGY"; + public static final String TYPOLOGY = "//CONFIGURATION/DATASOURCE_TYPE"; + public static final String WEBSITE_URL = "//CONFIGURATION/REPOSITORY_WEBPAGE"; + + // Paths used in the repository profile mapping: Interface + public static final String OAI_SET = "/ACCESS_PROTOCOL/@set"; + public static final String BASE_URL = "/BASE_URL"; + public static final String COMPLIANCE = "/@compliance"; + public static final String REMOVABLE = "/@removable"; + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/GenericArrayUserType.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/GenericArrayUserType.java new file mode 100644 index 00000000..51a4d738 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/GenericArrayUserType.java @@ -0,0 +1,106 @@ +package eu.dnetlib.openaire.common; + +import java.io.Serializable; +import java.sql.*; + +import org.hibernate.HibernateException; +import org.hibernate.engine.spi.SharedSessionContractImplementor; +import org.hibernate.usertype.UserType; + +/** + * Created by claudio on 05/07/2017. + */ +public class GenericArrayUserType implements UserType { + + protected static final int[] SQL_TYPES = { Types.ARRAY }; + + private Class typeParameterClass; + + @Override + public Object assemble(Serializable cached, Object owner) throws HibernateException { + return this.deepCopy(cached); + } + + @Override + public Object deepCopy(Object value) throws HibernateException { + return value; + } + + @SuppressWarnings("unchecked") + @Override + public Serializable disassemble(Object value) throws HibernateException { + return (T) this.deepCopy(value); + } + + @Override + public boolean equals(Object x, Object y) throws HibernateException { + + if (x == null) { + return y == null; + } + return x.equals(y); + } + + @Override + public int hashCode(Object x) throws HibernateException { + return x.hashCode(); + } + + @Override + public Object nullSafeGet(final ResultSet resultSet, + final String[] names, + final SharedSessionContractImplementor sharedSessionContractImplementor, + final Object o) + throws HibernateException, SQLException { + if (resultSet.wasNull()) { + return null; + } + if (resultSet.getArray(names[0]) == null) { + return new Integer[0]; + } + + Array array = resultSet.getArray(names[0]); + @SuppressWarnings("unchecked") + T javaArray = (T) array.getArray(); + return javaArray; + } + + @Override + public void nullSafeSet(final PreparedStatement statement, + final Object value, + final int index, + final SharedSessionContractImplementor session) + throws HibernateException, SQLException { + Connection connection = statement.getConnection(); + if (value == null) { + statement.setNull(index, SQL_TYPES[0]); + } else { + @SuppressWarnings("unchecked") + T castObject = (T) value; + Array array = connection.createArrayOf("integer", (Object[]) castObject); + statement.setArray(index, array); + } + } + + @Override + public boolean isMutable() { + return true; + } + + @Override + public Object replace(Object original, Object target, Object owner) throws HibernateException { + return original; + } + + @Override + public Class returnedClass() { + return typeParameterClass; + } + + @Override + public int[] sqlTypes() { + return new int[] { Types.ARRAY }; + } + + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClient.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClient.java new file mode 100644 index 00000000..848654cc --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClient.java @@ -0,0 +1,60 @@ +package eu.dnetlib.openaire.common; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import eu.dnetlib.enabling.datasources.common.DsmException; +import eu.dnetlib.enabling.datasources.common.DsmForbiddenException; +import eu.dnetlib.openaire.context.Context; +import eu.dnetlib.openaire.dsm.dao.utils.IndexDsInfo; +import eu.dnetlib.openaire.dsm.domain.ApiDetails; +import eu.dnetlib.openaire.dsm.domain.DatasourceDetails; + +public interface ISClient { + + IndexDsInfo calculateCurrentIndexDsInfo() throws DsmException; + + String getObjectStoreId(String dsId) throws DsmException; + + Map getFunderContextMap() throws IOException; + + Map getCommunityContextMap() throws IOException; + + Map getContextMap(final List type) throws IOException; + + void updateContextParam(String id, String name, String value); + + void updateContextAttribute(String id, String name, String value); + + void addConcept(String id, String categoryId, String data); + + void removeConcept(String id, String categoryId, String conceptId); + + void updateDatasourceFields(String dsId, Map changes); + + void addAPIAttribute(String dsId, String apiId, Map changes); + + void updateAPIField(String dsId, String apiId, Map changes); + + void registerDS(DatasourceDetails d); + + void registerAPI(ApiDetails api); + + void removeAPI(String apiId) throws DsmForbiddenException; + + void dropCache(); + + /** + * + * @param id id of the concept to be updated (i.e. ni::projects::2) + * @param name name of the attribute to be updated + * @param value new value for the attribute + */ + void updateConceptAttribute(String id, String name, String value); + + void updateConceptParam(String id, String name, String value); + + void updateConceptParamNoEscape(String id, String name, String value); + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClientImpl.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClientImpl.java new file mode 100644 index 00000000..07e28df9 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClientImpl.java @@ -0,0 +1,413 @@ +package eu.dnetlib.openaire.common; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Queue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.function.Function; +import java.util.stream.Collectors; + +import com.google.common.collect.Lists; +import com.google.common.escape.Escaper; +import com.google.common.xml.XmlEscapers; +import eu.dnetlib.OpenaireExporterConfig; +import eu.dnetlib.enabling.datasources.common.DsmException; +import eu.dnetlib.enabling.datasources.common.DsmForbiddenException; +import eu.dnetlib.enabling.datasources.common.DsmRuntimeException; +import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException; +import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; +import eu.dnetlib.enabling.is.registry.rmi.ISRegistryException; +import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService; +import eu.dnetlib.openaire.dsm.dao.utils.IndexDsInfo; +import eu.dnetlib.openaire.context.Context; +import eu.dnetlib.openaire.context.ContextMappingUtils; +import eu.dnetlib.openaire.dsm.domain.ApiDetails; +import eu.dnetlib.openaire.dsm.domain.DatasourceDetails; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.http.HttpStatus; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cache.annotation.CacheEvict; +import org.springframework.cache.annotation.Cacheable; +import org.springframework.core.io.ClassPathResource; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import static eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils.asRepositoryInterfce; +import static eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils.asRepositoryProfile; +import static eu.dnetlib.openaire.common.Utils.escape; + +/** + * Created by claudio on 20/10/2016. + */ +@Component +public class ISClientImpl implements ISClient { + + private static final Log log = LogFactory.getLog(ISClientImpl.class); + + @Autowired + private OpenaireExporterConfig config; + + @Autowired + private ISLookUpService isLookUpService; + + @Autowired + private ISRegistryService isRegistryService; + + @Autowired + private OperationManager operationManager; + + @Override + @Cacheable("indexdsinfo-cache") + public IndexDsInfo calculateCurrentIndexDsInfo() throws DsmException { + log.warn("calculateCurrentIndexDsInfo(): not using cache"); + final String[] arr; + try { + arr = _isLookUp(_getQuery(config.getFindIndexDsInfo())).split("@@@"); + return new IndexDsInfo( + _isLookUp(_getQuery(config.getFindSolrIndexUrl())), + arr[0].trim(), arr[1].trim(), arr[2].trim()); + } catch (IOException | ISLookUpException e) { + throw new DsmException("unable fetch index DS information from IS"); + } + } + + @Override + @Cacheable("objectstoreid-cache") + public String getObjectStoreId(final String dsId) throws DsmException { + log.warn(String.format("getObjectStoreId(%s): not using cache", dsId)); + try { + final String xqueryTemplate = _getQuery(config.getFindObjectStore()); + return _isLookUp(String.format(xqueryTemplate, dsId)); + } catch (IOException | ISLookUpException e) { + throw new DsmException("unble to find objectstore for ds " + dsId); + } + } + + @Override + @Cacheable("context-cache-funder") + public Map getFunderContextMap() throws IOException { + return _processContext(_getQuery(config.getFindFunderContexts())); + } + + @Override + @Cacheable("context-cache-community") + public Map getCommunityContextMap() throws IOException { + return _processContext(_getQuery(config.getFindCommunityContexts())); + } + + @Override + @Cacheable("context-cache") + public Map getContextMap(final List type) throws IOException { + if (Objects.isNull(type) || type.isEmpty()) { + return _processContext(_getQuery(config.getFindContextProfiles())); + } else { + final String xqueryTemplate = _getQuery(config.getFindContextProfilesByType()); + + final String xquery = String.format(xqueryTemplate, type.stream() + .map(t -> String.format("./RESOURCE_PROFILE/BODY/CONFIGURATION/context/@type = '%s'", t)) + .collect(Collectors.joining(" or "))); + + return _processContext(xquery); + } + } + + @Override + @CacheEvict(value = { "context-cache", "context-cache-funder"}, allEntries = true) + public void updateContextParam(final String id, final String name, final String value) { + try { + _quickSeachProfile(getXQuery(id, name, value)); + } catch (ISLookUpException e) { + throw new DsmRuntimeException(String.format("unable update context param [id: %s, name: %s, value: %s]", id, name, value), e); + } + } + + @Override + @CacheEvict(value = { "context-cache", "context-cache-funder"}, allEntries = true) + public void updateContextAttribute(final String id, final String name, final String value) { + final Escaper esc = XmlEscapers.xmlAttributeEscaper(); + try { + _quickSeachProfile(String.format( + "update value collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" + + "/RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s']/@%s with '%s'", id, name, escape(esc, value))); + } catch (ISLookUpException e) { + throw new DsmRuntimeException(String.format("unable update context attribute [id: %s, name: %s, data: %s]", id, name, value), e); + } + } + + @Override + @CacheEvict(value = { "context-cache", "context-cache-funder"}, allEntries = true) + public void addConcept(final String id, final String categoryId, final String data) { + try { + _quickSeachProfile(String.format( + "update insert %s into collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" + + "/RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s']/category[./@id = '%s']", data, id, categoryId)); + } catch (ISLookUpException e) { + throw new DsmRuntimeException(String.format("unable add concept [id: %s, categoryId: %s, data: %s]", id, categoryId, data), e); + } + } + + @Override + @CacheEvict(value = { "context-cache", "context-cache-funder"}, allEntries = true) + public void removeConcept(final String id, final String categoryId, final String conceptId) { + try { + _quickSeachProfile(String.format( + "for $concept in collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" + + "/RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s']" + + "/category[./@id = '%s']/concept[./@id = '%s'] " + + "return update delete $concept", id, categoryId, conceptId)); + } catch (ISLookUpException e) { + throw new DsmRuntimeException(String.format("unable remove concept [id: %s, categoryId: %s, conceptId: %s]", id, categoryId, conceptId), e); + } + } + + @Override + @CacheEvict(value = { "context-cache", "context-cache-community", "context-cache-funder"}, allEntries = true) + public void updateConceptAttribute(String id, String name, String value) { + final Escaper esc = XmlEscapers.xmlAttributeEscaper(); + try { + _quickSeachProfile(String.format( + "update value collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" + + "/RESOURCE_PROFILE/BODY/CONFIGURATION/context/category/concept[./@id = '%s']/@%s with '%s'", id, name, escape(esc, value))); + } catch (ISLookUpException e) { + throw new DsmRuntimeException(String.format("unable update concept attribute [id: %s, name: %s, value: %s]", id, name, value), e); + } + + } + + @Override + @CacheEvict(value = { "context-cache", "context-cache-funder"}, allEntries = true) + public void updateConceptParam(String id, String name, String value) { + try { + _quickSeachProfile(getConceptXQuery(id, name, value)); + } catch (ISLookUpException e) { + throw new DsmRuntimeException(String.format("unable update concept param [id: %s, name: %s, value: %s]", id, name, value), e); + } + } + + @Override + @CacheEvict(value = { "context-cache", "context-cache-funder"}, allEntries = true) + public void updateConceptParamNoEscape(String id, String name, String value) { + try { + _quickSeachProfile(getConceptXQueryNoEscape(id, name, value)); + } catch (ISLookUpException e) { + throw new DsmRuntimeException(String.format("unable update concept param [id: %s, name: %s, value: %s]", id, name, value), e); + } + } + + @Override + public void updateDatasourceFields(final String dsId, final Map changes) { + operationManager.addOperation(() -> { + Thread.currentThread().setName("update-ds:" + dsId); + changes.forEach((xpath, value) -> { + try { + _isLookUp(String.format( + "for $x in collection('/db/DRIVER/RepositoryServiceResources/RepositoryServiceResourceType')\n" + + "where $x/RESOURCE_PROFILE/BODY/CONFIGURATION/DATASOURCE_ORIGINAL_ID[@provenance='OPENAIRE']/text() = '%s' \n" + + "return update value $x%s with '%s'", dsId, xpath, value)); + } catch (ISLookUpException e) { + throw new DsmRuntimeException(String.format("unable update datasource fields [id: %s, changes: %s]", dsId, changes), e); + } + }); + }); + } + + @Override + public void addAPIAttribute(final String dsId, final String apiId, final Map changes) { + operationManager.addOperation(() -> { + Thread.currentThread().setName("update-api:" + dsId); + changes.forEach((xpath, value) -> { + try { + final String attribute = StringUtils.substringAfter(xpath, "@"); + final String parentElement = StringUtils.substringBeforeLast(xpath, "/"); + _isLookUp(String.format( + "let $x:=/RESOURCE_PROFILE/BODY/CONFIGURATION/DATASOURCE_ORIGINAL_ID[@provenance='OPENAIRE' and ./text() = '%s']\n" + + "return update insert attribute %s {'%s'} into $x/..//INTERFACE[./@id = '%s']%s", + dsId, attribute, value, apiId, parentElement)); + } catch (ISLookUpException e) { + throw new DsmRuntimeException(String.format("unable add API attribute [dsId: %s, apiId: %s, changes: %s]", dsId, apiId, changes), e); + } + }); + }); + } + + @Override + public void updateAPIField(final String dsId, final String apiId, final Map changes) { + operationManager.addOperation(() -> { + Thread.currentThread().setName("update-api:" + dsId); + changes.forEach((xpath, value) -> { + try { + _isLookUp(String.format( + "let $x:=/RESOURCE_PROFILE/BODY/CONFIGURATION/DATASOURCE_ORIGINAL_ID[@provenance='OPENAIRE' and ./text() = '%s']\n" + + "return update value $x/..//INTERFACE[./@id = '%s']%s with '%s'", + dsId, apiId, xpath, value)); + } catch (ISLookUpException e) { + throw new DsmRuntimeException(String.format("unable update API fields [dsId: %s, apiId: %s, changes: %s]", dsId, apiId, changes), e); + } + }); + }); + } + + @Override + public void registerDS(final DatasourceDetails d) { + operationManager.addOperation(() -> { + Thread.currentThread().setName("save-ds:" + d.getId()); + try { + final String id = isRegistryService.registerProfile(asRepositoryProfile(d)); + log.debug(String.format("registered DS profile %s", id)); + } catch (ISRegistryException e) { + throw new DsmRuntimeException("unable to register DS profile: " + d.getId(), e); + } + }); + } + + @Override + public void registerAPI(final ApiDetails api) { + operationManager.addOperation(() -> { + Thread.currentThread().setName("save-api:" + api.getId()); + try { + final String dsId = api.getDatasource(); + final String iface = asRepositoryInterfce(api); + _isLookUp(String.format( + "let $x:=/RESOURCE_PROFILE/BODY/CONFIGURATION/DATASOURCE_ORIGINAL_ID[@provenance='OPENAIRE' and ./text() = '%s']\n" + + "return update insert %s into $x/../INTERFACES", dsId, iface)); + + log.debug(String.format("registered API %s", api.getId())); + } catch (ISLookUpException e) { + throw new DsmRuntimeException("unable to register API: " + api.getId(), e); + } + }); + } + + @Override + public void removeAPI(final String apiId) throws DsmForbiddenException { + try { + final List metaWorkflows = _quickSeachProfile(String.format( + "distinct-values(for $x in collection('/db/DRIVER/MetaWorkflowDSResources/MetaWorkflowDSResourceType')\n" + + "where $x/RESOURCE_PROFILE/BODY/DATAPROVIDER[./@interface = '%s']\n" + + "return $x/RESOURCE_PROFILE/BODY/DATAPROVIDER/@id/string())", apiId)); + if (!metaWorkflows.isEmpty()) { + throw new DsmForbiddenException( + HttpStatus.SC_FORBIDDEN, + String.format("cannot remove api '%s', it has workflows associated", apiId)); + } + isLookUpService.quickSearchProfile(String.format( + " update delete /RESOURCE_PROFILE/BODY/CONFIGURATION/INTERFACES/INTERFACE[./@id = '%s']", apiId)); + + log.info(String.format("deleted API %s", apiId)); + } catch (ISLookUpException e) { + throw new DsmRuntimeException(String.format("unable to remove API %s", apiId), e); + } + } + + /// HELPERS + + private String getXQuery(final String id, final String name, final String value) { + final Escaper esc = XmlEscapers.xmlContentEscaper(); + if (StringUtils.isNotBlank(value)) { + return String.format( + "update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" + + "/RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s']/param[./@name = '%s'] with %s", id, name, name, + escape(esc, value)); + } else { + return String.format( + "update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" + + "/RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s']/param[./@name = '%s'] with ", id, name, name); + } + } + + private String getConceptXQuery(final String id, final String name, final String value) { + final Escaper esc = XmlEscapers.xmlContentEscaper(); + if (StringUtils.isNotBlank(value)) { + return String.format( + "update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')//" + + "concept[./@id = '%s']/param[./@name = '%s'] with %s", id, name, name, + escape(esc, value)); + } else { + return String.format( + "update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')//concept[./@id = '%s']/param[./@name = '%s'] with ", id, name, name); + } + } + + private String getConceptXQueryNoEscape(final String id, final String name, final String value) { + + if (StringUtils.isNotBlank(value)) { + return String.format( + "update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')//" + + "concept[./@id = '%s']/param[./@name = '%s'] with %s", id, name, name, + value); + } else { + return String.format( + "update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')//concept[./@id = '%s']/param[./@name = '%s'] with ", id, name, name); + } + } + + private Map _processContext(final String xquery) throws IOException { + return _processContext(new LinkedBlockingQueue<>(), xquery); + } + + private Map _processContext(final Queue errors, final String xquery) throws IOException { + try { + return getContextProfiles(errors, xquery).stream() + .filter(StringUtils::isNotBlank) + .map(s -> ContextMappingUtils.parseContext(s, errors)) + .collect(Collectors.toMap( + Context::getId, + Function.identity(), + (c1, c2) -> { + log.warn(String.format("found duplicate context profile '%s'", c1.getId())); + return c1; + })); + } finally { + if (!errors.isEmpty()) { + log.error(errors); + errors.forEach(Throwable::printStackTrace); + } + } + } + + private List getContextProfiles(final Queue errors, final String xquery) throws IOException { + log.warn("getContextProfiles(): not using cache"); + try { + return _quickSeachProfile(xquery); + } catch (ISLookUpException e) { + throw new DsmRuntimeException("unable to get context profiles", e); + } + } + + private String _getQuery(final ClassPathResource resource) throws IOException { + return IOUtils.toString(resource.getInputStream(), Charset.defaultCharset()); + } + + private String _isLookUp(final String xquery) throws ISLookUpException { + log.debug(String.format("running xquery:\n%s", xquery)); + //log.debug(String.format("query result: %s", res)); + return isLookUpService.getResourceProfileByQuery(xquery); + } + + private List _quickSeachProfile(final String xquery) throws ISLookUpException { + final List res = Lists.newArrayList(); + + log.debug(String.format("running xquery:\n%s", xquery)); + final List list = isLookUpService.quickSearchProfile(xquery); + if (list != null) { + res.addAll(list); + } + log.debug(String.format("query result size: %s", res.size())); + return res; + } + + @CacheEvict(cacheNames = { "context-cache", "indexdsinfo-cache", "objectstoreid-cache" }, allEntries = true) + @Scheduled(fixedDelayString = "${openaire.exporter.cache.ttl}") + public void dropCache() { + log.debug("dropped dsManager IS cache"); + } + + + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/OperationManager.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/OperationManager.java new file mode 100644 index 00000000..532c1890 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/OperationManager.java @@ -0,0 +1,60 @@ +package eu.dnetlib.openaire.common; + +import java.util.List; +import java.util.concurrent.*; +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.stereotype.Component; + +@Component +public class OperationManager { + + private static final Log log = LogFactory.getLog(OperationManager.class); + + private static final long SLEEP_TIME = 1000; + + private static final int Q_SIZE = 100; + + private static final int POOL_SIZE = 5; + + private final BlockingQueue ops = new ArrayBlockingQueue<>(Q_SIZE); + + private ExecutorService executor; + + @PostConstruct + public void init() { + executor = getExecutor(); + } + + public int dropAll() { + final List lostOperations = executor.shutdownNow(); + log.warn(String.format("discarding %s operations", lostOperations.size())); + executor = getExecutor(); + return lostOperations.size(); + } + + public int getOpSize() { + return ops.size(); + } + + public void addOperation(final Runnable op) { + executor.execute(op); + } + + @PreDestroy + public void tearDown() throws InterruptedException { + executor.shutdown(); + final boolean done = executor.awaitTermination(SLEEP_TIME, TimeUnit.MILLISECONDS); + log.debug(String.format("All operations were completed so far? %s", done)); + } + + // HELPERS + + private ThreadPoolExecutor getExecutor() { + return new ThreadPoolExecutor(POOL_SIZE, POOL_SIZE,0L, TimeUnit.MILLISECONDS, ops); + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/RFC3339DateFormat.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/RFC3339DateFormat.java new file mode 100644 index 00000000..28fe4e3f --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/RFC3339DateFormat.java @@ -0,0 +1,60 @@ +package eu.dnetlib.openaire.common; + +import java.text.FieldPosition; +import java.util.*; + +import com.fasterxml.jackson.databind.util.StdDateFormat; + +public class RFC3339DateFormat extends StdDateFormat { + + private static final TimeZone TIMEZONE_Z = TimeZone.getTimeZone("UTC"); + + // Same as ISO8601DateFormat but serializing milliseconds. + @Override + public StringBuffer format(Date date, StringBuffer toAppendTo, FieldPosition fieldPosition) { + String value = format(date, true, TIMEZONE_Z, Locale.US); + toAppendTo.append(value); + return toAppendTo; + } + + /** + * Format date into yyyy-MM-ddThh:mm:ss[.sss][Z|[+-]hh:mm] + * + * @param date the date to format + * @param millis true to include millis precision otherwise false + * @param tz timezone to use for the formatting (UTC will produce 'Z') + * @return the date formatted as yyyy-MM-ddThh:mm:ss[.sss][Z|[+-]hh:mm] + */ + private static String format(Date date, boolean millis, TimeZone tz, Locale loc) { + Calendar calendar = new GregorianCalendar(tz, loc); + calendar.setTime(date); + + // estimate capacity of buffer as close as we can (yeah, that's pedantic ;) + StringBuilder sb = new StringBuilder(30); + sb.append(String.format( + "%04d-%02d-%02dT%02d:%02d:%02d", + calendar.get(Calendar.YEAR), + calendar.get(Calendar.MONTH) + 1, + calendar.get(Calendar.DAY_OF_MONTH), + calendar.get(Calendar.HOUR_OF_DAY), + calendar.get(Calendar.MINUTE), + calendar.get(Calendar.SECOND) + )); + if (millis) { + sb.append(String.format(".%03d", calendar.get(Calendar.MILLISECOND))); + } + + int offset = tz.getOffset(calendar.getTimeInMillis()); + if (offset != 0) { + int hours = Math.abs((offset / (60 * 1000)) / 60); + int minutes = Math.abs((offset / (60 * 1000)) % 60); + sb.append(String.format("%c%02d:%02d", + (offset < 0 ? '-' : '+'), + hours, minutes)); + } else { + sb.append('Z'); + } + return sb.toString(); + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/Utils.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/Utils.java new file mode 100644 index 00000000..600e29d6 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/Utils.java @@ -0,0 +1,22 @@ +package eu.dnetlib.openaire.common; + +import java.util.Iterator; +import java.util.Spliterator; +import java.util.Spliterators; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +import com.google.common.escape.Escaper; +import org.apache.commons.lang3.StringUtils; + +public class Utils { + + public static Stream stream(Iterator iterator) { + return StreamSupport.stream(Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED), false); + } + + public static String escape(final Escaper esc, final String value) { + return StringUtils.isNotBlank(value) ? esc.escape(value) : ""; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityApiController.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityApiController.java new file mode 100644 index 00000000..e653b09a --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityApiController.java @@ -0,0 +1,304 @@ +package eu.dnetlib.openaire.community; + +import java.util.List; + +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiResponse; +import io.swagger.annotations.ApiResponses; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.web.bind.annotation.*; + +import static eu.dnetlib.openaire.common.ExporterConstants.*; + +@RestController +@CrossOrigin(origins = { "*" }) +@ConditionalOnProperty(value = "openaire.exporter.enable.community", havingValue = "true") +@io.swagger.annotations.Api(tags = "OpenAIRE Communities API", description = "the OpenAIRE Community API") +public class CommunityApiController { + + @Autowired + private CommunityApiCore communityApiCore; + + + @RequestMapping(value = "/community/communities", produces = { "application/json" }, method = RequestMethod.GET) + @ApiOperation( + value = "get all community profiles", + notes = "get all community profiles", + tags = { C, R }, + response = CommunitySummary[].class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = CommunitySummary[].class), + @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) }) + public List listCommunities() throws CommunityException { + return communityApiCore.listCommunities(); + } + + @RequestMapping(value = "/community/{id}", produces = { "application/json" }, method = RequestMethod.GET) + @ApiOperation( + value = "get community profile", + notes = "get community profile", + tags = { C, R }, + response = CommunityDetails.class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = CommunityDetails.class), + @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class), + @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) }) + public CommunityDetails getCommunity(@PathVariable final String id) throws CommunityException, CommunityNotFoundException { + return communityApiCore.getCommunity(id); + } + + @RequestMapping(value = "/community/{id}", produces = { "application/json" }, method = RequestMethod.POST) + @ApiOperation( + value = "update community details", + notes = "update community details", + tags = { C, R }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class), + @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) }) + public void setCommunity( + @PathVariable final String id, + @RequestBody CommunityWritableProperties properties) throws CommunityException, CommunityNotFoundException { + + communityApiCore.setCommunity(id, properties); + } + + @RequestMapping(value = "/community/{id}/projects", produces = { "application/json" }, method = RequestMethod.GET) + @ApiOperation( + value = "get community projects", + notes = "get community projects", + tags = { C_PJ, R }, + response = CommunityProject[].class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = CommunityProject[].class), + @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class), + @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) }) + public List getCommunityProjects(@PathVariable final String id) throws CommunityException, CommunityNotFoundException { + return communityApiCore.getCommunityProjects(id); + } + + @RequestMapping(value = "/community/{id}/projects", produces = { "application/json" }, method = RequestMethod.POST) + @ApiOperation( + value = "associate a project to the community", + notes = "associate a project to the community", + tags = { C_PJ, W }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class), + @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) }) + public CommunityProject addCommunityProject( + @PathVariable final String id, + @RequestBody final CommunityProject project) throws CommunityException, CommunityNotFoundException { + + return communityApiCore.addCommunityProject(id, project); + } + + @RequestMapping(value = "/community/{id}/projects", produces = { "application/json" }, method = RequestMethod.DELETE) + @ApiOperation( + value = "remove a project from the community", + notes = "remove a project from the community", + tags = { C_PJ, W }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class), + @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) }) + public void deleteCommunityProject( + @PathVariable final String id, + @RequestBody final Integer projectId) throws CommunityException, CommunityNotFoundException { + + communityApiCore.removeCommunityProject(id, projectId); + } + + @RequestMapping(value = "/community/{id}/contentproviders", produces = { "application/json" }, method = RequestMethod.GET) + @ApiOperation( + value = "get the list of content providers associated to a given community", + notes = "get the list of content providers associated to a given community", + tags = { C_CP, R }, + response = CommunityContentprovider[].class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = CommunityContentprovider[].class), + @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class), + @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) }) + public List getCommunityContentproviders(@PathVariable final String id) throws CommunityException, CommunityNotFoundException { + return communityApiCore.getCommunityContentproviders(id); + } + + @RequestMapping(value = "/community/{id}/contentproviders", produces = { "application/json" }, method = RequestMethod.POST) + @ApiOperation( + value = "associate a content provider to the community", + notes = "associate a content provider to the community", + tags = { C_CP, W }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class), + @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) }) + public CommunityContentprovider addCommunityContentprovider( + @PathVariable final String id, + @RequestBody final CommunityContentprovider contentprovider) throws CommunityException, CommunityNotFoundException { + + return communityApiCore.addCommunityContentprovider(id, contentprovider); + } + + @RequestMapping(value = "/community/{id}/contentproviders", produces = { "application/json" }, method = RequestMethod.DELETE) + @ApiOperation( + value = "remove the association between a content provider and the community", + notes = "remove the association between a content provider and the community", + tags = { C_CP, W }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class), + @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) }) + public void removeCommunityContentprovider( + @PathVariable final String id, + @RequestBody final Integer contentproviderId) throws CommunityException, CommunityNotFoundException { + + communityApiCore.removeCommunityContentProvider(id, contentproviderId); + } + + //ADDING CODE FOR COMMUNITY ORGANIZATIONS + + @RequestMapping(value = "/community/{id}/organizations", produces = { "application/json" }, method = RequestMethod.GET) + @ApiOperation( + value = "get the list of organizations for a given community", + notes = "get the list of organizations for a given community", + tags = { C_O, R }, + response = CommunityOrganization[].class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = CommunityContentprovider[].class), + @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class), + @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) }) + public List getCommunityOrganizations(@PathVariable final String id) throws CommunityException, CommunityNotFoundException { + return communityApiCore.getCommunityOrganizations(id); + } + + @RequestMapping(value = "/community/{id}/organizations", produces = { "application/json" }, method = RequestMethod.POST) + @ApiOperation( + value = "associate an organization to the community", + notes = "associate an organization to the community", + tags = { C_O, W }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class), + @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) }) + public CommunityOrganization addCommunityOrganization( + @PathVariable final String id, + @RequestBody final CommunityOrganization organization) throws CommunityException, CommunityNotFoundException { + + return communityApiCore.addCommunityOrganization(id, organization); + } + + @RequestMapping(value = "/community/{id}/organizations", produces = { "application/json" }, method = RequestMethod.DELETE) + @ApiOperation( + value = "remove the association between an organization and the community", + notes = "remove the association between an organization and the community", + tags = { C_O, W }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class), + @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) }) + public void removeCommunityOrganization( + @PathVariable final String id, + @RequestBody final Integer organizationId) throws CommunityException, CommunityNotFoundException { + + communityApiCore.removeCommunityOrganization(id, organizationId); + } + //********************** + + @RequestMapping(value = "/community/{id}/subjects", produces = { "application/json" }, method = RequestMethod.POST) + @ApiOperation( + value = "associate a subject to the community", + notes = "associate a subject to the community", + tags = { C, W }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class), + @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) }) + public CommunityDetails addCommunitySubjects( + @PathVariable final String id, + @RequestBody final List subjects) throws CommunityException, CommunityNotFoundException { + + return communityApiCore.addCommunitySubjects(id, subjects); + } + + @RequestMapping(value = "/community/{id}/subjects", produces = { "application/json" }, method = RequestMethod.DELETE) + @ApiOperation( + value = "remove subjects from a community", + notes = "remove subjects from a community", + tags = { C, W }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class), + @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) }) + public CommunityDetails removeCommunitySubjects( + @PathVariable final String id, + @RequestBody final List subjects) throws CommunityException, CommunityNotFoundException { + + return communityApiCore.removeCommunitySubjects(id, subjects); + } + + @RequestMapping(value = "/community/{id}/zenodocommunities", produces = { "application/json" }, method = RequestMethod.GET) + @ApiOperation( + value = "get the list of Zenodo communities associated to a given community", + notes = "get the list of Zenodo communities associated to a given community", + tags = { C_ZC, R }, + response = CommunityZenodoCommunity[].class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = CommunityZenodoCommunity[].class), + @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class), + @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) }) + public List getCommunityZenodoCommunities(@PathVariable final String id) throws CommunityException, CommunityNotFoundException { + return communityApiCore.getCommunityZenodoCommunities(id); + } + + @RequestMapping(value = "/community/{id}/zenodocommunities", produces = { "application/json" }, method = RequestMethod.POST) + @ApiOperation( + value = "associate a Zenodo community to the community", + notes = "associate a Zenodo community to the community", + tags = { C_ZC, W }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class), + @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) }) + public CommunityZenodoCommunity addCommunityZenodoCommunity( + @PathVariable final String id, + @RequestBody final CommunityZenodoCommunity zenodocommunity) throws CommunityException, CommunityNotFoundException { + + return communityApiCore.addCommunityZenodoCommunity(id, zenodocommunity); + + } + + @RequestMapping(value = "/community/{id}/zenodocommunities", produces = { "application/json" }, method = RequestMethod.DELETE) + @ApiOperation( + value = "remove a Zenodo community from a community", + notes = "remove a Zenodo community from a community", + tags = { C_ZC, W }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class), + @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) }) + public void removeCommunityZenodoCommunity( + @PathVariable final String id, + @RequestBody final Integer zenodoCommId) throws CommunityException, CommunityNotFoundException { + + communityApiCore.removeCommunityZenodoCommunity(id, zenodoCommId); + + } + + + @RequestMapping(value = "/community/{zenodoId}/openairecommunities", produces = { "application/json" }, method = RequestMethod.GET) + @ApiOperation( + value = "get the list of OpenAIRE communities associated to a given Zenodo community", + notes = "get the list of OpenAIRE communities associated to a given Zenodo community", + tags = { C_ZC, R }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class), + @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) }) + public CommunityOpenAIRECommunities getOpenAireCommunities( + @PathVariable final String zenodoId) throws CommunityException, CommunityNotFoundException { + + return communityApiCore.getOpenAIRECommunities(zenodoId); + + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityApiCore.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityApiCore.java new file mode 100644 index 00000000..c655a91a --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityApiCore.java @@ -0,0 +1,386 @@ +package eu.dnetlib.openaire.community; + +import com.google.common.base.Functions; +import com.google.common.base.Joiner; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import eu.dnetlib.openaire.common.ISClient; +import eu.dnetlib.openaire.context.Context; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.cache.annotation.CacheEvict; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.util.*; +import java.util.stream.Collectors; + +import static eu.dnetlib.openaire.community.CommunityConstants.*; + +@Component +@ConditionalOnProperty(value = "openaire.exporter.enable.community", havingValue = "true") +public class CommunityApiCore {//implements CommunityClient{ + + private static final Log log = LogFactory.getLog(CommunityApiCore.class); + + @Autowired + private CommunityClient cci; + + @Autowired + private ISClient isClient; + + @Autowired + private CommunityCommon cc; + + + public List listCommunities() throws CommunityException { + return cc.listCommunities(); + + } + + public CommunityDetails getCommunity(final String id) throws CommunityException, CommunityNotFoundException { + return cc.getCommunity(id); + + } + + public void setCommunity(final String id, final CommunityWritableProperties details) throws CommunityException, CommunityNotFoundException { + + cc.getCommunity(id); // ensure the community exists. + + if(details.getShortName() != null) { + isClient.updateContextAttribute(id, CLABEL, details.getShortName()); + + } + if (details.getName() != null){ + isClient.updateContextParam(id, CSUMMARY_NAME, details.getName()); + + } + if(details.getDescription() != null) { + isClient.updateContextParam(id, CSUMMARY_DESCRIPTION, details.getDescription()); + + } + if(details.getLogoUrl()!=null){ + isClient.updateContextParam(id, CSUMMARY_LOGOURL, details.getLogoUrl()); + + } + if (details.getStatus() != null) { + isClient.updateContextParam(id, CSUMMARY_STATUS, details.getStatus().name()); + + } + if (details.getSubjects() != null) { + isClient.updateContextParam(id, CPROFILE_SUBJECT, Joiner.on(CSV_DELIMITER).join(details.getSubjects())); + + } + if (details.getMainZenodoCommunity() != null) { + isClient.updateContextParam(id, CSUMMARY_ZENODOC, details.getMainZenodoCommunity()); + } + + cc.updateCommunity(id, details); + } + + public List getCommunityProjects(final String id) throws CommunityException, CommunityNotFoundException { + cc.getCommunity(id); // ensure the community exists. + return cc.getCommunityInfo(id, PROJECTS_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityProject(id, c)); + } + + public CommunityProject addCommunityProject(final String id, final CommunityProject project) throws CommunityException, CommunityNotFoundException { + if (!StringUtils.equalsIgnoreCase(id, project.getCommunityId())) { + throw new CommunityException("parameters 'id' and project.communityId must be coherent"); + } + + final TreeMap projects = getCommunityProjectMap(id); + String project_id = project.getId(); + + if (project_id != null && projects.keySet().contains(Integer.valueOf(project_id))){ + if (project.getName() != null) { + isClient.updateConceptParam(id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project_id, CPROJECT_FULLNAME,project.getName()); + + } + if(project.getAcronym()!= null){ + isClient.updateConceptParam(id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project_id, CPROJECT_ACRONYM,project.getAcronym()); + + } + if (project.getOpenaireId() != null){ + isClient.updateConceptParam(id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project_id, OPENAIRE_ID, project.getOpenaireId()); + + } + if (project.getFunder() != null){ + isClient.updateConceptParam(id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project_id, CPROJECT_FUNDER, project.getFunder()); + + } + if(project.getGrantId() != null){ + isClient.updateConceptParam(id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project_id, CPROJECT_NUMBER, project.getGrantId()); + + } + }else { + project.setId(nextId(projects != null && !projects.isEmpty() ? projects.lastKey() : 0)); + + isClient.addConcept(id, id + PROJECTS_ID_SUFFIX, CommunityMappingUtils.asProjectXML(id, project)); + + } + cc.updateProject(id, project ); + return project; + } + + private String nextId(final Integer id) { + return String.valueOf(id + 1); + } + + public void removeCommunityProject(final String id, final Integer projectId) throws CommunityException, CommunityNotFoundException { + final Map projects = getCommunityProjectMap(id); + if (!projects.containsKey(projectId)) { + throw new CommunityNotFoundException(String.format("project '%s' doesn't exist within context '%s'", projectId, id)); + } + isClient.removeConcept( + id, + id + PROJECTS_ID_SUFFIX, + id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + projectId); + cc.removeFromCategory(id, PROJECTS_ID_SUFFIX, String.valueOf(projectId)); + } + + public List getCommunityContentproviders(final String id) throws CommunityException, CommunityNotFoundException { + cc.getCommunity(id); // ensure the community exists. + return cc.getCommunityInfo(id, CONTENTPROVIDERS_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityDataprovider(id, c)); + } + + public CommunityContentprovider addCommunityContentprovider(final String id, final CommunityContentprovider cp) throws CommunityException, CommunityNotFoundException { + log.info("content provider to add " + cp.toString()); + if (!StringUtils.equalsIgnoreCase(id, cp.getCommunityId())) { + throw new CommunityException("parameters 'id' and cp.communityId must be coherent"); + } + + final TreeMap cps = getCommunityContentproviderMap(id); + final String concept_id = cp.getId(); + if (concept_id != null && cps.keySet().contains(Integer.valueOf(concept_id))){ + if (cp.getName() != null) { + isClient.updateConceptParam(id + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + concept_id, CCONTENTPROVIDER_NAME,cp.getName()); + } + if(cp.getOfficialname()!= null){ + isClient.updateConceptParam(id + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + concept_id, CCONTENTPROVIDER_OFFICIALNAME,cp.getOfficialname()); + } + if (cp.getOpenaireId() != null){ + isClient.updateConceptParam(id + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + concept_id, OPENAIRE_ID,cp.getOpenaireId()); + } + if(cp.getSelectioncriteria() != null){ + isClient.updateConceptParamNoEscape(id + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + concept_id, CCONTENTPROVIDER_SELCRITERIA, cp.toXML()); + } + }else{ + log.info("adding new concept for community " + id); + cp.setId(nextId(!cps.isEmpty() ? cps.lastKey() : 0)); + + + isClient.addConcept(id, id + CONTENTPROVIDERS_ID_SUFFIX, CommunityMappingUtils.asContentProviderXML(id, cp)); + } + + cc.updateDatasource(id, cp); + return cp; + } + + public void removeCommunityContentProvider(final String id, final Integer contentproviderId) throws CommunityException, CommunityNotFoundException { + final Map providers = getCommunityContentproviderMap(id); + if (!providers.containsKey(contentproviderId)) { + throw new CommunityNotFoundException(String.format("content provider '%s' doesn't exist within context '%s'", contentproviderId, id)); + } + isClient.removeConcept( + id, + id + CONTENTPROVIDERS_ID_SUFFIX, + id + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + contentproviderId); + cc.removeFromCategory(id, CONTENTPROVIDERS_ID_SUFFIX, String.valueOf(contentproviderId)); + } + + public void removeCommunityOrganization(String id, Integer organizationId) throws CommunityException, CommunityNotFoundException { + final Map organizations = getCommunityOrganizationMap(id); + if (!organizations.containsKey(organizationId)) { + throw new CommunityNotFoundException(String.format("organization '%s' doesn't exist within context '%s'", organizationId, id)); + } + isClient.removeConcept( + id, + id + ORGANIZATION_ID_SUFFIX, + id + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organizationId); + cc.removeFromCategory(id, ORGANIZATION_ID_SUFFIX, String.valueOf(organizationId)); + } + + public List getCommunityZenodoCommunities(final String id) throws CommunityException, CommunityNotFoundException { + + return cc.getCommunityZenodoCommunities(id); + } + + public List getCommunityOrganizations(final String id) throws CommunityException, CommunityNotFoundException { + cc.getCommunity(id); + return cc.getCommunityInfo(id,ORGANIZATION_ID_SUFFIX,c->CommunityMappingUtils.asCommunityOrganization(id,c)); + } + + + public CommunityDetails addCommunitySubjects(final String id, final List subjects) throws CommunityException, CommunityNotFoundException { + + final CommunityDetails cd = new CommunityDetails(); + + final Set current = Sets.newHashSet(cc.getCommunity(id).getSubjects()); + + current.addAll(subjects); + + cd.setSubjects(Lists.newArrayList(current)); + + setCommunity(id, CommunityWritableProperties.fromDetails(cd)); + + return cd; + } + + public CommunityDetails removeCommunitySubjects(final String id, final List subjects) throws CommunityException, CommunityNotFoundException { + + final CommunityDetails cd = new CommunityDetails(); + + final Set current = Sets.newHashSet(cc.getCommunity(id).getSubjects()); + + current.removeAll(subjects); + + cd.setSubjects(Lists.newArrayList(current)); + + setCommunity(id, CommunityWritableProperties.fromDetails(cd)); + + return cd; + } + + @CacheEvict(value="community-cache",allEntries = true) + public void removeCommunityZenodoCommunity(final String id, final Integer zenodoCommId) throws CommunityException, CommunityNotFoundException { + + final Map zcomms = getZenodoCommunityMap(id); + if (!zcomms.containsKey(zenodoCommId)) { + throw new CommunityNotFoundException(String.format("Zenodo community '%s' doesn't exist within context '%s'", zenodoCommId, id)); + } + isClient.removeConcept( + id, + id + ZENODOCOMMUNITY_ID_SUFFIX, + id + ZENODOCOMMUNITY_ID_SUFFIX + ID_SEPARATOR + zenodoCommId); + + + cc.removeFromCategory(id, ZENODOCOMMUNITY_ID_SUFFIX, String.valueOf(zenodoCommId)); + + } + + @CacheEvict(value="community-cache",allEntries = true) + public CommunityZenodoCommunity addCommunityZenodoCommunity(final String id, final CommunityZenodoCommunity zc) throws CommunityException, CommunityNotFoundException { + if (!StringUtils.equalsIgnoreCase(id, zc.getCommunityId())) { + throw new CommunityException("parameters 'id' and zc.communityId must be coherent"); + } + if(!StringUtils.isNotBlank(zc.getZenodoid())){ + throw new CommunityException("parameter zenodoid cannot be null or empty"); + } + final TreeMap zcs = getZenodoCommunityMap(id); + + for(CommunityZenodoCommunity czc : zcs.values()){ + if (czc.getZenodoid().equals(zc.getZenodoid())){ + throw new CommunityException("Zenodo community already associated to the RCD"); + } + } + + zc.setId(nextId(!zcs.isEmpty() ? zcs.lastKey() : 0)); + + isClient.addConcept(id, id + ZENODOCOMMUNITY_ID_SUFFIX, CommunityMappingUtils.asZenodoCommunityXML(id, zc)); + cc.updateZenodoCommunity(id, zc); + + return zc; + } + + public CommunityOpenAIRECommunities getOpenAIRECommunities(String zenodoId) throws CommunityException, CommunityNotFoundException { + + if(cci.getInverseZenodoCommunityMap().containsKey(zenodoId)) + return new CommunityOpenAIRECommunities().setZenodoid(zenodoId).setOpenAirecommunitylist(cci.getInverseZenodoCommunityMap().get(zenodoId).stream().collect(Collectors.toList())); + return new CommunityOpenAIRECommunities(); + + } + + // HELPERS + + private TreeMap getCommunityProjectMap(final String id) throws CommunityException, CommunityNotFoundException { + return getCommunityProjects(id).stream() + .collect(Collectors.toMap( + p -> Integer.valueOf(p.getId()), + Functions.identity(), + (p1, p2) -> { + log.warn(String.format("duplicate project found: '%s'", p1.getId())); + return p2; + }, + TreeMap::new)); + } + + private TreeMap getCommunityContentproviderMap(final String id) throws CommunityException, CommunityNotFoundException { + log.info("getting community content provider map"); + return getCommunityContentproviders(id).stream() + .collect(Collectors.toMap( + cp -> Integer.valueOf(cp.getId()), + Functions.identity(), + (cp1, cp2) -> { + log.warn(String.format("duplicate content provider found: '%s'", cp1.getId())); + return cp2; + }, + TreeMap::new)); + } + + private TreeMap getZenodoCommunityMap(final String id) throws CommunityException, CommunityNotFoundException{ + return getCommunityZenodoCommunities(id).stream() + .collect(Collectors.toMap( + cp -> Integer.valueOf(cp.getId()), + Functions.identity(), + (cp1, cp2) -> { + log.warn(String.format("duplicate Zenodo community found: '%s'", cp1.getId())); + return cp2; + }, + TreeMap::new)); + } + + private TreeMap getCommunityOrganizationMap(final String id) throws CommunityException, CommunityNotFoundException { + return getCommunityOrganizations(id).stream() + .collect(Collectors.toMap( + o -> Integer.valueOf(o.getId()), + Functions.identity(), + (o1, o2) -> { + log.warn(String.format("duplicate content provider found: '%s'", o1.getId())); + return o2; + }, + TreeMap::new)); + } + + private Map getContextMap() throws CommunityException { + try { + return isClient.getCommunityContextMap(); + } catch (IOException e) { + throw new CommunityException(e); + } + } + + public CommunityOrganization addCommunityOrganization(String id, CommunityOrganization organization) throws CommunityException, CommunityNotFoundException { + if (!StringUtils.equalsIgnoreCase(id, organization.getCommunityId())) { + throw new CommunityException("parameters 'id' and organization.communityId must be coherent"); + } + + final TreeMap cps = getCommunityOrganizationMap(id); + + final String organization_id = organization.getId(); + if (organization_id != null && cps.keySet().contains(Integer.valueOf(organization_id))){ + if (organization.getName() != null) { + isClient.updateConceptParam(id + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organization_id, CORGANIZATION_NAME,organization.getName()); + } + if(organization.getLogo_url()!= null){ + isClient.updateConceptParam(id + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organization_id, CORGANIZATION_LOGOURL, Base64.getEncoder().encodeToString(organization.getLogo_url().getBytes())); + } + if (organization.getWebsite_url() != null){ + isClient.updateConceptParam(id + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organization_id, CORGANIZATION_WEBSITEURL,Base64.getEncoder().encodeToString(organization.getWebsite_url().getBytes())); + } + }else{ + organization.setId(nextId(!cps.isEmpty() ? cps.lastKey() : 0)); + isClient.addConcept(id, id + ORGANIZATION_ID_SUFFIX, CommunityMappingUtils.asOrganizationXML(id, organization)); + } + + + cc.updateOrganization(id, organization); + return organization; + } + + + + +} \ No newline at end of file diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityClient.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityClient.java new file mode 100644 index 00000000..dbc18fc5 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityClient.java @@ -0,0 +1,12 @@ +package eu.dnetlib.openaire.community; + +import java.util.Map; +import java.util.Set; + +public interface CommunityClient { + + Map> getInverseZenodoCommunityMap() throws CommunityException, CommunityNotFoundException; + + void dropCache(); + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityClientImpl.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityClientImpl.java new file mode 100644 index 00000000..96ac59a6 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityClientImpl.java @@ -0,0 +1,58 @@ +package eu.dnetlib.openaire.community; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cache.annotation.CacheEvict; +import org.springframework.cache.annotation.Cacheable; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.util.*; + +@Component +public class CommunityClientImpl implements CommunityClient { + + private static final Log log = LogFactory.getLog(CommunityClient.class); + + @Autowired + private CommunityCommon communityCommon; + + @Override + @Cacheable("community-cache") + public Map> getInverseZenodoCommunityMap () throws CommunityException, CommunityNotFoundException { + log.info("Creating the data structure. Not using cache"); + final Map> inverseListMap = new HashMap<>(); + + final List communityList = communityCommon.listCommunities(); + + for(CommunitySummary cs :communityList){ + final String communityId = cs.getId(); + List czc = communityCommon.getCommunityZenodoCommunities(communityId); + for(CommunityZenodoCommunity zc:czc){ + final String zenodoId = zc.getZenodoid(); + if(!inverseListMap.containsKey(zenodoId)) { + inverseListMap.put(zc.getZenodoid(),new HashSet<>()); + } + inverseListMap.get(zc.getZenodoid()).add(communityId); + } + final String zenodoMainCommunity = communityCommon.getCommunity(communityId).getZenodoCommunity(); + if(!inverseListMap.containsKey(zenodoMainCommunity)) { + inverseListMap.put(zenodoMainCommunity,new HashSet<>()); + } + + inverseListMap.get(zenodoMainCommunity).add(communityId); + } + return inverseListMap; + } + + + + @Override + @CacheEvict(cacheNames = { "community-cache", "context-cache-community"}, allEntries = true) + @Scheduled(fixedDelayString = "${openaire.exporter.cache.ttl}") + public void dropCache(){ + log.debug("dropped community cache"); + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityCommon.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityCommon.java new file mode 100644 index 00000000..fde84333 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityCommon.java @@ -0,0 +1,438 @@ +package eu.dnetlib.openaire.community; + +import com.google.common.base.Joiner; +import com.google.common.collect.Lists; +import eu.dnetlib.openaire.common.ISClient; +import eu.dnetlib.openaire.context.Category; +import eu.dnetlib.openaire.context.Concept; +import eu.dnetlib.openaire.context.Context; +import eu.dnetlib.openaire.context.Param; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.util.*; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static eu.dnetlib.openaire.community.CommunityConstants.*; + +@Component +public class CommunityCommon { + + @Autowired + private ISClient isClient; + + public Map getContextMap() throws CommunityException { + try { + return isClient.getCommunityContextMap(); + } catch (IOException e) { + throw new CommunityException(e); + } + } + + public List listCommunities() throws CommunityException { + return getContextMap().values().stream() + .filter(context -> !communityBlackList.contains(context.getId())) + .map(CommunityMappingUtils::asCommunitySummary) + .collect(Collectors.toList()); + } + + + public List getCommunityInfo(final String id, final String idSuffix, final Function mapping) throws CommunityException { + final Map contextMap = getContextMap(); + final Context context = contextMap.get(id); + if (context != null) { + final Map categories = context.getCategories(); + final Category category = categories.get(id + idSuffix); + if (category != null) { + return category.getConcepts().stream() + .map(mapping) + .collect(Collectors.toList()); + } + } + return Lists.newArrayList(); + } + + public CommunityDetails getCommunity(final String id) throws CommunityException, CommunityNotFoundException { + final Context context = getContextMap().get(id); + if (context == null || CommunityConstants.communityBlackList.contains(id)) { + throw new CommunityNotFoundException(String.format("community '%s' does not exist", id)); + } + return CommunityMappingUtils.asCommunityProfile(context); + } + + public List getCommunityZenodoCommunities(final String id) throws CommunityException, CommunityNotFoundException { + getCommunity(id); // ensure the community exists. + return getCommunityInfo(id, ZENODOCOMMUNITY_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityZenodoCommunity(id, c)); + } + + + public void updateProject(String communityId, CommunityProject project) throws CommunityException { + final Context context = getContextMap().get(communityId); + Category prj = context.getCategories().get(communityId + PROJECTS_ID_SUFFIX); + if (prj.getConcepts().stream().map(c -> c.getId()).collect(Collectors.toList()) + .contains(communityId + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project.getId())){ + prj.getConcepts().forEach(concept -> { + if (concept.getId().equals(communityId + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project.getId())) { + if (project.getName() != null) { + + concept.getParams().replace(CPROJECT_FULLNAME, Arrays.asList(new Param() + .setName(CPROJECT_FULLNAME).setValue(project.getName()))); + } + if (project.getAcronym() != null) { + if(concept.getParams().keySet().contains(CPROJECT_ACRONYM)){ + concept.getParams().replace(CPROJECT_ACRONYM, Arrays.asList(new Param() + .setName(CPROJECT_ACRONYM).setValue(project.getAcronym()))); + } + else{ + concept.getParams().put(CPROJECT_ACRONYM, Arrays.asList(new Param() + .setName(CPROJECT_ACRONYM).setValue(project.getAcronym()))); + } + + } + if (project.getOpenaireId() != null) { + if(concept.getParams().keySet().contains(OPENAIRE_ID)){ + concept.getParams().replace(OPENAIRE_ID, Arrays.asList(new Param() + .setName(OPENAIRE_ID).setValue(project.getOpenaireId()))); + } + else{ + concept.getParams().put(OPENAIRE_ID, Arrays.asList(new Param() + .setName(OPENAIRE_ID).setValue(project.getOpenaireId()))); + } + + } + if (project.getFunder() != null) { + concept.getParams().replace(CPROJECT_FUNDER, Arrays.asList(new Param() + .setName(CPROJECT_FUNDER).setValue(project.getFunder()))); + } + if (project.getGrantId() != null) { + concept.getParams().replace(CPROJECT_NUMBER, Arrays.asList(new Param() + .setName(CPROJECT_NUMBER).setValue(project.getGrantId()))); + + } + } + }); + } + else{ + Concept concept = new Concept(); + concept.setId(communityId + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project.getId()); + concept.setClaim(false); + if(project.getAcronym() != null) + concept.setLabel(project.getAcronym()); + else + concept.setLabel(""); + + Map> params = new TreeMap<>(); + + if(project.getAcronym() != null){ + params.put(CPROJECT_ACRONYM, Arrays.asList(new Param().setName(CPROJECT_ACRONYM) + .setValue(project.getAcronym()))); + } + + if (project.getName() != null){ + params.put(CPROJECT_FULLNAME, Arrays.asList(new Param() + .setName(CPROJECT_FULLNAME) + .setValue(project.getName()) + )); + } + + if (project.getOpenaireId() != null){ + params.put(OPENAIRE_ID, Arrays.asList(new Param() + .setName(OPENAIRE_ID) + .setValue(project.getOpenaireId()) + )); + } + + if(project.getFunder() != null){ + params.put(CPROJECT_FUNDER, Arrays.asList(new Param() + .setName(CPROJECT_FUNDER) + .setValue(project.getFunder()) + )); + } + + if (project.getGrantId()!=null){ + params.put(CPROJECT_NUMBER, Arrays.asList(new Param() + .setName(CPROJECT_NUMBER) + .setValue(project.getGrantId()) + )); + } + + concept.setParams(params); + prj.getConcepts().add(concept); + + + } + + } + + public void updateCommunity(String id, CommunityWritableProperties community) throws CommunityException { + final Context context = getContextMap().get(id); + + if(community.getShortName() != null) { + context.setLabel(community.getShortName()); + } + + if (community.getName() != null){ + context.getParams().replace(CSUMMARY_NAME, Arrays.asList(new Param() + .setValue(community.getName()).setName(CSUMMARY_NAME))); + } + if(community.getDescription() != null) { + context.getParams() + .replace(CSUMMARY_DESCRIPTION, Arrays.asList(new Param() + .setName(CSUMMARY_DESCRIPTION).setValue(community.getDescription()))); + } + if(community.getLogoUrl() != null){ + context.getParams() + .replace(CSUMMARY_LOGOURL, Arrays.asList(new Param() + .setName(CSUMMARY_LOGOURL).setValue(community.getLogoUrl()))); + + } + if (community.getStatus() != null) { + context.getParams() + .replace(CSUMMARY_STATUS, Arrays.asList(new Param() + .setName(CSUMMARY_STATUS).setValue(community.getStatus().name()))); + } + if (community.getSubjects() != null) { + context.getParams() + .replace(CPROFILE_SUBJECT, Arrays.asList(new Param().setName(CPROFILE_SUBJECT) + .setValue(Joiner.on(CSV_DELIMITER) + .join(community.getSubjects())))); + } + if(community.getMainZenodoCommunity() != null){ + context.getParams() + .replace(CSUMMARY_ZENODOC, Arrays.asList(new Param() + .setName(CSUMMARY_ZENODOC).setValue(community.getMainZenodoCommunity()))); + + } + + } + + + public void removeFromCategory(String communityId, String category, String conceptId) throws CommunityException { + Map cmap = getContextMap(); + Context context = cmap.get(communityId); + Map cat = context.getCategories(); + + List concepts = cat.get(communityId + category).getConcepts() + .stream().filter(c -> !c.getId().equals(communityId + category + ID_SEPARATOR + conceptId)).collect(Collectors.toList()); + + cat.get(communityId + category).setConcepts(concepts); + } + + public void updateDatasource(String communityId, CommunityContentprovider cp) throws CommunityException { + final Context context = getContextMap().get(communityId); + Category dts = context.getCategories().get(communityId + CONTENTPROVIDERS_ID_SUFFIX); + if (dts.getConcepts().stream().map(c -> c.getId()).collect(Collectors.toList()) + .contains(communityId + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + cp.getId())){ + dts.getConcepts().forEach(concept -> { + if (concept.getId().equals(communityId + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + cp.getId())) { + + + if (cp.getName() != null) { + if(concept.getParams().keySet().contains(CCONTENTPROVIDER_NAME)){ + concept.getParams().replace(CCONTENTPROVIDER_NAME, Arrays.asList(new Param() + .setName(CCONTENTPROVIDER_NAME).setValue(cp.getName()))); + } + else{ + concept.getParams().put(CCONTENTPROVIDER_NAME, Arrays.asList(new Param() + .setName(CCONTENTPROVIDER_NAME).setValue(cp.getName()))); + } + + } + if (cp.getOfficialname() != null) { + if(concept.getParams().keySet().contains(CCONTENTPROVIDER_OFFICIALNAME)){ + concept.getParams().replace(CCONTENTPROVIDER_OFFICIALNAME, Arrays.asList(new Param() + .setName(CCONTENTPROVIDER_OFFICIALNAME).setValue(cp.getOfficialname()))); + } + else{ + concept.getParams().put(CCONTENTPROVIDER_OFFICIALNAME, Arrays.asList(new Param() + .setName(CCONTENTPROVIDER_OFFICIALNAME).setValue(cp.getOfficialname()))); + } + + } + + + if (cp.getOpenaireId() != null) { + if(concept.getParams().keySet().contains(OPENAIRE_ID)){ + concept.getParams().replace(OPENAIRE_ID, Arrays.asList(new Param() + .setName(OPENAIRE_ID).setValue(cp.getOpenaireId()))); + } + else{ + concept.getParams().put(OPENAIRE_ID, Arrays.asList(new Param() + .setName(OPENAIRE_ID).setValue(cp.getOpenaireId()))); + } + + } + + if (cp.getSelectioncriteria() != null) { + if(concept.getParams().keySet().contains(CCONTENTPROVIDER_SELCRITERIA)){ + concept.getParams().replace(CCONTENTPROVIDER_SELCRITERIA, Arrays.asList(new Param() + .setName(CCONTENTPROVIDER_SELCRITERIA).setValue(cp.toJson()))); + } + else{ + concept.getParams().put(CCONTENTPROVIDER_SELCRITERIA, Arrays.asList(new Param() + .setName(CCONTENTPROVIDER_SELCRITERIA).setValue(cp.toJson()))); + } + + } + } + }); + } + else{ + Concept concept = new Concept(); + concept.setId(communityId + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + cp.getId()); + concept.setClaim(false); + concept.setLabel(""); + + Map> params = new TreeMap<>(); + + if (cp.getName() != null) { + params.put( CCONTENTPROVIDER_NAME, Arrays.asList(new Param().setValue(cp.getName()).setName(CCONTENTPROVIDER_NAME))); + } + if(cp.getOfficialname()!= null){ + params.put( CCONTENTPROVIDER_OFFICIALNAME, Arrays.asList(new Param().setValue(cp.getOfficialname()).setName(CCONTENTPROVIDER_OFFICIALNAME))); + } + if (cp.getOpenaireId() != null){ + params.put( OPENAIRE_ID, Arrays.asList(new Param().setValue(cp.getOpenaireId()).setName(OPENAIRE_ID))); + } + if(cp.getSelectioncriteria() != null){ + params.put( CCONTENTPROVIDER_SELCRITERIA, Arrays.asList(new Param().setValue(cp.toJson()).setName(CCONTENTPROVIDER_SELCRITERIA))); + + } + + concept.setParams(params); + dts.getConcepts().add(concept); + + + } + + } + + public void updateOrganization(String communityId, CommunityOrganization organization) throws CommunityException { + + final Context context = getContextMap().get(communityId); + Category orgs = context.getCategories().get(communityId + ORGANIZATION_ID_SUFFIX); + if (orgs.getConcepts().stream().map(c -> c.getId()).collect(Collectors.toList()) + .contains(communityId + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organization.getId())){ + orgs.getConcepts().forEach(concept -> { + if (concept.getId().equals(communityId + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organization.getId())) { + + + if (organization.getName() != null) { + if(concept.getParams().keySet().contains(CORGANIZATION_NAME)){ + concept.getParams().replace(CORGANIZATION_NAME, Arrays.asList(new Param() + .setName(CORGANIZATION_NAME).setValue(organization.getName()))); + } + else{ + concept.getParams().put(CORGANIZATION_NAME, Arrays.asList(new Param() + .setName(CORGANIZATION_NAME).setValue(organization.getName()))); + } + + } + if (organization.getLogo_url() != null) { + if(concept.getParams().keySet().contains(CORGANIZATION_LOGOURL)){ + concept.getParams().replace(CORGANIZATION_LOGOURL, Arrays.asList(new Param() + .setName(CORGANIZATION_LOGOURL).setValue(Base64.getEncoder().encodeToString(organization.getLogo_url().getBytes())))); + } + else{ + concept.getParams().put(CORGANIZATION_LOGOURL, Arrays.asList(new Param() + .setName(CORGANIZATION_LOGOURL).setValue(Base64.getEncoder().encodeToString(organization.getLogo_url().getBytes())))); + } + + } + + + if (organization.getWebsite_url() != null) { + if(concept.getParams().keySet().contains(CORGANIZATION_WEBSITEURL)){ + concept.getParams().replace(CORGANIZATION_WEBSITEURL, Arrays.asList(new Param() + .setName(CORGANIZATION_WEBSITEURL).setValue(Base64.getEncoder().encodeToString(organization.getWebsite_url().getBytes())))); + } + else{ + concept.getParams().put(CORGANIZATION_WEBSITEURL, Arrays.asList(new Param() + .setName(CORGANIZATION_WEBSITEURL).setValue(Base64.getEncoder().encodeToString(organization.getWebsite_url().getBytes())))); + } + + } + + } + + }); + } + else{ + + Concept concept = new Concept(); + concept.setId(communityId + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organization.getId()); + concept.setClaim(false); + concept.setLabel(""); + + Map> params = new TreeMap<>(); + + if (organization.getName() != null) { + params.put( CORGANIZATION_NAME, Arrays.asList(new Param().setValue(organization.getName()).setName(CORGANIZATION_NAME))); + } + if(organization.getLogo_url()!= null){ + + params.put( CORGANIZATION_LOGOURL, Arrays.asList(new Param().setValue(Base64.getEncoder().encodeToString(organization.getLogo_url().getBytes())).setName(CORGANIZATION_LOGOURL))); + } + if (organization.getWebsite_url() != null){ + params.put( CORGANIZATION_WEBSITEURL, Arrays.asList(new Param().setValue(Base64.getEncoder().encodeToString(organization.getWebsite_url().getBytes())).setName(CORGANIZATION_WEBSITEURL))); + } + + + concept.setParams(params); + orgs.getConcepts().add(concept); + + + } + + } + + public void updateZenodoCommunity(String communityId, CommunityZenodoCommunity zc) throws CommunityException { + final Context context = getContextMap().get(communityId); + Category zcs = context.getCategories().get(communityId + ZENODOCOMMUNITY_ID_SUFFIX); + if (zcs.getConcepts().stream().map(c -> c.getId()).collect(Collectors.toList()) + .contains(communityId + ZENODOCOMMUNITY_ID_SUFFIX + ID_SEPARATOR + zc.getId())){ + zcs.getConcepts().forEach(concept -> { + if (concept.getId().equals(communityId + ZENODOCOMMUNITY_ID_SUFFIX + ID_SEPARATOR + zc.getId())) { + + + if (zc.getZenodoid() != null) { + if(concept.getParams().keySet().contains(CZENODOCOMMUNITY_ID)){ + concept.getParams().replace(CZENODOCOMMUNITY_ID, Arrays.asList(new Param() + .setName(CZENODOCOMMUNITY_ID).setValue(zc.getZenodoid()))); + } + else{ + concept.getParams().put(CZENODOCOMMUNITY_ID, Arrays.asList(new Param() + .setName(CZENODOCOMMUNITY_ID).setValue(zc.getZenodoid()))); + } + + } + + } + + }); + } + else{ + + Concept concept = new Concept(); + concept.setId(communityId + ZENODOCOMMUNITY_ID_SUFFIX + ID_SEPARATOR + zc.getId()); + concept.setClaim(false); + + + Map> params = new TreeMap<>(); + + if (zc.getZenodoid() != null) { + params.put( CZENODOCOMMUNITY_ID, Arrays.asList(new Param().setValue(zc.getZenodoid()).setName(CZENODOCOMMUNITY_ID))); + concept.setLabel(zc.getZenodoid()); + }else{ + concept.setLabel(""); + } + concept.setParams(params); + zcs.getConcepts().add(concept); + + } + + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityConstants.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityConstants.java new file mode 100644 index 00000000..9ac100f9 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityConstants.java @@ -0,0 +1,58 @@ +package eu.dnetlib.openaire.community; + +import java.util.Set; + +import com.google.common.collect.Sets; + +public class CommunityConstants { + + public final static Set communityBlackList = Sets.newHashSet("fet-fp7", "fet-h2020"); + + // common + public final static String OPENAIRE_ID = "openaireId"; + public final static String PIPE_SEPARATOR = "||"; + public final static String ID_SEPARATOR = "::"; + public final static String CSV_DELIMITER = ","; + public final static String CLABEL = "label"; + + // id suffixes + public final static String PROJECTS_ID_SUFFIX = ID_SEPARATOR + "projects"; + public final static String CONTENTPROVIDERS_ID_SUFFIX = ID_SEPARATOR + "contentproviders"; + public final static String ZENODOCOMMUNITY_ID_SUFFIX = ID_SEPARATOR + "zenodocommunities"; + public final static String ORGANIZATION_ID_SUFFIX = ID_SEPARATOR + "organizations"; + + // community summary + public final static String CSUMMARY_DESCRIPTION = "description"; + public final static String CSUMMARY_LOGOURL = "logourl"; + public final static String CSUMMARY_STATUS = "status"; + public final static String CSUMMARY_NAME = "name"; + public final static String CSUMMARY_MANAGER = "manager"; + public final static String CSUMMARY_ZENODOC = "zenodoCommunity"; + + // community profile + public final static String CPROFILE_SUBJECT = "subject"; + public final static String CPROFILE_CREATIONDATE = "creationdate"; + + // community project + public final static String CPROJECT_FUNDER = "funder"; + public final static String CPROJECT_NUMBER = "CD_PROJECT_NUMBER"; + public final static String CPROJECT_FULLNAME = "projectfullname"; + public final static String CPROJECT_ACRONYM = "acronym"; + + // community content provider + public final static String CCONTENTPROVIDER_NAME = "name"; + public final static String CCONTENTPROVIDER_OFFICIALNAME = "officialname"; + public final static String CCONTENTPROVIDER_ENABLED = "enabled"; + public final static String CCONTENTPROVIDERENABLED_DEFAULT = "true"; + public final static String CCONTENTPROVIDER_SELCRITERIA = "selcriteria"; + + //community zenodo community + public final static String CZENODOCOMMUNITY_ID = "zenodoid"; + + //community organization + public final static String CORGANIZATION_NAME = "name"; + public final static String CORGANIZATION_LOGOURL = "logourl"; + public final static String CORGANIZATION_WEBSITEURL = "websiteurl"; + + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityContentprovider.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityContentprovider.java new file mode 100644 index 00000000..9d470dc7 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityContentprovider.java @@ -0,0 +1,105 @@ +package eu.dnetlib.openaire.community; + +import javax.validation.constraints.NotNull; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.google.gson.Gson; +import eu.dnetlib.openaire.community.selectioncriteria.SelectionCriteria; +import io.swagger.annotations.ApiModelProperty; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +@JsonAutoDetect +public class CommunityContentprovider { + + private static final Log log = LogFactory.getLog(CommunityContentprovider.class); + + @ApiModelProperty(value = "OpenAIRE identifier for this content provider, if available", required = false) + private String openaireId; + + @NotNull + @ApiModelProperty(value = "the community identifier this content provider belongs to", required = true) + private String communityId; + + @NotNull + @ApiModelProperty(value = "identifies this content provider within the context it belongs to", required = true) + private String id; + + @ApiModelProperty(value = "content provider name", required = false) + private String name; + + @NotNull + @ApiModelProperty(value = "content provider official name", required = true) + private String officialname; + + //@NotNull + @ApiModelProperty(value = "content provider selection criteria", required = false) + private SelectionCriteria selectioncriteria; + + public String getOpenaireId() { + return openaireId; + } + + public void setOpenaireId(final String openaireId) { + this.openaireId = openaireId; + } + + public String getCommunityId() { + return communityId; + } + + public void setCommunityId(final String communityId) { + this.communityId = communityId; + } + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + public String getOfficialname() { + return officialname; + } + + public void setOfficialname(final String officialname) { + this.officialname = officialname; + } + + public SelectionCriteria getSelectioncriteria() { + + return this.selectioncriteria; + } + + public void setSelectioncriteria(SelectionCriteria selectioncriteria) { + this.selectioncriteria = selectioncriteria; + + } + + public String toString(){ + return String.format("id %s, name %s, selection criteria %s" , this.id, this.name, toJson()); + } + + + public String toJson() { + if (selectioncriteria == null) + return ""; + return new Gson().toJson(selectioncriteria); + } + + public String toXML() { + if (selectioncriteria == null) + return ""; + return ""; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityDetails.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityDetails.java new file mode 100644 index 00000000..5d315dd7 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityDetails.java @@ -0,0 +1,51 @@ +package eu.dnetlib.openaire.community; + +import java.util.Date; +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import io.swagger.annotations.ApiModelProperty; + +@JsonAutoDetect +public class CommunityDetails extends CommunitySummary { + + @ApiModelProperty("date of creation for this community") + private Date creationDate; + + @ApiModelProperty("date of the last update for this communityu") + private Date lastUpdateDate; + + @ApiModelProperty("list of subjects (keywords) that characterise this community") + private List subjects; + + public CommunityDetails() { + } + + public CommunityDetails(final CommunitySummary summary) { + super(summary); + } + + public Date getCreationDate() { + return creationDate; + } + + public void setCreationDate(final Date creationDate) { + this.creationDate = creationDate; + } + + public List getSubjects() { + return subjects; + } + + public void setSubjects(final List subjects) { + this.subjects = subjects; + } + + public Date getLastUpdateDate() { + return lastUpdateDate; + } + + public void setLastUpdateDate(Date lastUpdateDate) { + this.lastUpdateDate = lastUpdateDate; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityException.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityException.java new file mode 100644 index 00000000..9b2c2746 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityException.java @@ -0,0 +1,20 @@ +package eu.dnetlib.openaire.community; + +import java.io.IOException; + +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.ResponseStatus; + +@ResponseBody +@ResponseStatus(value = HttpStatus.INTERNAL_SERVER_ERROR) +public class CommunityException extends Exception { + + public CommunityException(final String message) { + super(message); + } + + public CommunityException(final IOException e) { + super(e); + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityMappingUtils.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityMappingUtils.java new file mode 100644 index 00000000..271c1cbe --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityMappingUtils.java @@ -0,0 +1,231 @@ +package eu.dnetlib.openaire.community; + +import java.text.ParseException; +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import com.google.common.escape.Escaper; +import com.google.common.xml.XmlEscapers; +import eu.dnetlib.openaire.community.selectioncriteria.SelectionCriteria; +import eu.dnetlib.openaire.context.Concept; +import eu.dnetlib.openaire.context.Context; +import eu.dnetlib.openaire.context.Param; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import static eu.dnetlib.openaire.common.Utils.escape; +import static eu.dnetlib.openaire.community.CommunityConstants.*; + +public class CommunityMappingUtils { + + private final static String pattern = "yyyy-MM-dd'T'hh:mm:ss"; + + private static final Log log = LogFactory.getLog(CommunityMappingUtils.class); + + public static CommunitySummary asCommunitySummary(final Context c) { + final CommunitySummary summary = new CommunitySummary(); + + summary.setId(c.getId()); + summary.setShortName(c.getLabel()); + summary.setLastUpdateDate(c.getLastUpdateDate()); + summary.setCreationDate(c.getCreationDate()); + summary.setQueryId(c.getId() + PIPE_SEPARATOR + c.getLabel()); + summary.setType(c.getType()); + + final Map> params = c.getParams(); + if (params.containsKey(CSUMMARY_DESCRIPTION)) { + summary.setDescription(asCsv(params.get(CSUMMARY_DESCRIPTION))); + } + if (params.containsKey(CSUMMARY_LOGOURL)) { + summary.setLogoUrl(asCsv(params.get(CSUMMARY_LOGOURL))); + } + if (params.containsKey(CSUMMARY_STATUS)) { + summary.setStatus(CommunityStatus.valueOf(firstValue(params, CSUMMARY_STATUS))); + } + if (params.containsKey(CSUMMARY_NAME)) { + summary.setName(asCsv(params.get(CSUMMARY_NAME))); + } + if (params.containsKey(CSUMMARY_ZENODOC)) { + summary.setZenodoCommunity(asCsv(params.get(CSUMMARY_ZENODOC))); + } + + return summary; + } + + public static CommunityDetails asCommunityProfile(final Context c) { + + final CommunityDetails p = new CommunityDetails(asCommunitySummary(c)); + p.setLastUpdateDate(c.getLastUpdateDate()); + final Map> params = c.getParams(); + if (params.containsKey(CPROFILE_SUBJECT)) { + p.setSubjects(splitValues(asValues(params.get(CPROFILE_SUBJECT)), CSV_DELIMITER)); + } + if (params.containsKey(CPROFILE_CREATIONDATE)){ + try { + p.setCreationDate(org.apache.commons.lang3.time.DateUtils.parseDate(asCsv(params.get(CPROFILE_CREATIONDATE)), pattern)); + }catch(ParseException e) { + e.printStackTrace(); + } + } + + return p; + } + + public static CommunityProject asCommunityProject(final String communityId, final Concept c) { + + final Map> p = c.getParams(); + final CommunityProject project = new CommunityProject(); + project.setCommunityId(communityId); + project.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR)); + project.setOpenaireId(firstValue(p, OPENAIRE_ID)); + project.setFunder(firstValue(p, CPROJECT_FUNDER)); + project.setGrantId(firstValue(p, CPROJECT_NUMBER)); + project.setName(firstValue(p, CPROJECT_FULLNAME)); + project.setAcronym(firstValue(p, CPROJECT_ACRONYM)); + + return project; + } + + public static CommunityContentprovider asCommunityDataprovider(final String communityId, final Concept c) { + + final Map> p = c.getParams(); + final CommunityContentprovider d = new CommunityContentprovider(); + d.setCommunityId(communityId); + d.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR)); + d.setOpenaireId(firstValue(p, OPENAIRE_ID)); + d.setName(firstValue(p, CCONTENTPROVIDER_NAME)); + d.setOfficialname(firstValue(p, CCONTENTPROVIDER_OFFICIALNAME)); + d.setSelectioncriteria(SelectionCriteria.fromJson(firstValue(p, CCONTENTPROVIDER_SELCRITERIA))); + return d; + } + + public static CommunityZenodoCommunity asCommunityZenodoCommunity(final String communityId, final Concept c){ + final CommunityZenodoCommunity z = new CommunityZenodoCommunity(); + final Map> p = c.getParams(); + z.setCommunityId(communityId); + z.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR)); + z.setZenodoid(firstValue(p,CZENODOCOMMUNITY_ID)); + //z.setName(c.getLabel()); + return z; + } + + + public static CommunityOrganization asCommunityOrganization(String id, Concept c) { + final Map> p = c.getParams(); + final CommunityOrganization o = new CommunityOrganization(); + o.setCommunityId(id); + o.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR)); + o.setName(firstValue(p,CORGANIZATION_NAME)); + o.setLogo_url(getDecodedUrl(firstValue(p,CORGANIZATION_LOGOURL))); + o.setWebsite_url(getDecodedUrl(firstValue(p,CORGANIZATION_WEBSITEURL))); + + return o; + } + + private static String getDecodedUrl(final String encoded_url){ + if(encoded_url == null){ + return encoded_url; + } + return new String(Base64.getDecoder().decode(encoded_url)); + } + + + private static List splitValues(final Stream stream, final String separator) { + return stream.map(s -> s.split(separator)) + .map(Arrays::asList) + .flatMap(List::stream) + .filter(StringUtils::isNotBlank) + .map(StringUtils::trim) + .collect(Collectors.toList()); + } + + private static String firstValue(final Map> p, final String paramName) { + return asValues(p.get(paramName)).findFirst().orElse(null); + } + + private static String asCsv(final List params) { + return asValues(params) + .collect(Collectors.joining(CSV_DELIMITER)); + } + + private static Stream asValues(final List params) { + return params == null ? Stream.empty() : params.stream() + .map(Param::getValue) + .map(StringUtils::trim) + .distinct(); + } + + public static String asProjectXML(final String contextId, final CommunityProject project) { + final Escaper esc = XmlEscapers.xmlAttributeEscaper(); + final StringBuilder sb = new StringBuilder(); + sb.append( + String.format( + "\n", + escape(esc, contextId), PROJECTS_ID_SUFFIX, ID_SEPARATOR, escape(esc, String.valueOf(project.getId())), escape(esc, project.getAcronym()))); + sb.append(paramXML(CPROJECT_FULLNAME, project.getName())); + sb.append(paramXML(CPROJECT_ACRONYM, project.getAcronym())); + sb.append(paramXML(CPROJECT_NUMBER, project.getGrantId())); + sb.append(paramXML(CPROJECT_FUNDER, project.getFunder())); + sb.append(paramXML(OPENAIRE_ID, project.getOpenaireId())); + sb.append("\n"); + return sb.toString(); + } + + public static String asContentProviderXML(final String contextId, final CommunityContentprovider ccp) { + log.info("creating the XML for the content provider"); + final Escaper esc = XmlEscapers.xmlAttributeEscaper(); + final StringBuilder sb = new StringBuilder(); + sb.append( + String.format( + "\n", + escape(esc, contextId), CONTENTPROVIDERS_ID_SUFFIX, ID_SEPARATOR, escape(esc, String.valueOf(ccp.getId())), escape(esc, ccp.getName()))); + sb.append(paramXML(OPENAIRE_ID, ccp.getOpenaireId())); + sb.append(paramXML(CCONTENTPROVIDER_NAME, ccp.getName())); + sb.append(paramXML(CCONTENTPROVIDER_OFFICIALNAME, ccp.getOfficialname())); + sb.append(paramXML(CCONTENTPROVIDER_ENABLED,CCONTENTPROVIDERENABLED_DEFAULT)); + sb.append(paramXMLNoEscape(CCONTENTPROVIDER_SELCRITERIA, ccp.toXML())); + sb.append("\n"); + log.info(sb.toString()); + return sb.toString(); + } + + public static String asZenodoCommunityXML(final String contextId, final CommunityZenodoCommunity zc) { + final Escaper esc = XmlEscapers.xmlAttributeEscaper(); + final StringBuilder sb = new StringBuilder(); + sb.append( + String.format( + "\n", + escape(esc, contextId), ZENODOCOMMUNITY_ID_SUFFIX, ID_SEPARATOR, escape(esc, String.valueOf(zc.getId())), escape(esc, zc.getZenodoid()))); + + sb.append(paramXML(CZENODOCOMMUNITY_ID, zc.getZenodoid())); + sb.append("\n"); + return sb.toString(); + } + + + public static String asOrganizationXML(final String contextId, CommunityOrganization organization) { + final Escaper esc = XmlEscapers.xmlAttributeEscaper(); + final StringBuilder sb = new StringBuilder(); + sb.append( + String.format( + "\n", + escape(esc, contextId), ORGANIZATION_ID_SUFFIX, ID_SEPARATOR, escape(esc, String.valueOf(organization.getId())), escape(esc, organization.getName()))); + sb.append(paramXML(CORGANIZATION_NAME, organization.getName())); + sb.append(paramXML(CORGANIZATION_LOGOURL, Base64.getEncoder().encodeToString(organization.getLogo_url().getBytes()))); + sb.append(paramXML(CORGANIZATION_WEBSITEURL,Base64.getEncoder().encodeToString(organization.getWebsite_url().getBytes()))); + sb.append("\n"); + return sb.toString(); + } + + + private static String paramXML(final String paramName, final String value) { + return String.format("%s\n", paramName, escape(XmlEscapers.xmlContentEscaper(), value)); + } + + private static String paramXMLNoEscape(final String paramName, final String value) { + return String.format("%s\n", paramName, value); + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityNotFoundException.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityNotFoundException.java new file mode 100644 index 00000000..3138d64a --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityNotFoundException.java @@ -0,0 +1,19 @@ +package eu.dnetlib.openaire.community; + +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.ResponseStatus; + +@ResponseBody +@ResponseStatus(value = HttpStatus.NOT_FOUND) +public class CommunityNotFoundException extends Exception { + + public CommunityNotFoundException(final String msg) { + super(msg); + } + + public CommunityNotFoundException(final Exception e) { + super(e); + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityOpenAIRECommunities.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityOpenAIRECommunities.java new file mode 100644 index 00000000..3cf3403f --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityOpenAIRECommunities.java @@ -0,0 +1,42 @@ +package eu.dnetlib.openaire.community; + +import io.swagger.annotations.ApiModelProperty; + +import javax.validation.constraints.NotNull; +import java.util.ArrayList; +import java.util.List; + +public class CommunityOpenAIRECommunities { + + @NotNull + @ApiModelProperty(value = "the zenodo community identifier", required = true) + private String zenodoid; + + @NotNull + @ApiModelProperty(value = "identifies this zenodo community within the context it belongs to", required = true) + private List openAirecommunitylist; + + public CommunityOpenAIRECommunities() { + this.zenodoid = ""; + openAirecommunitylist=new ArrayList<>(); + } + + public List getOpenAirecommunitylist() { + return openAirecommunitylist; + } + + public CommunityOpenAIRECommunities setOpenAirecommunitylist(List openAirecommunitylist) { + this.openAirecommunitylist = openAirecommunitylist; + return this; + } + + public String getZenodoid() { + return zenodoid; + } + + public CommunityOpenAIRECommunities setZenodoid(String zenodoid) { + this.zenodoid = zenodoid; + return this; + } + +} \ No newline at end of file diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityOrganization.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityOrganization.java new file mode 100644 index 00000000..7ced9784 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityOrganization.java @@ -0,0 +1,75 @@ +package eu.dnetlib.openaire.community; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +import io.swagger.annotations.ApiModelProperty; +import javax.validation.constraints.NotNull; + +@JsonAutoDetect +public class CommunityOrganization { + @NotNull + @ApiModelProperty(value = "the community identifier this organization belongs to", required = true) + private String communityId; + + @NotNull + @ApiModelProperty(value = "name of the organization", required = true) + private String name; + + @NotNull + @ApiModelProperty(value = "identifies this organization within the context it belongs to", required = true) + private String id; + + @NotNull + @ApiModelProperty(value="url of the logo for this organization", required = true) + private String logo_url; + + + @NotNull + @ApiModelProperty(value="website url for this organization", required = true) + private String website_url; + + public String getCommunityId() { + return communityId; + } + + public CommunityOrganization setCommunityId(String communityId) { + this.communityId = communityId; + return this; + } + + public String getName() { + return name; + } + + public CommunityOrganization setName(String name) { + this.name = name; + return this; + } + + public String getId() { + return id; + } + + public CommunityOrganization setId(String id) { + this.id = id; + return this; + } + + public String getLogo_url() { + return logo_url; + } + + public CommunityOrganization setLogo_url(String logo_url) { + this.logo_url = logo_url; + return this; + } + + public String getWebsite_url() { + return website_url; + } + + public CommunityOrganization setWebsite_url(String website_url) { + this.website_url = website_url; + return this; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityProject.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityProject.java new file mode 100644 index 00000000..205e844a --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityProject.java @@ -0,0 +1,86 @@ +package eu.dnetlib.openaire.community; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import io.swagger.annotations.ApiModelProperty; + +@JsonAutoDetect +public class CommunityProject { + + @ApiModelProperty(value = "OpenAIRE identifier for this project, if available", required = false) + private String openaireId; + + @ApiModelProperty(value = "the community identifier this project belongs to", required = true) + private String communityId; + + @ApiModelProperty(value = "identifies this project within the context it belongs to", required = true) + private String id; + + @ApiModelProperty(value = "project name", required = true) + private String name; + + @ApiModelProperty(value = "project acronym", required = false) + private String acronym; + + @ApiModelProperty(value = "project funder", required = true) + private String funder; + + @ApiModelProperty(value = "project grant id", required = true) + private String grantId; + + public String getOpenaireId() { + return openaireId; + } + + public void setOpenaireId(final String openaireId) { + this.openaireId = openaireId; + } + + public String getCommunityId() { + return communityId; + } + + public void setCommunityId(final String communityId) { + this.communityId = communityId; + } + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + public String getAcronym() { + return acronym; + } + + public void setAcronym(final String acronym) { + this.acronym = acronym; + } + + public String getFunder() { + return funder; + } + + public void setFunder(final String funder) { + this.funder = funder; + } + + public String getGrantId() { + return grantId; + } + + public void setGrantId(final String grantId) { + this.grantId = grantId; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityStatus.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityStatus.java new file mode 100644 index 00000000..e3b4902a --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityStatus.java @@ -0,0 +1,17 @@ +package eu.dnetlib.openaire.community; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import io.swagger.annotations.ApiModelProperty; + +@JsonAutoDetect +public enum CommunityStatus { + + @ApiModelProperty("restricted visibility") + hidden, + + @ApiModelProperty("visible only to RCD managers") + manager, + + @ApiModelProperty("visible to RCD managers and to the community users") + all +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunitySummary.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunitySummary.java new file mode 100644 index 00000000..c3453e0e --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunitySummary.java @@ -0,0 +1,167 @@ +package eu.dnetlib.openaire.community; + +import java.util.Date; +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import io.swagger.annotations.ApiModelProperty; + +@JsonAutoDetect +public class CommunitySummary { + + @ApiModelProperty("identifies the community") + protected String id; + + @ApiModelProperty("values for this field reflect the index field _community_ in the index, e.g. 'egi||EGI Federation'") + protected String queryId; + + @ApiModelProperty("community type") + protected String type; + + @ApiModelProperty("community name") + protected String name; + + @ApiModelProperty("community short name") + protected String shortName; + + @ApiModelProperty("community creation date") + protected Date creationDate; + + @ApiModelProperty("community last update date") + protected Date lastUpdateDate; + + @ApiModelProperty("community description") + protected String description; + + @ApiModelProperty("http url for the community logo") + protected String logoUrl; + + @ApiModelProperty("status of the community, drives its visibility") + protected CommunityStatus status; + + @ApiModelProperty("Zenodo community associated to this community") + protected String zenodoCommunity; + + public CommunitySummary() { + } + + public CommunitySummary( + final String id, + final String queryId, + final String type, + final String name, + final String shortName, + final Date creationDate, + final Date lastUpdateDate, + final String description, + final String logoUrl, + final CommunityStatus status, + final String zenodoCommunity) { + this.id = id; + this.queryId = queryId; + this.type = type; + this.name = name; + this.shortName = shortName; + this.creationDate = creationDate; + this.lastUpdateDate = lastUpdateDate; + this.description = description; + this.logoUrl = logoUrl; + this.status = status; + this.zenodoCommunity = zenodoCommunity; + } + + public CommunitySummary(final CommunitySummary summary) { + this(summary.getId(), + summary.getQueryId(), + summary.getType(), + summary.getName(), + summary.getShortName(), + summary.getCreationDate(), + summary.getLastUpdateDate(), + summary.getDescription(), + summary.getLogoUrl(), + summary.getStatus(), + summary.getZenodoCommunity()); + } + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + public String getQueryId() { + return queryId; + } + + public void setQueryId(final String queryId) { + this.queryId = queryId; + } + + public String getType() { + return type; + } + + public void setType(final String type) { + this.type = type; + } + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + public String getShortName() { + return shortName; + } + + public void setShortName(final String shortName) { + this.shortName = shortName; + } + + public Date getCreationDate() { return creationDate; } + + public void setCreationDate(final Date creationDate) { this.creationDate = creationDate; } + + public Date getLastUpdateDate() { return lastUpdateDate; } + + public void setLastUpdateDate(final Date lastUpdateDate) { this.lastUpdateDate = lastUpdateDate; } + + public String getDescription() { + return description; + } + + public void setDescription(final String description) { + this.description = description; + } + + public String getLogoUrl() { + return logoUrl; + } + + public void setLogoUrl(final String logoUrl) { + this.logoUrl = logoUrl; + } + + public CommunityStatus getStatus() { + return status; + } + + public void setStatus(final CommunityStatus status) { + this.status = status; + } + + public String getZenodoCommunity() { + return zenodoCommunity; + } + + public void setZenodoCommunity(String zenodoCommunity) { + this.zenodoCommunity = zenodoCommunity; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityWritableProperties.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityWritableProperties.java new file mode 100644 index 00000000..08ca7a26 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityWritableProperties.java @@ -0,0 +1,96 @@ +package eu.dnetlib.openaire.community; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import io.swagger.annotations.ApiModelProperty; + +@JsonAutoDetect +public class CommunityWritableProperties { + + @ApiModelProperty("community name") + private String name; + + @ApiModelProperty("community short name") + private String shortName; + + @ApiModelProperty("community description") + private String description; + + @ApiModelProperty("http url for the community logo") + private String logoUrl; + + @ApiModelProperty("list of subjects (keywords) that characterise this community") + private List subjects; + + @ApiModelProperty("status of the community, drives its visibility") + private CommunityStatus status; + + @ApiModelProperty("id of the main Zenodo community") + private String mainZenodoCommunity; + + + public static CommunityWritableProperties fromDetails(final CommunityDetails details) { + CommunityWritableProperties p = new CommunityWritableProperties(); + p.setName(details.getName()); + p.setShortName(details.getShortName()); + p.setDescription(details.getDescription()); + p.setLogoUrl(details.getLogoUrl()); + p.setSubjects(details.getSubjects()); + p.setStatus(details.getStatus()); + p.setMainZenodoCommunity(details.getZenodoCommunity()); + return p; + } + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + public String getShortName() { + return shortName; + } + + public void setShortName(final String shortName) { + this.shortName = shortName; + } + + public String getDescription() { + return description; + } + + public void setDescription(final String description) { + this.description = description; + } + + public String getLogoUrl() { + return logoUrl; + } + + public void setLogoUrl(final String logoUrl) { + this.logoUrl = logoUrl; + } + + public List getSubjects() { + return subjects; + } + + public void setSubjects(final List subjects) { + this.subjects = subjects; + } + + public CommunityStatus getStatus() { + return status; + } + + public void setStatus(final CommunityStatus status) { + this.status = status; + } + + public String getMainZenodoCommunity() { return mainZenodoCommunity; } + + public void setMainZenodoCommunity(String mainZenodoCommunity) { this.mainZenodoCommunity = mainZenodoCommunity; } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityZenodoCommunity.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityZenodoCommunity.java new file mode 100644 index 00000000..c84c1692 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityZenodoCommunity.java @@ -0,0 +1,47 @@ +package eu.dnetlib.openaire.community; + +import javax.validation.constraints.NotNull; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import io.swagger.annotations.ApiModelProperty; + +@JsonAutoDetect +public class CommunityZenodoCommunity { + + @NotNull + @ApiModelProperty(value = "the community identifier this zenodo Community belongs to", required = true) + private String communityId; + + @NotNull + @ApiModelProperty(value = "Zenodo identifier for this community", required = true) + private String zenodoid; + + @NotNull + @ApiModelProperty(value = "identifies this zenodo community within the context it belongs to", required = true) + private String id; + + public String getZenodoid() { + return zenodoid; + } + + public void setZenodoid(String zenodoid) { + this.zenodoid = zenodoid; + } + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + public String getCommunityId() { + return communityId; + } + + public void setCommunityId(String communityId) { + this.communityId = communityId; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/selectioncriteria/Constraint.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/selectioncriteria/Constraint.java new file mode 100644 index 00000000..9f31965c --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/selectioncriteria/Constraint.java @@ -0,0 +1,44 @@ +package eu.dnetlib.openaire.community.selectioncriteria; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +import java.io.Serializable; + + +@JsonAutoDetect +public class Constraint implements Serializable { + private String verb; + private String field; + private String value; + + + public Constraint() { + } + + public String getVerb() { + return verb; + } + + public void setVerb(String verb) { + this.verb = verb; + } + + public String getField() { + return field; + } + + public void setField(String field) { + this.field = field; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/selectioncriteria/Constraints.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/selectioncriteria/Constraints.java new file mode 100644 index 00000000..863d97c8 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/selectioncriteria/Constraints.java @@ -0,0 +1,23 @@ +package eu.dnetlib.openaire.community.selectioncriteria; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +import java.io.Serializable; +import java.util.List; + +@JsonAutoDetect +public class Constraints implements Serializable { + + private List constraint; + + + public Constraints() { + } + public List getConstraint() { + return constraint; + } + + public void setConstraint(List constraint) { + this.constraint = constraint; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/selectioncriteria/SelectionCriteria.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/selectioncriteria/SelectionCriteria.java new file mode 100644 index 00000000..3b0c09f1 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/selectioncriteria/SelectionCriteria.java @@ -0,0 +1,30 @@ +package eu.dnetlib.openaire.community.selectioncriteria; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.google.gson.Gson; +import java.io.Serializable; +import java.util.List; + +@JsonAutoDetect +public class SelectionCriteria implements Serializable { + private List criteria; + + public SelectionCriteria() { + } + + + public List getCriteria() { + return criteria; + } + + public void setCriteria(List criteria) { + this.criteria = criteria; + } + + + + public static SelectionCriteria fromJson(final String json) { + return new Gson().fromJson(json, SelectionCriteria.class); + + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/Category.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/Category.java new file mode 100644 index 00000000..1c029777 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/Category.java @@ -0,0 +1,67 @@ +package eu.dnetlib.openaire.context; + +import java.util.List; +import java.util.Map; + +public class Category { + + private String id; + + private String label; + + private boolean claim; + + private Map> params; + + private List concepts; + + public String getId() { + return id; + } + + public String getLabel() { + return label; + } + + public boolean isClaim() { + return claim; + } + + public boolean hasConcepts() { + return getConcepts() != null && !getConcepts().isEmpty(); + } + + public Map> getParams() { + return params; + } + + public List getConcepts() { + return concepts; + } + + public Category setId(final String id) { + this.id = id; + return this; + } + + public Category setLabel(final String label) { + this.label = label; + return this; + } + + public Category setClaim(final boolean claim) { + this.claim = claim; + return this; + } + + public Category setParams(final Map> params) { + this.params = params; + return this; + } + + public Category setConcepts(final List concepts) { + this.concepts = concepts; + return this; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/CategorySummary.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/CategorySummary.java new file mode 100644 index 00000000..c2acb200 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/CategorySummary.java @@ -0,0 +1,38 @@ +package eu.dnetlib.openaire.context; + +public class CategorySummary { + + private String id; + + private String label; + + private boolean hasConcept; + + public String getId() { + return id; + } + + public String getLabel() { + return label; + } + + public boolean isHasConcept() { + return hasConcept; + } + + public CategorySummary setId(final String id) { + this.id = id; + return this; + } + + public CategorySummary setLabel(final String label) { + this.label = label; + return this; + } + + public CategorySummary setHasConcept(final boolean hasConcept) { + this.hasConcept = hasConcept; + return this; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/Concept.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/Concept.java new file mode 100644 index 00000000..ddbcdec1 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/Concept.java @@ -0,0 +1,67 @@ +package eu.dnetlib.openaire.context; + +import java.util.List; +import java.util.Map; + +public class Concept { + + private String id; + + private String label; + + private boolean claim; + + private Map> params; + + private List concepts; + + public boolean hasSubConcepts() { + return getConcepts() != null && !getConcepts().isEmpty(); + } + + public String getId() { + return id; + } + + public String getLabel() { + return label; + } + + public boolean isClaim() { + return claim; + } + + public Map> getParams() { + return params; + } + + public List getConcepts() { + return concepts; + } + + public Concept setId(final String id) { + this.id = id; + return this; + } + + public Concept setLabel(final String label) { + this.label = label; + return this; + } + + public Concept setClaim(final boolean claim) { + this.claim = claim; + return this; + } + + public Concept setParams(final Map> params) { + this.params = params; + return this; + } + + public Concept setConcepts(final List concepts) { + this.concepts = concepts; + return this; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ConceptSummary.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ConceptSummary.java new file mode 100644 index 00000000..58600836 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ConceptSummary.java @@ -0,0 +1,51 @@ +package eu.dnetlib.openaire.context; + +import java.util.List; + +public class ConceptSummary { + + private String id; + + private String label; + + public boolean hasSubConcept; + + private List concepts; + + public String getId() { + return id; + } + + public String getLabel() { + return label; + } + + public List getConcepts() { + return concepts; + } + + public ConceptSummary setId(final String id) { + this.id = id; + return this; + } + + public ConceptSummary setLabel(final String label) { + this.label = label; + return this; + } + + public boolean isHasSubConcept() { + return hasSubConcept; + } + + public ConceptSummary setHasSubConcept(final boolean hasSubConcept) { + this.hasSubConcept = hasSubConcept; + return this; + } + + public ConceptSummary setConcept(final List concepts) { + this.concepts = concepts; + return this; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/Context.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/Context.java new file mode 100644 index 00000000..badc1ff1 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/Context.java @@ -0,0 +1,91 @@ +package eu.dnetlib.openaire.context; + +import java.util.Date; +import java.util.List; +import java.util.Map; + +public class Context { + + private String id; + + private String label; + + private String type; + + private Date creationDate; + + private Date lastUpdateDate; + + private Map> params; + + private Map categories; + + public String getId() { + return id; + } + + public String getLabel() { + return label; + } + + public String getType() { + return type; + } + + public Date getCreationDate() { + return creationDate; + } + + public Map> getParams() { + return params; + } + + public Map getCategories() { + return categories; + } + + + + public Context setId(final String id) { + this.id = id; + return this; + } + + public Context setLabel(final String label) { + this.label = label; + return this; + } + + public Context setType(final String type) { + this.type = type; + return this; + } + + public Context setCreationDate(final Date dateofcreation) { + this.creationDate = dateofcreation; + return this; + } + + + public Date getLastUpdateDate() { + return lastUpdateDate; + } + + public Context setLastUpdateDate(Date lastUpdateDate) { + this.lastUpdateDate = lastUpdateDate; + return this; + } + + + public Context setParams(final Map> params) { + this.params = params; + return this; + } + + public Context setCategories(final Map categories) { + this.categories = categories; + return this; + } + +} + diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextApiController.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextApiController.java new file mode 100644 index 00000000..64aae1fe --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextApiController.java @@ -0,0 +1,86 @@ +package eu.dnetlib.openaire.context; + +import java.util.List; +import java.util.Optional; + +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiResponse; +import io.swagger.annotations.ApiResponses; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.web.bind.annotation.*; + +@RestController +@CrossOrigin(origins = { "*" }) +@ConditionalOnProperty(value = "openaire.exporter.enable.context", havingValue = "true") +@io.swagger.annotations.Api(tags = "OpenAIRE Context API", description = "the OpenAIRE Context API") +public class ContextApiController { + + @Autowired + private ContextApiCore contextApiCore; + + @RequestMapping(value = "/contexts", produces = { "application/json" }, method = RequestMethod.GET) + @ApiOperation( + value = "list brief information about all the context profiles", + notes = "list brief information about all the context profiles.", + tags = { }, + response = ContextSummary[].class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = ContextSummary[].class), + @ApiResponse(code = 500, message = "unexpected error", response = ContextException.class) }) + public List listContexts(@RequestParam(required = false, defaultValue = "") List type) throws ContextException { + return contextApiCore.listContexts(type); + } + + @RequestMapping(value = "/context/{contextId}", produces = { "application/json" }, method = RequestMethod.GET) + @ApiOperation( + value = "list the categories defined within a context", + notes = "list the categories defined within a context", + tags = { }, + response = CategorySummary[].class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = CategorySummary[].class), + @ApiResponse(code = 500, message = "unexpected error", response = ContextException.class) }) + public List listCategories( + @PathVariable final String contextId, + @RequestParam(required = false, defaultValue = "false") final Boolean all) throws ContextException { + + Boolean allFilter = Optional.ofNullable(all).orElse(false); + return contextApiCore.listCategories(contextId, allFilter); + } + + @RequestMapping(value = "/context/category/{categoryId}", produces = { "application/json" }, method = RequestMethod.GET) + @ApiOperation( + value = "list the concepts defined within a category", + notes = "list the concepts defined within a category", + tags = { }, + response = ConceptSummary[].class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = ConceptSummary[].class), + @ApiResponse(code = 500, message = "unexpected error", response = ContextException.class) }) + public List listConcepts( + @PathVariable final String categoryId, + @RequestParam(required = false, defaultValue = "false") final Boolean all) throws ContextException { + + Boolean allFilter = Optional.ofNullable(all).orElse(false); + return contextApiCore.listConcepts(categoryId, allFilter); + } + + @RequestMapping(value = "/context/category/concept/{conceptId}", produces = { "application/json" }, method = RequestMethod.GET) + @ApiOperation( + value = "list the concepts defined within a category", + notes = "list the concepts defined within a category", + tags = { }, + response = ConceptSummary[].class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = ConceptSummary[].class), + @ApiResponse(code = 500, message = "unexpected error", response = ContextException.class) }) + public List listSubConcepts( + @PathVariable final String conceptId, + @RequestParam(required = false, defaultValue = "false") final Boolean all) throws ContextException { + + Boolean allFilter = Optional.ofNullable(all).orElse(false); + return contextApiCore.listSubConcepts(conceptId, allFilter); + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextApiCore.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextApiCore.java new file mode 100644 index 00000000..95dea205 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextApiCore.java @@ -0,0 +1,120 @@ +package eu.dnetlib.openaire.context; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import com.google.common.base.Splitter; +import com.google.common.collect.Lists; +import eu.dnetlib.openaire.common.ISClient; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +@Component +@ConditionalOnProperty(value = "openaire.exporter.enable.context", havingValue = "true") +public class ContextApiCore { + + private static final Log log = LogFactory.getLog(ContextApiCore.class); + private static final String SEPARATOR = "::"; + + @Autowired + private ISClient isClient; + + public List listContexts(final List type) throws ContextException { + return getContextMap(type).values().stream() + .map(c -> new ContextSummary() + .setId(c.getId()) + .setType(c.getType()) + .setLabel(c.getLabel()) + .setStatus(c.getParams().containsKey("status") ? c.getParams().get("status").get(0).getValue() : "")) + .collect(Collectors.toList()); + } + + public List listCategories(final String contextId, Boolean all) throws ContextException { + final Stream categories = getContextMap().get(contextId).getCategories().values().stream(); + return all ? asCategorySummaries(categories) : asCategorySummaries(categories.filter(Category::isClaim)); + } + + private List asCategorySummaries(Stream categories) { + return categories + .map(c -> new CategorySummary() + .setId(c.getId()) + .setLabel(c.getLabel()) + .setHasConcept(c.hasConcepts())) + .collect(Collectors.toList()); + } + + public List listConcepts(final String categoryId, Boolean all) throws ContextException { + final String contextId = StringUtils.substringBefore(categoryId, SEPARATOR); + final Stream concepts = getContextMap().get(contextId) + .getCategories() + .get(categoryId) + .getConcepts() + .stream(); + + return all ? asConceptSummaries(concepts) : asConceptSummaries(concepts.filter(Concept::isClaim)); + } + + private List asConceptSummaries(Stream concepts) { + return concepts + .map(c -> new ConceptSummary() + .setId(c.getId()) + .setLabel(c.getLabel()) + .setHasSubConcept(c.hasSubConcepts())) + .collect(Collectors.toList()); + } + + public List listSubConcepts(final String conceptId, Boolean all) throws ContextException { + final List ids = Splitter.on(SEPARATOR).splitToList(conceptId); + if (ids.size() < 3) { + throw new ContextException(""); + } + + final String contextId = ids.get(0); + final String categoryId = contextId + SEPARATOR + ids.get(1); + + final Stream concepts = getContextMap().get(contextId) + .getCategories() + .get(categoryId) + .getConcepts() + .stream() + .filter(c -> conceptId.equals(c.getId())); + + return all ? + mapConcepts(concepts.filter(Concept::isClaim).collect(Collectors.toList())) : + mapConcepts(concepts.collect(Collectors.toList())); + } + + private List mapConcepts(final List concepts) { + if (concepts == null || concepts.isEmpty()) { + return null; + } + return concepts.stream() + .map(c -> new ConceptSummary() + .setId(c.getId()) + .setLabel(c.getLabel()) + .setHasSubConcept(c.hasSubConcepts()) + .setConcept(mapConcepts(c.getConcepts()))) + .collect(Collectors.toList()); + } + + private Map getContextMap() throws ContextException { + return getContextMap(Lists.newArrayList()); + } + + private Map getContextMap(final List type) throws ContextException { + try { + return isClient.getContextMap(type); + } catch (IOException e) { + throw new ContextException(e); + } + } + + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextException.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextException.java new file mode 100644 index 00000000..be1fb392 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextException.java @@ -0,0 +1,20 @@ +package eu.dnetlib.openaire.context; + +import java.io.IOException; + +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.ResponseStatus; + +@ResponseBody +@ResponseStatus(value = HttpStatus.INTERNAL_SERVER_ERROR) +public class ContextException extends Exception { + + public ContextException(final String message) { + super(message); + } + + public ContextException(final IOException e) { + super(e); + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextMappingUtils.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextMappingUtils.java new file mode 100644 index 00000000..c3537343 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextMappingUtils.java @@ -0,0 +1,113 @@ +package eu.dnetlib.openaire.context; + +import java.text.ParseException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.time.DateUtils; +import org.dom4j.Document; +import org.dom4j.DocumentException; +import org.dom4j.DocumentHelper; +import org.dom4j.Element; +import org.dom4j.Node; + +import com.google.common.base.Functions; +import com.google.common.collect.Lists; + +import eu.dnetlib.openaire.funders.domain.FunderDetails; + +public class ContextMappingUtils { + + private static final List DATE_PATTERN = Lists.newArrayList("yyyy-MM-dd'T'hh:mm:ss", "yyyy-MM-dd'T'hh:mm:ssXXX", "yyyy-MM-dd'T'hh:mm:ss+00:00"); + + public static Context parseContext(final String s, final Queue errors) { + try { + final Document doc = DocumentHelper.parseText(s); + final Element eContext = (Element) doc.selectSingleNode("/RESOURCE_PROFILE/BODY/CONFIGURATION/context"); + final String creationDate = eContext.valueOf("./param[./@name='creationdate']/text()"); + final Context c = new Context() + .setId(eContext.attributeValue("id")) + .setLabel(eContext.attributeValue("label")) + .setType(eContext.attributeValue("type")) + .setLastUpdateDate(asDate(doc.valueOf("/RESOURCE_PROFILE/HEADER/DATE_OF_CREATION/@value"))) + .setParams(parseParams(eContext)) + .setCategories(parseCategories(eContext)); + // the creation date will be added in the param elements of the community profile. Funders may not have it, hence the check. + if (StringUtils.isNotBlank(creationDate)) { + c.setCreationDate(asDate(creationDate)); + } + return c; + } catch (final DocumentException e) { + errors.add(e); + return new Context(); + } + } + + private static Date asDate(final String s) { + for (final String pattern : DATE_PATTERN) { + try { + return DateUtils.parseDate(s, pattern); + } catch (final ParseException e) {} + } + return null; + } + + private static Map parseCategories(final Element eContext) { + final List eCategory = eContext.selectNodes("//category"); + return eCategory.stream() + .map(n -> (Element) n) + .map(eCat -> new Category() + .setClaim(getClaim(eCat)) + .setId(eCat.attributeValue("id")) + .setLabel(eCat.attributeValue("label")) + .setParams(parseParams(eCat)) + .setConcepts(parseConcepts(eCat))) + .collect(Collectors.toMap(Category::getId, Functions.identity())); + } + + private static List parseConcepts(final Element eCategory) { + final List eConcepts = eCategory.selectNodes("./concept"); + return eConcepts.stream() + .map(n -> (Element) n) + .map(eCon -> new Concept() + .setClaim(getClaim(eCon)) + .setId(eCon.attributeValue("id")) + .setLabel(eCon.attributeValue("label")) + .setParams(parseParams(eCon)) + .setConcepts(parseConcepts(eCon))) + .collect(Collectors.toList()); + } + + private static Boolean getClaim(final Element eCon) { + final String claim = eCon.attributeValue("claim"); + return BooleanUtils.toBooleanObject(StringUtils.isNotBlank(claim) ? claim : "false"); + } + + private static Map> parseParams(final Element e) { + final List params = e.selectNodes("./param"); + return params.stream() + .map(n -> (Element) n) + .map(p -> new Param() + .setName(p.attributeValue("name")) + .setValue(p.getTextTrim())) + .collect(Collectors.toMap(Param::getName, Lists::newArrayList, (p1, p2) -> { + final List p = new ArrayList<>(p1); + p.addAll(p2); + return p; + })); + } + + public static FunderDetails asFunderDetails(final Context c) { + return new FunderDetails() + .setId(c.getId()) + .setName(c.getLabel()) + .setShortname(c.getId()); + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextNotFoundException.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextNotFoundException.java new file mode 100644 index 00000000..212b5334 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextNotFoundException.java @@ -0,0 +1,18 @@ +package eu.dnetlib.openaire.context; + +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.ResponseStatus; + +@ResponseBody +@ResponseStatus(value = HttpStatus.NOT_FOUND) +public class ContextNotFoundException extends Exception { + + public ContextNotFoundException(final String msg) { + super(msg); + } + + public ContextNotFoundException(final Exception e) { + super(e); + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextSummary.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextSummary.java new file mode 100644 index 00000000..9e974a3d --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextSummary.java @@ -0,0 +1,49 @@ +package eu.dnetlib.openaire.context; + +public class ContextSummary { + + private String id; + + private String label; + + private String type; + + private String status; + + public String getId() { + return id; + } + + public String getLabel() { + return label; + } + + public String getType() { + return type; + } + + public String getStatus() { + return status; + } + + public ContextSummary setId(final String id) { + this.id = id; + return this; + } + + public ContextSummary setLabel(final String label) { + this.label = label; + return this; + } + + public ContextSummary setType(final String type) { + this.type = type; + return this; + } + + public ContextSummary setStatus(final String status) { + this.status = status; + return this; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/Param.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/Param.java new file mode 100644 index 00000000..e83a6fd2 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/Param.java @@ -0,0 +1,27 @@ +package eu.dnetlib.openaire.context; + +public class Param { + + private String name; + + private String value; + + public String getName() { + return name; + } + + public String getValue() { + return value; + } + + public Param setName(final String name) { + this.name = name; + return this; + } + + public Param setValue(final String value) { + this.value = value; + return this; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmApiController.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmApiController.java new file mode 100755 index 00000000..c380a6cf --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmApiController.java @@ -0,0 +1,498 @@ +package eu.dnetlib.openaire.dsm; + +import static eu.dnetlib.openaire.common.ExporterConstants.API; +import static eu.dnetlib.openaire.common.ExporterConstants.D; +import static eu.dnetlib.openaire.common.ExporterConstants.DS; +import static eu.dnetlib.openaire.common.ExporterConstants.M; +import static eu.dnetlib.openaire.common.ExporterConstants.R; +import static eu.dnetlib.openaire.common.ExporterConstants.W; + +import java.util.List; + +import javax.validation.Valid; + +import eu.dnetlib.enabling.datasources.common.AggregationInfo; +import eu.dnetlib.openaire.dsm.domain.*; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.time.StopWatch; +import org.apache.http.HttpStatus; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.web.bind.annotation.CrossOrigin; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import eu.dnetlib.enabling.datasources.common.DsmException; +import eu.dnetlib.enabling.datasources.common.DsmForbiddenException; +import eu.dnetlib.enabling.datasources.common.DsmNotFoundException; +import eu.dnetlib.openaire.common.AbstractExporterController; +import eu.dnetlib.openaire.common.OperationManager; +import eu.dnetlib.openaire.vocabularies.Country; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiResponse; +import io.swagger.annotations.ApiResponses; + +@RestController +@CrossOrigin(origins = { "*" }) +@ConditionalOnProperty(value = "openaire.exporter.enable.dsm", havingValue = "true") +@io.swagger.annotations.Api(tags = "OpenAIRE DSM API", description = "the OpenAIRE Datasource Manager API") +public class DsmApiController extends AbstractExporterController { + + @Autowired + private DsmCore dsmCore; + + @RequestMapping(value = "/ds/countries", produces = { "application/json" }, method = RequestMethod.GET) + @ApiOperation(value = "list the datasource countries", notes = "list the datasource countries", tags = { DS, R }, response = Country[].class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = Country[].class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + public List listCountries() throws DsmException { + return dsmCore.listCountries(); + } + + @Deprecated + @RequestMapping(value = "/ds/search/{page}/{size}", produces = { "application/json" }, method = RequestMethod.POST) + @ApiOperation(value = "search datasources", notes = "Returns list of Datasource details.", tags = { DS, R }, response = DatasourceSearchResponse.class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = DatasourceSearchResponse.class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + public DatasourceSearchResponse search( + @RequestParam final RequestSort requestSortBy, + @RequestParam final RequestSortOrder order, + @RequestBody final RequestFilter requestFilter, + @PathVariable final int page, + @PathVariable final int size) throws DsmException { + final StopWatch stop = StopWatch.createStarted(); + final DatasourceSearchResponse rsp = dsmCore.search(requestSortBy, order, requestFilter, page, size); + return prepareResponse(page, size, stop, rsp); + } + + @RequestMapping(value = "/ds/searchdetails/{page}/{size}", produces = { "application/json" }, method = RequestMethod.POST) + @ApiOperation(value = "search datasources", notes = "Returns list of Datasource details.", tags = { DS, R }, response = DatasourceDetailResponse.class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = DatasourceDetailResponse.class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + public DatasourceDetailResponse searchDsDetails( + @RequestParam final RequestSort requestSortBy, + @RequestParam final RequestSortOrder order, + @RequestBody final RequestFilter requestFilter, + @PathVariable final int page, + @PathVariable final int size) throws DsmException { + final StopWatch stop = StopWatch.createStarted(); + final DatasourceDetailResponse rsp = dsmCore.searchDsDetails(requestSortBy, order, requestFilter, page, size); + return prepareResponse(page, size, stop, rsp); + } + + @RequestMapping(value = "/ds/aggregationhistory/{dsId}", produces = { "application/json" }, method = RequestMethod.GET) + @ApiOperation(value = "search datasources", notes = "Returns list of Datasource details.", tags = { DS, R }, response = AggregationHistoryResponse.class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = AggregationHistoryResponse.class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + public AggregationHistoryResponse aggregationHistory(@PathVariable final String dsId) throws DsmException { + final StopWatch stop = StopWatch.createStarted(); + final AggregationHistoryResponse rsp = dsmCore.aggregationhistory(dsId); + return prepareResponse(0, rsp.getAggregationInfo().size(), stop, rsp); + } + + @RequestMapping(value = "/ds/searchsnippet/{page}/{size}", produces = { "application/json" }, method = RequestMethod.POST) + @ApiOperation(value = "search datasources", notes = "Returns list of Datasource basic info.", tags = { DS, R }, response = DatasourceSnippetResponse.class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = DatasourceSnippetResponse.class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + public DatasourceSnippetResponse searchSnippet( + @RequestParam final RequestSort requestSortBy, + @RequestParam final RequestSortOrder order, + @RequestBody final RequestFilter requestFilter, + @PathVariable final int page, + @PathVariable final int size) throws DsmException { + final StopWatch stop = StopWatch.createStarted(); + final DatasourceSnippetResponse rsp = dsmCore.searchSnippet(requestSortBy, order, requestFilter, page, size); + return prepareResponse(page, size, stop, rsp); + } + + @RequestMapping(value = "/ds/searchregistered/{page}/{size}", produces = { "application/json" }, method = RequestMethod.POST) + @ApiOperation(value = "search among registered datasources", notes = "Returns list of Datasource basic info.", tags = { DS, + R }, response = DatasourceSnippetResponse.class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = DatasourceSnippetResponse.class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + public DatasourceSnippetResponse searchRegistered( + @RequestParam final RequestSort requestSortBy, + @RequestParam final RequestSortOrder order, + @RequestBody final RequestFilter requestFilter, + @PathVariable final int page, + @PathVariable final int size) throws DsmException { + final StopWatch stop = StopWatch.createStarted(); + final DatasourceSnippetResponse rsp = dsmCore.searchRegistered(requestSortBy, order, requestFilter, page, size); + return prepareResponse(page, size, stop, rsp); + } + + @RequestMapping(value = "/ds/recentregistered/{size}", produces = { "application/json" }, method = RequestMethod.GET) + @ApiOperation(value = "return the latest datasources that were registered through Provide", notes = "Returns list of Datasource basic info.", tags = { DS, + R }, response = SimpleResponse.class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = DatasourceResponse.class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + public SimpleResponse recentRegistered(@PathVariable final int size) throws Throwable { + final StopWatch stop = StopWatch.createStarted(); + final SimpleResponse rsp = dsmCore.searchRecentRegistered(size); + return prepareResponse(1, size, stop, rsp); + } + + @RequestMapping(value = "/ds/countregistered", produces = { "application/json" }, method = RequestMethod.GET) + @ApiOperation(value = "return the number of datasources registered after the given date", notes = "Returns a number.", tags = { DS, + R }, response = Long.class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = Long.class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + public Long countRegistered(@RequestParam final String fromDate, + @RequestParam(required = false) final String typologyFilter) throws Throwable { + return dsmCore.countRegisteredAfter(fromDate, typologyFilter); + } + + @RequestMapping(value = "/ds/api/{dsId}", produces = { "application/json" }, method = RequestMethod.GET) + @ApiOperation(value = "get the list of API for a given datasource", notes = "Returns the list of API for a given datasource.", tags = { API, + R }, response = ApiDetailsResponse.class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = ApiDetailsResponse.class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + public ApiDetailsResponse getApi( + @PathVariable final String dsId) throws DsmException { + + final StopWatch stop = StopWatch.createStarted(); + final ApiDetailsResponse rsp = dsmCore.getApis(dsId); + return prepareResponse(0, rsp.getApi().size(), stop, rsp); + } + + @RequestMapping(value = "/api/baseurl/{page}/{size}", produces = { "application/json" }, method = RequestMethod.POST) + @ApiOperation(value = "search for the list of base URLs of Datasource APIs managed by a user", notes = "Returns the list of base URLs of Datasource APIs managed by a user", tags = { + DS, API, R }, response = String[].class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = String[].class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + public List searchBaseUrls( + @RequestBody final RequestFilter requestFilter, + @PathVariable final int page, + @PathVariable final int size) throws DsmException { + + return dsmCore.findBaseURLs(requestFilter, page, size); + } + + @RequestMapping(value = "/ds/api/{apiId}", method = RequestMethod.DELETE) + @ApiOperation(value = "delete an API", notes = "delete an API, if removable", tags = { API, W }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 400, message = "Api not found", response = ErrorMessage.class), + @ApiResponse(code = 403, message = "Api not removable", response = ErrorMessage.class), + @ApiResponse(code = 500, message = "DSM Server error", response = ErrorMessage.class)}) + public void deleteApi(@PathVariable final String apiId) throws DsmForbiddenException, DsmNotFoundException { + dsmCore.deleteApi(apiId); + } + + @RequestMapping(value = "/ds/manage", method = RequestMethod.POST) + @ApiOperation(value = "set the managed status for a given datasource", notes = "set the managed status for a given datasource", tags = { DS, W }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + public void setManaged( + @RequestParam final String id, + @RequestParam final boolean managed) throws DsmException { + + dsmCore.setManaged(id, managed); + } + + @RequestMapping(value = "/ds/managed/{id}", method = RequestMethod.GET) + @ApiOperation(value = "get the datasource managed status", notes = "get the datasource managed status", tags = { DS, R }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + public boolean isManaged(@PathVariable final String id) throws DsmException { + return dsmCore.isManaged(id); + } + + @RequestMapping(value = "/ds/add", method = RequestMethod.POST) + @ApiOperation(value = "add a new Datasource", notes = "add a new Datasource", tags = { DS, W }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 400, message = "Malformed request", response = ErrorMessage[].class), + @ApiResponse(code = 500, message = "Unexpected error", response = ErrorMessage.class) }) + public void saveDs(@Valid @RequestBody final DatasourceDetails datasource) throws DsmException { + + if (dsmCore.exist(datasource)) { // TODO further check that the DS doesn't have any API + throw new DsmException(HttpStatus.SC_CONFLICT, String.format("cannot register, datasource already defined '%s'", datasource.getId())); + } + dsmCore.save(datasource); + } + + @RequestMapping(value = "/ds/update", method = RequestMethod.POST) + @ApiOperation(value = "update Datasource details", notes = "update Datasource details", tags = { DS, W }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + public void updateDatasource( + @RequestBody final DatasourceDetailsUpdate ds) throws DsmException, DsmNotFoundException { + + dsmCore.updateDatasource(ds); + } + + @Deprecated + @RequestMapping(value = "/ds/logourl", method = RequestMethod.POST) + @ApiOperation(value = "update a datasource name", notes = "update a datasource name", tags = { DS, W, D }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + public void updateDatasourceLogoURL( + @RequestParam final String dsId, + @RequestParam final String logourl) throws DsmException { + + dsmCore.updateDatasourceLogoUrl(dsId, logourl); + } + + @Deprecated + @RequestMapping(value = "/ds/name", method = RequestMethod.POST) + @ApiOperation(value = "update a datasource name", notes = "update a datasource name", tags = { DS, W, D }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + public void updateDatasourceName( + @RequestParam final String dsId, + @RequestParam final String officialname, + @RequestParam final String englishname) throws DsmException { + + dsmCore.updateDatasourcename(dsId, officialname, englishname); + } + + @Deprecated + @RequestMapping(value = "/ds/coordinates", method = RequestMethod.POST) + @ApiOperation(value = "update the datasource coordinates (latitude, longitude)", notes = "update the datasource coordinates (latitude, longitude)", tags = { + DS, W, D }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + public void updateCoordinates( + @RequestParam final String dsId, + @RequestParam final Double latitude, + @RequestParam final Double longitude) throws DsmException { + + dsmCore.updateCoordinates(dsId, latitude, longitude); + } + + @Deprecated + @RequestMapping(value = "/ds/timezone", method = RequestMethod.POST) + @ApiOperation(value = "update a datasource timezone", notes = "update a datasource timezone", tags = { DS, W, D }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + public void updateTimezone( + @RequestParam final String dsId, + @RequestParam final String timezone) throws DsmException { + + dsmCore.updateTimezone(dsId, timezone); + } + + @Deprecated + @RequestMapping(value = "/ds/typology", method = RequestMethod.POST) + @ApiOperation(value = "update a datasource typology code", notes = "update a datasource typology code", tags = { DS, W, D }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + public void updateTypology( + @RequestParam final String dsId, + @RequestParam final String typology) throws DsmException { + + dsmCore.updateDsTypology(dsId, typology); + } + + @Deprecated + @RequestMapping(value = "/ds/registeredby", method = RequestMethod.POST) + @ApiOperation(value = "update a datasource registeredBy", notes = "update a datasource registeredBy", tags = { DS, W, D }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + public void updateRegisteringUser( + @RequestParam final String dsId, + @RequestParam final String registeredBy) throws DsmException { + + dsmCore.updateDsRegisteringUser(dsId, registeredBy); + } + + @Deprecated + @RequestMapping(value = "/ds/platform", method = RequestMethod.POST) + @ApiOperation(value = "update a datasource platform", notes = "update a datasource platform", tags = { DS, W, D }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + public void updatePlatform( + @RequestParam final String dsId, + @RequestParam final String platform) throws DsmException { + + dsmCore.updateDsPlatform(dsId, platform); + } + + @RequestMapping(value = "/ds/api/baseurl", method = RequestMethod.POST) + @ApiOperation(value = "update the base URL of a datasource interface", notes = "update the base URL of a datasource interface", tags = { API, W }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + public void updateBaseUrl( + @RequestParam final String dsId, + @RequestParam final String apiId, + @RequestParam final String baseUrl) throws DsmException { + + dsmCore.updateApiBaseurl(dsId, apiId, baseUrl); + } + + @RequestMapping(value = "/ds/api/compliance", method = RequestMethod.POST) + @ApiOperation(value = "update the compatibility of a datasource interface", notes = "update the compatibility of a datasource interface", tags = { API, W }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + public void updateCompliance( + @RequestParam final String dsId, + @RequestParam final String apiId, + @RequestParam final String compliance, + @RequestParam(required = false, defaultValue = "false") final boolean override) throws DsmException { + + dsmCore.updateApiCompatibility(dsId, apiId, compliance, override); + } + + @RequestMapping(value = "/ds/api/oaiset", method = RequestMethod.POST) + @ApiOperation(value = "update the OAI set of a datasource interface", notes = "update the OAI set of a datasource interface", tags = { API, W }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + public void updateOaiSetl( + @RequestParam final String dsId, + @RequestParam final String apiId, + @RequestParam final String oaiSet) throws DsmException { + + dsmCore.updateApiOaiSet(dsId, apiId, oaiSet); + } + + @RequestMapping(value = "/ds/api/add", method = RequestMethod.POST) + @ApiOperation(value = "adds a new Interface to one Datasource", notes = "adds an Interface to one Datasource", tags = { API, W }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + public void addApi(@RequestBody final ApiDetails api) throws DsmException { + if (StringUtils.isBlank(api.getDatasource())) { throw new DsmException(HttpStatus.SC_BAD_REQUEST, "missing datasource id"); } + dsmCore.addApi(api); + } + + // MANAGEMENT + + @Autowired + private OperationManager operationManager; + + @RequestMapping(value = "/dsm/ops", method = RequestMethod.GET) + @ApiOperation(value = "get the number of pending operations", notes = "get the number of pending operations", tags = { R, M }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + public int getOps() throws DsmException { + return operationManager.getOpSize(); + } + + @RequestMapping(value = "/dsm/killops", method = RequestMethod.POST) + @ApiOperation(value = "interrupts the pending operations", notes = "return the number of interrupted operations", tags = { W, M }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + public int killOps() throws DsmException { + return operationManager.dropAll(); + } + + @RequestMapping(value = "/dsm/dropcache", method = RequestMethod.POST) + @ApiOperation(value = "drop the caches", notes = "drop the internal caches", tags = { W, M }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + public void dropCache() throws DsmException { + dsmCore.dropCaches(); + } + + // OLD DEPRECATED METHODS + + @Deprecated + @ApiOperation(value = "search datasources by name", notes = "Returns list of Datasource details.", tags = { D }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = DatasourceSearchResponse.class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + @RequestMapping(value = "/ds/search/name/{page}/{size}", produces = { "application/json" }, method = RequestMethod.GET) + DatasourceSearchResponse searchByName(final String name, final int page, final int size) throws DsmException { + final RequestSort sort = RequestSort.id; + final RequestSortOrder order = RequestSortOrder.ASCENDING; + final RequestFilter filter = new RequestFilter(); + filter.put(FilterName.englishname, name); + + final StopWatch stop = StopWatch.createStarted(); + final DatasourceSearchResponse rsp = dsmCore.search(sort, order, filter, page, size); + return prepareResponse(page, size, stop, rsp); + } + + @Deprecated + @ApiOperation(value = "search datasources by contact email", notes = "Returns list of Datasource details.", tags = { D }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = DatasourceSearchResponse.class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + @RequestMapping(value = "/ds/search/email/{page}/{size}", produces = { "application/json" }, method = RequestMethod.GET) + DatasourceSearchResponse searchByContactemail(final String contactemail, final int page, final int size) throws DsmException { + final RequestSort sort = RequestSort.id; + final RequestSortOrder order = RequestSortOrder.ASCENDING; + final RequestFilter filter = new RequestFilter(); + filter.put(FilterName.contactemail, contactemail); + + final StopWatch stop = StopWatch.createStarted(); + final DatasourceSearchResponse rsp = dsmCore.search(sort, order, filter, page, size); + return prepareResponse(page, size, stop, rsp); + } + + @Deprecated + @ApiOperation(value = "search datasources by country", notes = "Returns list of Datasource details.", tags = { D }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = DatasourceSearchResponse.class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + @RequestMapping(value = "/ds/search/country/{page}/{size}", produces = { "application/json" }, method = RequestMethod.GET) + DatasourceSearchResponse searchByCountry(final String country, final Boolean managed, final int page, final int size) throws DsmException { + final RequestSort sort = RequestSort.id; + final RequestSortOrder order = RequestSortOrder.ASCENDING; + final RequestFilter filter = new RequestFilter(); + filter.put(FilterName.country, country); + filter.put(FilterName.managed, managed); + + final StopWatch stop = StopWatch.createStarted(); + final DatasourceSearchResponse rsp = dsmCore.search(sort, order, filter, page, size); + return prepareResponse(page, size, stop, rsp); + } + + @Deprecated + @ApiOperation(value = "search datasources by registering user", notes = "Returns list of Datasource details.", tags = { D }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = DatasourceSearchResponse.class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + @RequestMapping(value = "/ds/search/registeredby/{page}/{size}", produces = { "application/json" }, method = RequestMethod.GET) + DatasourceSearchResponse searchByRegisteringUser(final String registeredBy, final int page, final int size) throws DsmException { + final RequestSort sort = RequestSort.id; + final RequestSortOrder order = RequestSortOrder.ASCENDING; + final RequestFilter filter = new RequestFilter(); + filter.put(FilterName.registeredby, registeredBy); + + final StopWatch stop = StopWatch.createStarted(); + final DatasourceSearchResponse rsp = dsmCore.search(sort, order, filter, page, size); + return prepareResponse(page, size, stop, rsp); + } + + // HELPERS + + private T prepareResponse(final int page, final int size, final StopWatch stopWatch, final T rsp) { + rsp.getHeader() + .setTime(stopWatch.getTime()) + .setPage(page) + .setSize(size); + return rsp; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmCore.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmCore.java new file mode 100644 index 00000000..8e9139fd --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmCore.java @@ -0,0 +1,579 @@ +package eu.dnetlib.openaire.dsm; + +import static eu.dnetlib.openaire.common.ExporterConstants.BASE_URL; +import static eu.dnetlib.openaire.common.ExporterConstants.COMPLIANCE; +import static eu.dnetlib.openaire.common.ExporterConstants.ENGLISH_NAME; +import static eu.dnetlib.openaire.common.ExporterConstants.LATITUDE; +import static eu.dnetlib.openaire.common.ExporterConstants.LONGITUDE; +import static eu.dnetlib.openaire.common.ExporterConstants.OAI_SET; +import static eu.dnetlib.openaire.common.ExporterConstants.OFFICIAL_NAME; +import static eu.dnetlib.openaire.common.ExporterConstants.PLATFORM; +import static eu.dnetlib.openaire.common.ExporterConstants.REMOVABLE; +import static eu.dnetlib.openaire.common.ExporterConstants.TIMEZONE; +import static eu.dnetlib.openaire.common.ExporterConstants.TYPOLOGY; +import static eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils.asDbEntry; +import static eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils.asDetails; +import static eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils.asMapOfChanges; +import static eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils.copyNonNullProperties; +import static eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils.createId; + +import java.nio.charset.Charset; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import javax.annotation.PostConstruct; + +import eu.dnetlib.openaire.dsm.domain.*; +import eu.dnetlib.openaire.dsm.domain.ApiDetailsResponse; +import eu.dnetlib.openaire.dsm.domain.SimpleResponse; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.domain.Page; +import org.springframework.jdbc.core.BeanPropertyRowMapper; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Component; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Queues; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import com.google.common.xml.XmlEscapers; + +import eu.dnetlib.OpenaireExporterConfig; +import eu.dnetlib.enabling.datasources.common.AggregationInfo; +import eu.dnetlib.enabling.datasources.common.AggregationStage; +import eu.dnetlib.enabling.datasources.common.Datasource; +import eu.dnetlib.enabling.datasources.common.DsmException; +import eu.dnetlib.enabling.datasources.common.DsmForbiddenException; +import eu.dnetlib.enabling.datasources.common.DsmNotFoundException; +import eu.dnetlib.openaire.common.ISClient; +import eu.dnetlib.openaire.community.CommunityClient; +import eu.dnetlib.openaire.dsm.dao.DatasourceDao; +import eu.dnetlib.openaire.dsm.dao.DatasourceIndexClient; +import eu.dnetlib.openaire.dsm.dao.MongoLoggerClient; +import eu.dnetlib.openaire.dsm.dao.ObjectStoreClient; +import eu.dnetlib.openaire.dsm.dao.ResponseUtils; +import eu.dnetlib.openaire.dsm.dao.VocabularyClient; +import eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils; +import eu.dnetlib.openaire.dsm.dao.utils.IndexDsInfo; +import eu.dnetlib.openaire.dsm.dao.utils.IndexRecordsInfo; +import eu.dnetlib.openaire.dsm.domain.db.ApiDbEntry; +import eu.dnetlib.openaire.dsm.domain.db.DatasourceDbEntry; +import eu.dnetlib.openaire.dsm.domain.db.IdentityDbEntry; +import eu.dnetlib.openaire.vocabularies.Country; + +@Component +@ConditionalOnProperty(value = "openaire.exporter.enable.dsm", havingValue = "true") +public class DsmCore { + + private static final Log log = LogFactory.getLog(DsmCore.class); + + @Autowired + private MongoLoggerClient mongoLoggerClient; + + @Autowired + private ISClient isClient; + + @Autowired + private ObjectStoreClient objectStoreClient; + + @Autowired + private DatasourceIndexClient datasourceIndexClient; + + @Autowired + private VocabularyClient vocabularyClient; + + @Autowired + private DatasourceDao dsDao; + + @Autowired + private OpenaireExporterConfig config; + + @Autowired + private JdbcTemplate jdbcTemplate; + + @Autowired + private CommunityClient communityClient; + + private ListeningExecutorService executor; + + @PostConstruct + public void init() { + executor = MoreExecutors.listeningDecorator(new ScheduledThreadPoolExecutor(config.getRequestWorkers(), + new ThreadFactoryBuilder().setNameFormat("dsm-client-%d").build())); + } + + public List listCountries() throws DsmException { + try { + return dsDao.listCountries(); + } catch (final Throwable e) { + log.error("error listing countries", e); + throw e; + } + } + + @Deprecated + public DatasourceSearchResponse search(final RequestSort requestSortBy, + final RequestSortOrder order, + final RequestFilter requestFilter, + final int page, + final int size) + throws DsmException { + + try { + final List datasourceInfo = Lists.newArrayList(); + final Queue errors = Queues.newLinkedBlockingQueue(); + final CountDownLatch outerLatch = new CountDownLatch(2); + + final Page dsPage = dsDao.search(requestSortBy, order, requestFilter, page, size); + if (dsPage.getTotalElements() > 0 && dsPage.getNumberOfElements() > 0) { + dsPage.forEach(d -> datasourceInfo.add(enrichDatasourceInfo(asDetails(d), outerLatch, errors))); + waitLatch(outerLatch, errors, config.getRequestTimeout()); + } + + if (!errors.isEmpty()) { + // TODO report on error metrics + errors.forEach(log::error); + } + return ResponseUtils.searchResponse(datasourceInfo, dsPage.getTotalElements()); + } catch (final Throwable e) { + log.error("error searching datasources", e); + throw e; + } + } + + public DatasourceDetailResponse searchDsDetails(final RequestSort requestSortBy, + final RequestSortOrder order, + final RequestFilter requestFilter, + final int page, + final int size) + throws DsmException { + + try { + final Page dsPage = dsDao.search(requestSortBy, order, requestFilter, page, size); + return ResponseUtils.detailsResponse( + dsPage.map(d -> asDetails(d)).getContent(), + dsPage.getTotalElements()); + } catch (final Throwable e) { + log.error("error searching datasources", e); + throw e; + } + } + + public DatasourceSnippetResponse searchSnippet(final RequestSort requestSortBy, + final RequestSortOrder order, + final RequestFilter requestFilter, + final int page, + final int size) + throws DsmException { + try { + final Page dsPage = dsDao.search(requestSortBy, order, requestFilter, page, size); + return ResponseUtils.snippetResponse( + dsPage.map(DsmMappingUtils::asSnippetExtended).getContent(), + dsPage.getTotalElements()); + } catch (final Throwable e) { + log.error("error searching datasources", e); + throw e; + } + } + + public DatasourceSnippetResponse searchRegistered(final RequestSort requestSortBy, + final RequestSortOrder order, + final RequestFilter requestFilter, + final int page, + final int size) + throws DsmException { + try { + final Page dsPage = dsDao.searchRegistered(requestSortBy, order, requestFilter, page, size); + return ResponseUtils.snippetResponse( + dsPage.map(DsmMappingUtils::asSnippetExtended).getContent(), + dsPage.getTotalElements()); + } catch (final Throwable e) { + log.error("error searching datasources", e); + throw e; + } + } + + public List findBaseURLs(final RequestFilter requestFilter, final int page, final int size) throws DsmException { + try { + return dsDao.findApiBaseURLs(requestFilter, page, size); + } catch (final Throwable e) { + log.error("error searching datasource base urls", e); + throw e; + } + } + + public ApiDetailsResponse getApis(final String dsId) throws DsmException { + try { + final List apis = dsDao.getApis(dsId); + final List api = apis.stream() + .map(DsmMappingUtils::asDetails) + .collect(Collectors.toList()); + return ResponseUtils.apiResponse(api, api.size()); + } catch (final Throwable e) { + log.error(String.format("error searching datasource api %s", dsId), e); + throw e; + } + } + + public void setManaged(final String dsId, final boolean managed) throws DsmException { + log.info(String.format("updated ds '%s' managed with '%s'", dsId, managed)); + dsDao.setManaged(dsId, managed); + final List apis = dsDao.getApis(dsId); + for (final ApiDbEntry a : apis) { + setApiRemovable(dsId, a.getId(), true); + } + } + + protected void setApiRemovable(final String dsId, final String apiId, final boolean removable) { + log.info(String.format("updated api '%s' removable with '%s'", apiId, removable)); + final Map changes = Maps.newHashMap(); + changes.put(REMOVABLE, String.valueOf(removable)); + isClient.updateAPIField(dsId, apiId, changes); + } + + public boolean isManaged(final String dsId) throws DsmException { + return dsDao.isManaged(dsId); + } + + public boolean exist(final DatasourceDetails d) throws DsmException { + return dsDao.existDs(d.getId()); + } + + public void save(final DatasourceDetails d) throws DsmException { + try { + dsDao.saveDs(asDbEntry(d)); + isClient.registerDS(d); + } catch (final Throwable e) { + log.error(ExceptionUtils.getStackTrace(e)); + throw e; + } + } + + public void updateDatasource(final DatasourceDetailsUpdate d) throws DsmException, DsmNotFoundException { + try { + // initialize with current values from DB + final Datasource ds = dsDao.getDs(d.getId()); + final DatasourceDbEntry dbEntry = (DatasourceDbEntry) ds; + + if (dbEntry == null) { throw new DsmNotFoundException(String.format("ds '%s' does not exist", d.getId())); } + + final DatasourceDbEntry update = asDbEntry(d); + if (d.getIdentities() != null) { + final Set identities = new HashSet<>( + Stream.of(update.getIdentities(), dbEntry.getIdentities()) + .flatMap(Collection::stream) + .collect(Collectors.toMap(i -> i.getIssuertype() + i.getPid(), Function.identity(), (i1, i2) -> i1)) + .values()); + copyNonNullProperties(update, dbEntry); + dbEntry.setIdentities(identities); + } else { + copyNonNullProperties(update, dbEntry); + } + + dsDao.saveDs(dbEntry); + isClient.updateDatasourceFields(d.getId(), asMapOfChanges(d)); + } catch (final Throwable e) { + log.error(ExceptionUtils.getStackTrace(e)); + throw e; + } + } + + @Deprecated + public void updateDatasourcename(final String dsId, final String officialname, final String englishname) throws DsmException { + log.info(String.format("updated datasource '%s' with officialname '%s' and englishname '%s'", dsId, officialname, englishname)); + dsDao.updateName(dsId, officialname, englishname); + + final Map changes = Maps.newHashMap(); + changes.put(OFFICIAL_NAME, XmlEscapers.xmlContentEscaper().escape(officialname)); + changes.put(ENGLISH_NAME, XmlEscapers.xmlContentEscaper().escape(englishname)); + isClient.updateDatasourceFields(dsId, changes); + } + + @Deprecated + public void updateDatasourceLogoUrl(final String dsId, final String logourl) throws DsmException { + log.info(String.format("updated datasource '%s' with logo URL '%s'", dsId, logourl)); + + dsDao.updateLogoUrl(dsId, logourl); + } + + @Deprecated + public void updateCoordinates(final String dsId, final Double latitude, final Double longitude) throws DsmException { + log.info(String.format("updated datasource '%s' with coordinates Lat:'%s', Lon:'%s'", dsId, latitude, longitude)); + dsDao.updateCoordinates(dsId, latitude, longitude); + + final Map changes = Maps.newHashMap(); + changes.put(LATITUDE, XmlEscapers.xmlContentEscaper().escape(String.valueOf(latitude))); + changes.put(LONGITUDE, XmlEscapers.xmlContentEscaper().escape(String.valueOf(longitude))); + isClient.updateDatasourceFields(dsId, changes); + } + + @Deprecated + public void updateTimezone(final String dsId, final String timezone) throws DsmException { + log.info(String.format("updated datasource '%s' timezone with '%s'", dsId, timezone)); + dsDao.updateTimezone(dsId, timezone); + + final Map changes = Maps.newHashMap(); + changes.put(TIMEZONE, XmlEscapers.xmlContentEscaper().escape(timezone)); + isClient.updateDatasourceFields(dsId, changes); + } + + @Deprecated + public void updateDsTypology(final String dsId, final String typology) throws DsmException { + log.info(String.format("updated datasource '%s' typology with '%s'", dsId, typology)); + dsDao.updateTypology(dsId, typology); + + final Map changes = Maps.newHashMap(); + changes.put(TYPOLOGY, XmlEscapers.xmlContentEscaper().escape(typology)); + isClient.updateDatasourceFields(dsId, changes); + } + + @Deprecated + public void updateDsRegisteringUser(final String dsId, final String registeredBy) throws DsmException { + log.info(String.format("setting datasource '%s' registering user with '%s'", dsId, registeredBy)); + dsDao.updateRegisteringUser(dsId, registeredBy); + } + + @Deprecated + public void updateDsPlatform(final String dsId, final String platform) throws DsmException { + log.info(String.format("updated datasource '%s' platform with '%s'", dsId, platform)); + dsDao.updatePlatform(dsId, platform); + + final Map changes = Maps.newHashMap(); + changes.put(PLATFORM, XmlEscapers.xmlContentEscaper().escape(platform)); // this is not a typo, Repository profiles map the platform + // in the DATASOURCE_TYPE field. + isClient.updateDatasourceFields(dsId, changes); + } + + // TODO remove if unused + public void deleteDs(final String dsId) throws DsmException { + log.info(String.format("deleted datasource '%s'", dsId)); + dsDao.deleteDs(dsId); + } + + // API + + public void updateApiOaiSet(final String dsId, final String apiId, final String oaiSet) throws DsmException { + final boolean insert = dsDao.upsertApiOaiSet(apiId, oaiSet); + final Map changes = Maps.newHashMap(); + changes.put(OAI_SET, XmlEscapers.xmlContentEscaper().escape(oaiSet)); + + if (!insert) { + isClient.updateAPIField(dsId, apiId, changes); + } else { + isClient.addAPIAttribute(dsId, apiId, changes); + } + } + + public void updateApiBaseurl(final String dsId, final String apiId, final String baseUrl) throws DsmException { + log.info(String.format("updated api '%s' baseurl with '%s'", apiId, baseUrl)); + dsDao.updateApiBaseUrl(apiId, baseUrl); + + final Map changes = Maps.newHashMap(); + changes.put(BASE_URL, XmlEscapers.xmlContentEscaper().escape(baseUrl)); + + isClient.updateAPIField(dsId, apiId, changes); + } + + public void updateApiCompatibility(final String dsId, final String apiId, final String compliance, final boolean override) throws DsmException { + log.info(String.format("updated api '%s' compliance with '%s'", apiId, compliance)); + dsDao.updateCompliance(null, apiId, compliance, override); + + final Map changes = Maps.newHashMap(); + changes.put(COMPLIANCE, XmlEscapers.xmlAttributeEscaper().escape(compliance)); + + isClient.updateAPIField(dsId, apiId, changes); + } + + public void addApi(final ApiDetails api) throws DsmException { + if (StringUtils.isBlank(api.getId())) { + api.setId(createId(api)); + log.info(String.format("missing api id, created '%s'", api.getId())); + } + + dsDao.addApi(asDbEntry(api)); + isClient.registerAPI(api); + } + + public void deleteApi(final String apiId) throws DsmForbiddenException, DsmNotFoundException { + // TODO handle the api removal in case of associated workflows. + isClient.removeAPI(apiId); + dsDao.deleteApi(null, apiId); + } + + public void dropCaches() { + mongoLoggerClient.dropCache(); + isClient.dropCache(); + vocabularyClient.dropCache(); + communityClient.dropCache(); + } + + // HELPERS ////////////// + + private DatasourceInfo enrichDatasourceInfo(final DatasourceDetails d, final CountDownLatch outerLatch, final Queue errors) { + final DatasourceInfo dsInfo = new DatasourceInfo().setDatasource(d); + getAggregationHistory(d.getId(), outerLatch, errors, dsInfo); + getIndexDsInfo(d.getId(), outerLatch, errors, dsInfo); + return dsInfo; + } + + private void getAggregationHistory(final String dsId, + final CountDownLatch outerLatch, + final Queue errors, + final DatasourceInfo datasourceInfo) { + Futures.addCallback(executor.submit(() -> mongoLoggerClient.getAggregationHistory(dsId)), new FutureCallback>() { + + @Override + public void onSuccess(final List info) { + setAggregationHistory(datasourceInfo, info); + outerLatch.countDown(); + } + + @Override + public void onFailure(final Throwable e) { + log.error(ExceptionUtils.getStackTrace(e)); + errors.offer(e); + outerLatch.countDown(); + } + }, executor); + } + + private void setAggregationHistory(final DatasourceInfo datasourceInfo, final List info) { + datasourceInfo.setAggregationHistory(info); + if (!info.isEmpty()) { + datasourceInfo + .setLastCollection(info.stream().filter(a -> AggregationStage.COLLECT.equals(a.getAggregationStage())).findFirst().get()) + .setLastTransformation(info.stream().filter(a -> AggregationStage.TRANSFORM.equals(a.getAggregationStage())).findFirst().get()); + } + } + + private void getIndexDsInfo(final String dsId, + final CountDownLatch outerLatch, + final Queue errors, + final DatasourceInfo datasourceInfo) { + Futures.addCallback(executor.submit(() -> isClient.calculateCurrentIndexDsInfo()), new FutureCallback() { + + @Override + public void onSuccess(final IndexDsInfo info) { + + final CountDownLatch innerLatch = new CountDownLatch(2); + + Futures.addCallback(executor.submit(() -> datasourceIndexClient.getIndexInfo(dsId, info, errors)), new FutureCallback() { + + @Override + public void onSuccess(final IndexRecordsInfo info) { + datasourceInfo + .setIndexRecords(info.getTotal()) + .setFundedContent(info.getFunded()) + .setLastIndexingDate(info.getDate()); + innerLatch.countDown(); + } + + @Override + public void onFailure(final Throwable e) { + errors.offer(e); + innerLatch.countDown(); + } + }, executor); + + Futures.addCallback(executor.submit(() -> objectStoreClient.getObjectStoreSize(isClient.getObjectStoreId(dsId))), new FutureCallback() { + + @Override + public void onSuccess(final Long objectStoreSize) { + datasourceInfo.setFulltexts(objectStoreSize); + innerLatch.countDown(); + } + + @Override + public void onFailure(final Throwable e) { + errors.offer(e); + innerLatch.countDown(); + } + }, executor); + + waitLatch(innerLatch, errors, config.getRequestTimeout()); + + outerLatch.countDown(); + } + + @Override + public void onFailure(final Throwable e) { + // log.error(ExceptionUtils.getStackTrace(e)); + errors.offer(e); + outerLatch.countDown(); + } + }, executor); + } + + private void waitLatch(final CountDownLatch latch, final Queue errors, final int waitSeconds) { + try { + if (!latch.await(waitSeconds, TimeUnit.SECONDS)) { + errors.offer(new TimeoutException("Waiting for requests to complete has timed out.")); + } + } catch (final InterruptedException e) { + errors.offer(e); + } + } + + public SimpleResponse searchRecentRegistered(final int size) throws Throwable { + try { + final String sql = + IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/recent_registered_datasources.sql.st"), Charset.defaultCharset()); + + final List list = jdbcTemplate.query(sql, BeanPropertyRowMapper.newInstance(RegisteredDatasourceInfo.class), size); + + return ResponseUtils.simpleResponse(list); + } catch (final Throwable e) { + log.error("error searching recent datasources", e); + throw e; + } + } + public Long countRegisteredAfter(final String fromDate, final String typologyFilter) throws Throwable { + try { + if (StringUtils.isNotBlank(typologyFilter)) { + final String sql = + IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate_typology.st.sql"), Charset.defaultCharset()); + + return jdbcTemplate.queryForObject(sql, new Object[] { fromDate, typologyFilter + "%" }, Long.class); + } else { + final String sql = + IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate.st.sql"), Charset.defaultCharset()); + + return jdbcTemplate.queryForObject(sql, new Object[] { fromDate }, Long.class); + } + + + } catch (final Throwable e) { + log.error("error searching recent datasources", e); + throw e; + } + } + + public AggregationHistoryResponse aggregationhistory(String dsId) throws DsmException { + final List history = mongoLoggerClient.getAggregationHistory(dsId); + final AggregationHistoryResponse rsp = new AggregationHistoryResponse(history); + rsp.setHeader(ResponseUtils.header(history.size())); + return rsp; + + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ApiDbEntryRepository.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ApiDbEntryRepository.java new file mode 100644 index 00000000..044ed440 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ApiDbEntryRepository.java @@ -0,0 +1,53 @@ +package eu.dnetlib.openaire.dsm.dao; + +import java.util.List; +import javax.transaction.Transactional; + +import eu.dnetlib.openaire.dsm.domain.db.ApiDbEntry; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +import org.springframework.stereotype.Repository; + +/** + * Created by claudio on 15/06/2017. + */ +@Repository +@ConditionalOnProperty(value = "openaire.exporter.enable.dsm", havingValue = "true") +public interface ApiDbEntryRepository extends JpaRepository { + + @Query("select a from #{#entityName} a where a.datasource = ?1") + List findByDatasource(String dsId); + + @Modifying + @Transactional + @Query("update #{#entityName} a set a.baseurl = ?2 where a.id = ?1") + void setBaseurl(String id, String baseurl); + + @Modifying + @Transactional + @Query("update #{#entityName} a set a.compatibility = ?2 where a.id = ?1") + void updateCompatibility(String apiId, String compatibility); + + @Modifying + @Transactional + @Query("update #{#entityName} a set a.compatibilityOverride = ?2 where a.id = ?1") + void updateCompatibilityOverride(String apiId, String compatibility); + + @Modifying + @Transactional + @Query(value = "update dsm_apiparams ap set value = ?2 where ap.param = 'set' and ap.api = ?1", nativeQuery = true) + void updateOaiSet(String apiId, String oaiSet); + + @Modifying + @Transactional + @Query(value = "insert into dsm_apiparams(api, param, value, _dnet_resource_identifier_) values(?1, ?2, ?3, ?1||'@@'||?2)", nativeQuery = true) + void addApiParam(String apiId, String param, String value); + + @Modifying + @Transactional + @Query("update #{#entityName} d set d.removable = ?2 where d.datasource = ?1") + void setRemovable(String id, boolean removable); + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/CountryTermRepository.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/CountryTermRepository.java new file mode 100644 index 00000000..221810f1 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/CountryTermRepository.java @@ -0,0 +1,15 @@ +package eu.dnetlib.openaire.dsm.dao; + +import eu.dnetlib.openaire.dsm.domain.db.CountryTerm; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +/** + * Created by claudio on 19/04/2017. + */ +@Repository +@ConditionalOnProperty(value = "openaire.exporter.enable.dsm", havingValue = "true") +public interface CountryTermRepository extends JpaRepository { + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceApiDbEntryRepository.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceApiDbEntryRepository.java new file mode 100644 index 00000000..af13e141 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceApiDbEntryRepository.java @@ -0,0 +1,14 @@ +package eu.dnetlib.openaire.dsm.dao; + +import eu.dnetlib.openaire.dsm.domain.db.DatasourceApiDbEntry; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.JpaSpecificationExecutor; +import org.springframework.stereotype.Repository; + +@Repository +@ConditionalOnProperty(value = "openaire.exporter.enable.dsm", havingValue = "true") +public interface DatasourceApiDbEntryRepository extends JpaRepository, JpaSpecificationExecutor { + + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDao.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDao.java new file mode 100644 index 00000000..4fb71855 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDao.java @@ -0,0 +1,58 @@ +package eu.dnetlib.openaire.dsm.dao; + +import java.util.List; + +import eu.dnetlib.enabling.datasources.common.Api; +import eu.dnetlib.enabling.datasources.common.Datasource; +import eu.dnetlib.enabling.datasources.common.DatasourceManagerCommon; +import eu.dnetlib.enabling.datasources.common.DsmException; +import eu.dnetlib.openaire.dsm.domain.RequestFilter; +import eu.dnetlib.openaire.dsm.domain.RequestSort; +import eu.dnetlib.openaire.dsm.domain.RequestSortOrder; +import eu.dnetlib.openaire.vocabularies.Country; +import org.springframework.data.domain.Page; + +public interface DatasourceDao, API extends Api> extends DatasourceManagerCommon { + + // DATASOURCE + + List listCountries() throws DsmException; + + boolean existDs(final String dsId) throws DsmException; + + Page search(RequestSort requestSortBy, RequestSortOrder order, RequestFilter requestFilter, int page, int size) throws DsmException; + + Page searchRegistered(RequestSort requestSortBy, RequestSortOrder order, RequestFilter requestFilter, int page, int size) throws DsmException; + + void updateName(String dsId, String officialname, String englishname) throws DsmException; + + void updateLogoUrl(String dsId, String logourl) throws DsmException; + + void updateCoordinates(String dsId, Double latitude, Double longitude) throws DsmException; + + void updateTimezone(String dsId, String timezone) throws DsmException; + + void updateTypology(String dsId, String timezone) throws DsmException; + + void updateRegisteringUser(String dsId, String registeredBy) throws DsmException; + + void updatePlatform(String dsId, String platform) throws DsmException; + + // API + + List findApiBaseURLs(RequestFilter requestFilter, int page, int size) throws DsmException; + + /** + * Insert the oai set in case it does not exists, updates it otherwise + * + * @param apiId + * @param oaiSet + * @return true in case of insert, false in case of update + * @throws DsmException + */ + boolean upsertApiOaiSet(String apiId, String oaiSet) throws DsmException; + + void updateApiBaseUrl(String apiId, String baseUrl) throws DsmException; + + void addApi(final API api) throws DsmException; +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDaoImpl.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDaoImpl.java new file mode 100644 index 00000000..cb53f505 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDaoImpl.java @@ -0,0 +1,262 @@ +package eu.dnetlib.openaire.dsm.dao; + +import com.google.common.collect.Lists; +import eu.dnetlib.OpenaireExporterConfig; +import eu.dnetlib.enabling.datasources.common.DsmException; +import eu.dnetlib.enabling.datasources.common.DsmForbiddenException; +import eu.dnetlib.enabling.datasources.common.DsmNotFoundException; +import eu.dnetlib.openaire.dsm.domain.RequestFilter; +import eu.dnetlib.openaire.dsm.domain.RequestSort; +import eu.dnetlib.openaire.dsm.domain.RequestSortOrder; +import eu.dnetlib.openaire.dsm.domain.db.ApiDbEntry; +import eu.dnetlib.openaire.dsm.domain.db.ApiParamDbEntry; +import eu.dnetlib.openaire.dsm.domain.db.DatasourceApiDbEntry; +import eu.dnetlib.openaire.dsm.domain.db.DatasourceDbEntry; +import eu.dnetlib.openaire.vocabularies.Country; +import eu.dnetlib.openaire.vocabularies.Vocabulary; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.http.HttpStatus; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.jpa.domain.Specification; +import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; + +import javax.persistence.EntityNotFoundException; +import java.sql.Date; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +import static eu.dnetlib.openaire.common.ExporterConstants.OAI; +import static eu.dnetlib.openaire.common.ExporterConstants.SET; +import static eu.dnetlib.openaire.dsm.dao.DatasourceSpecs.apiSpec; +import static eu.dnetlib.openaire.dsm.dao.DatasourceSpecs.dsSpec; +import static eu.dnetlib.openaire.dsm.dao.DatasourceSpecs.dsRegisteredbyNotNullSpec; + + +/** + * Created by claudio on 20/10/2016. + */ +@Component +@ConditionalOnProperty(value = "openaire.exporter.enable.dsm", havingValue = "true") +public class DatasourceDaoImpl implements DatasourceDao { + + private static final Log log = LogFactory.getLog(DatasourceDao.class); + + @Autowired + private OpenaireExporterConfig config; + + @Autowired + private CountryTermRepository countryTermRepository; + + @Autowired + private DatasourceDbEntryRepository dsRepository; + + @Autowired + private ApiDbEntryRepository apiRepository; + + @Autowired + private DatasourceApiDbEntryRepository dsApiRepository; + + @Autowired + private VocabularyClient vocabularyClient; + + @Override + public List listCountries() throws DsmException { + final List countries = Lists.newArrayList(); + final Vocabulary v = vocabularyClient.getCountries(); + countries.addAll(countryTermRepository.findAll().stream() + .filter(Objects::nonNull) + .map(t -> new Country(t.getTerm(), v.getEnglishName(t.getTerm()))) + .collect(Collectors.toList())); + return countries; + } + + @Override + public Page search(final RequestSort requestSortBy, RequestSortOrder order, RequestFilter requestFilter, final int page, final int size) + throws DsmException { + + final Specification spec = dsSpec(requestSortBy, order, requestFilter); + return dsRepository.findAll(spec, PageRequest.of(page, size)); + } + + + @Override + public Page searchRegistered(final RequestSort requestSortBy, RequestSortOrder order, RequestFilter requestFilter, final int page, final int size) + throws DsmException { + + final Specification spec = dsSpec(requestSortBy, order, requestFilter).and(dsRegisteredbyNotNullSpec()); + return dsRepository.findAll(spec, PageRequest.of(page, size)); + } + + @Override + public DatasourceDbEntry getDs(final String dsId) throws DsmException { + return dsRepository.getOne(dsId); + } + + @Override + public void setManaged(final String id, final boolean managed) { + log.info(String.format("setting managed = '%s' for ds '%s'", managed, id)); + dsRepository.setManaged(id, managed); + apiRepository.setRemovable(id, true); + } + + @Override + public boolean isManaged(final String id) { + return dsRepository.isManaged(id); + } + + @Override + public void updateCompliance(String dsId, String apiId, String compliance, boolean override) { + log.info(String.format("setting compatibility = '%s' for ds '%s'", compliance, apiId)); + apiRepository.updateCompatibility(apiId, compliance); + } + + @Override + public List getApis(final String dsId) { + return apiRepository.findByDatasource(dsId); + } + + @Override + public void deleteApi(final String dsId, final String apiId) throws DsmForbiddenException, DsmNotFoundException { + final ApiDbEntry api = apiRepository.getOne(apiId); + try { + if (!api.getRemovable()) { + throw new DsmForbiddenException(HttpStatus.SC_UNAUTHORIZED, "api is not removable"); + } + + apiRepository.deleteById(apiId); + log.info(String.format("deleted api '%s'", apiId)); + } catch (EntityNotFoundException e) { + throw new DsmNotFoundException(HttpStatus.SC_NOT_FOUND, "api not found"); + } + } + + @Override + public void addApi(final ApiDbEntry api) { + apiRepository.save(api); + } + + @Override + public boolean existDs(final String dsId) throws DsmException { + return dsRepository.existsById(dsId); + } + + @Override + public void saveDs(final DatasourceDbEntry d) { + log.info(String.format("saving datasource '%s'", d.getId())); + + final DatasourceDbEntry datasource = dsRepository.save(d); + log.info(String.format("saved datasource '%s'", datasource.getId())); + + ensureRegistrationDate(d.getId()); + } + + @Override + public void deleteDs(final String dsId) { + dsRepository.deleteById(dsId); + log.info(String.format("deleted datasource '%s'", dsId)); + } + + @Override + public void updateName(final String dsId, final String officialname, final String englishname) { + //TODO what if one of the two names is null or empty? + dsRepository.setDatasourcename(dsId, officialname, englishname); + } + + @Override + public void updateLogoUrl(final String dsId, final String logourl) throws DsmException { + dsRepository.setLogoUrl(dsId, logourl); + } + + @Override + public void updateCoordinates(final String dsId, final Double latitude, final Double longitude) { + dsRepository.setCoordinates(dsId, latitude, longitude); + } + + @Override + public void updateApiBaseUrl(final String apiId, final String baseurl) { + apiRepository.setBaseurl(apiId, baseurl); + } + + @Override + @Transactional + public boolean upsertApiOaiSet(final String apiId, final String oaiSet) throws DsmException { + final ApiDbEntry api = apiRepository.getOne(apiId); + if (OAI.equalsIgnoreCase(api.getProtocol())) { + final Set apiParams = api.getApiParams(); + + if (!apiParams.stream().anyMatch(ap -> SET.equals(ap.getParam()))) { + apiRepository.addApiParam(apiId, SET, oaiSet); + log.info(String.format("added api '%s' oai set with '%s'", apiId, oaiSet)); + return true; + } else { + apiRepository.updateOaiSet(apiId, oaiSet); + log.info(String.format("updated api '%s' oai set with '%s'", apiId, oaiSet)); + return false; + } + } else { + throw new DsmException(String.format("won't add OAI set to a non OAI interface: '%s' has protocol '%s'", apiId, api.getProtocol())); + } + } + + @Override + public List findApiBaseURLs(final RequestFilter requestFilter, final int page, final int size) throws DsmException { + final PageRequest pageable = PageRequest.of(page, size); + final Specification spec = apiSpec(requestFilter); + final Set set = dsApiRepository.findAll(spec, pageable).getContent().stream() + .map(DatasourceApiDbEntry::getBaseurl) + .filter(StringUtils::isNotBlank) + .collect(Collectors.toCollection(HashSet::new)); + return Lists.newArrayList(set); + } + + @Override + public void updateTimezone(final String dsId, final String timezone) { + dsRepository.setTimezone(dsId, timezone); + } + + @Override + public void updateTypology(final String dsId, final String typology) throws DsmException { + final Vocabulary typologies = vocabularyClient.getDatasourceTypologies(); + if (!typologies.hasCode(typology)) { + throw new DsmException( + HttpStatus.SC_BAD_REQUEST, + String.format( + "invalid datasource typology '%s', provide one according to vocabulary %s", + typology, + config.getVocabularies().getDatasourceTypologiesEndpoint())); + } + dsRepository.setTypology(dsId, typology); + } + + @Override + public void updateRegisteringUser(final String dsId, final String registeredBy) throws DsmException { + + ensureRegistrationDate(dsId); + + dsRepository.setRegisteringUser(dsId, registeredBy); + + } + + @Override + public void updatePlatform(final String dsId, final String platform) throws DsmException { + dsRepository.setPlatform(dsId, platform); + } + + //HELPER + private void ensureRegistrationDate(String dsId) { + if (!dsRepository.hasRegistrationdate(dsId)) { + log.info("setting registration date for datasource: " + dsId); + dsRepository.setRegistrationDate(dsId, new Date(System.currentTimeMillis())); + } + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDbEntryRepository.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDbEntryRepository.java new file mode 100644 index 00000000..5771b7d4 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDbEntryRepository.java @@ -0,0 +1,75 @@ +package eu.dnetlib.openaire.dsm.dao; + +import javax.transaction.Transactional; + +import eu.dnetlib.openaire.dsm.domain.db.DatasourceDbEntry; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.JpaSpecificationExecutor; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +import org.springframework.stereotype.Repository; + +import java.sql.Date; + +/** + * Created by claudio on 12/04/2017. + */ +@Repository +@ConditionalOnProperty(value = "openaire.exporter.enable.dsm", havingValue = "true") +public interface DatasourceDbEntryRepository extends JpaRepository, JpaSpecificationExecutor { + + DatasourceDbEntry findOneById(String id); + + @Query("select d.managed from #{#entityName} d where d.id = ?1") + boolean isManaged(String id); + + @Modifying + @Transactional + @Query("update #{#entityName} d set d.managed = ?2 where d.id = ?1") + void setManaged(String id, boolean managed); + + @Modifying + @Transactional + @Query("update #{#entityName} d set d.officialname = ?2, d.englishname = ?3 where d.id = ?1") + void setDatasourcename(String id, String officialname, String englishname); + + @Modifying + @Transactional + @Query("update #{#entityName} d set d.logourl = ?2 where d.id = ?1") + void setLogoUrl(String dsId, String logourl); + + @Modifying + @Transactional + @Query("update #{#entityName} d set d.latitude = ?2, d.longitude = ?3 where d.id = ?1") + void setCoordinates(String dsId, Double latitude, Double longitude); + + @Modifying + @Transactional + @Query("update #{#entityName} d set d.timezone = ?2 where d.id = ?1") + void setTimezone(String dsId, String timezone); + + @Modifying + @Transactional + @Query("update #{#entityName} d set d.typology = ?2 where d.id = ?1") + void setTypology(String dsId, String typology); + + @Modifying + @Transactional + @Query("update #{#entityName} d set d.registeredby = ?2 where d.id = ?1") + void setRegisteringUser(String id, String registeredby); + + @Query("select case when registrationdate <> null then true else false end as hasregistrationdate from #{#entityName} where id = ?1") + Boolean hasRegistrationdate(String id); + + @Modifying + @Transactional + @Query("update #{#entityName} d set d.registrationdate = ?2 where d.id = ?1") + void setRegistrationDate(String id, Date registrationdate); + + @Modifying + @Transactional + @Query("update #{#entityName} d set d.platform = ?2 where d.id = ?1") + void setPlatform(String id, String platform); + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceIndexClient.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceIndexClient.java new file mode 100644 index 00000000..79d8e5e8 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceIndexClient.java @@ -0,0 +1,17 @@ +package eu.dnetlib.openaire.dsm.dao; + +import java.util.Queue; + +import eu.dnetlib.enabling.datasources.common.DsmException; +import eu.dnetlib.openaire.dsm.dao.utils.IndexDsInfo; +import eu.dnetlib.openaire.dsm.dao.utils.IndexRecordsInfo; + +public interface DatasourceIndexClient { + + IndexRecordsInfo getIndexInfo(final String dsId, final IndexDsInfo info, final Queue errors) throws DsmException; + + String getLastIndexingDate(final IndexDsInfo info) throws DsmException; + + + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceIndexClientImpl.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceIndexClientImpl.java new file mode 100644 index 00000000..0788bed1 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceIndexClientImpl.java @@ -0,0 +1,207 @@ +package eu.dnetlib.openaire.dsm.dao; + +import java.io.IOException; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.concurrent.*; +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; + +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.util.concurrent.*; +import eu.dnetlib.OpenaireExporterConfig; +import eu.dnetlib.enabling.datasources.common.DsmException; +import eu.dnetlib.miscutils.functional.hash.Hashing; +import eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils; +import eu.dnetlib.openaire.dsm.dao.utils.IndexDsInfo; +import eu.dnetlib.openaire.dsm.dao.utils.IndexRecordsInfo; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.time.DateFormatUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.impl.CloudSolrClient.Builder; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.common.SolrDocument; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Component; + +/** + * Created by claudio on 20/10/2016. + */ +@Component +@ConditionalOnProperty(value = "openaire.exporter.enable.dsm", havingValue = "true") +public class DatasourceIndexClientImpl implements DatasourceIndexClient { + + private static final Log log = LogFactory.getLog(DatasourceIndexClientImpl.class); + + public static final String SEPARATOR = "::"; + public static final String DSVERSION = "__dsversion"; + + @Autowired + private OpenaireExporterConfig config; + + private ListeningExecutorService executor; + + private static final Map indexClientMap = new ConcurrentHashMap<>(); + + @PostConstruct + public void init() { + executor = MoreExecutors.listeningDecorator( + new ScheduledThreadPoolExecutor(5, + new ThreadFactoryBuilder().setNameFormat("datasource-index-client-%d").build())); + } + + @PreDestroy + public void tearDown() { + indexClientMap.forEach((name, client) -> { + try { + client.close(); + } catch (IOException e) { + log.warn(String.format("unable to gracefully shutdown client for index %s", name)); + } + }); + } + + @Override + public IndexRecordsInfo getIndexInfo(final String dsId, final IndexDsInfo info, final Queue errors) throws DsmException { + try { + final String collectedFrom = StringUtils.substringBefore(dsId, SEPARATOR) + SEPARATOR + Hashing.md5(StringUtils.substringAfter(dsId, SEPARATOR)); + final CloudSolrClient indexClient = getIndexClient(info); + final CountDownLatch latch = new CountDownLatch(2); + final IndexRecordsInfo indexRecordInfo = new IndexRecordsInfo(); + + Futures.addCallback( + executor.submit(() -> setDateAndTotal(dsId, collectedFrom, indexClient)), + new FutureCallback() { + + @Override + public void onSuccess(final IndexRecordsInfo info) { + indexRecordInfo + .setTotal(info.getTotal()) + .setDate(info.getDate()); + latch.countDown(); + } + + @Override + public void onFailure(final Throwable e) { + errors.offer(e); + latch.countDown(); + } + }, executor); + + Futures.addCallback( + executor.submit(() -> setFunded(dsId, collectedFrom, indexClient)), + new FutureCallback() { + + @Override + public void onSuccess(final Long numFound) { + indexRecordInfo.setFunded(numFound); + latch.countDown(); + } + + @Override + public void onFailure(final Throwable e) { + errors.offer(e); + latch.countDown(); + } + }, executor); + + waitLatch(latch, errors, config.getRequestTimeout()); + return indexRecordInfo; + } catch (final Throwable e) { + throw new DsmException(HttpStatus.INTERNAL_SERVER_ERROR.value(), String.format("error reading index info", dsId), e); + } + } + + @Override + public String getLastIndexingDate(final IndexDsInfo info) throws DsmException { + try { + final SolrQuery query = new SolrQuery("oaftype:datasource").setRows(1); + final QueryResponse rsp = getIndexClient(info).query(query); + final SolrDocument doc = Iterables.getFirst(rsp.getResults(), null); + final String dsversion = doc.get("__dsversion").toString(); + return StringUtils.substringBefore(dsversion, "T"); + } catch (SolrServerException | IOException e) { + throw new DsmException(HttpStatus.INTERNAL_SERVER_ERROR.value(), "Error querying index DS profile: " + info, e); + } + } + + private Long setFunded( + final String dsId, + final String collectedFrom, + final CloudSolrClient indexClient) throws DsmException { + final String query = + String.format("oaftype:result AND deletedbyinference:false AND collectedfromdatasourceid:\"%s\" AND relprojectid:*", collectedFrom); + log.debug(String.format("query: %s", query)); + try { + return indexClient.query(new SolrQuery(query).setRows(0)).getResults().getNumFound(); + } catch (Throwable e) { + throw new DsmException(HttpStatus.INTERNAL_SERVER_ERROR.value(), String.format("Error querying index for funded results '%s'", dsId), e); + } + } + + private IndexRecordsInfo setDateAndTotal( + final String dsId, + final String collectedFrom, + final CloudSolrClient indexClient) throws DsmException { + try { + final String query = String.format("oaftype:result AND deletedbyinference:false AND collectedfromdatasourceid:\"%s\"", collectedFrom); + log.debug(String.format("query: %s", query)); + + final QueryResponse rsp = indexClient.query(new SolrQuery(query).setRows(1)); + final SolrDocument doc = Iterables.getFirst(rsp.getResults(), new SolrDocument()); + if (log.isDebugEnabled()) { + log.debug(String.format("got document %s", doc.get("__indexrecordidentifier"))); + } + // if (doc.isEmpty()) { + // throw new DatasourceManagerException(HttpStatus.INTERNAL_SERVER_ERROR.value(), String.format("cannot find document matching + // query: %s", queryTotal)); + // } + return new IndexRecordsInfo() + .setDate(getDate(doc)) + .setTotal(rsp.getResults().getNumFound()); + } catch (Throwable e) { + throw new DsmException(HttpStatus.INTERNAL_SERVER_ERROR.value(), String.format("Error querying index for date and total '%s'", dsId), e); + } + } + + @SuppressWarnings("unchecked") + private String getDate(final SolrDocument doc) throws DsmException { + final List dsversion = (List) doc.get(DSVERSION); + if (dsversion == null || dsversion.isEmpty()) { throw new DsmException(HttpStatus.INTERNAL_SERVER_ERROR.value(), + String.format("cannot find %s in matched solr document", DSVERSION)); } + final Date date = Iterables.getLast(dsversion); + + return DateFormatUtils.format(date, DsmMappingUtils.DATE_FORMAT); + } + + private synchronized CloudSolrClient getIndexClient(final IndexDsInfo info) { + if (!indexClientMap.containsKey(info.getColl())) { + + final CloudSolrClient client = new Builder(Lists.newArrayList(info.getIndexBaseUrl())).build(); + client.setDefaultCollection(info.getColl()); + + indexClientMap.put(info.getColl(), client); + } + return indexClientMap.get(info.getColl()); + } + + private void waitLatch(final CountDownLatch latch, final Queue errors, final int waitSeconds) { + try { + if (!latch.await(waitSeconds, TimeUnit.SECONDS)) { + errors.offer(new TimeoutException("Waiting for requests to complete has timed out.")); + } + } catch (final InterruptedException e) { + errors.offer(e); + } + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceSpecs.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceSpecs.java new file mode 100644 index 00000000..218c527d --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceSpecs.java @@ -0,0 +1,141 @@ +package eu.dnetlib.openaire.dsm.dao; + +import eu.dnetlib.enabling.datasources.common.DsmRuntimeException; +import eu.dnetlib.openaire.dsm.domain.FilterName; +import eu.dnetlib.openaire.dsm.domain.RequestFilter; +import eu.dnetlib.openaire.dsm.domain.RequestSort; +import eu.dnetlib.openaire.dsm.domain.RequestSortOrder; +import eu.dnetlib.openaire.dsm.domain.db.DatasourceApiDbEntry; +import eu.dnetlib.openaire.dsm.domain.db.DatasourceDbEntry; +import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.data.jpa.domain.Specification; + +import javax.persistence.criteria.*; +import java.util.List; +import java.util.Map.Entry; + +public class DatasourceSpecs { + + private static final Log log = LogFactory.getLog(DatasourceSpecs.class); + + public static final String WILDCARD = "%"; + + public static Specification dsRegisteredbyNotNullSpec() { + return (ds, query, cb) -> cb.and( + cb.isNull(ds.get(FilterName.registeredby.name())).not(), + cb.isNull(ds.get("registrationdate")).not()); + } + + public static Specification dsSpec(final RequestSort requestSortBy, final RequestSortOrder order, final RequestFilter requestFilter) { + log.debug(String.format("RequestFilter:'%s', RequestSort:'%s', RequestSortOrder:'%s'", requestFilter, requestSortBy, order)); + return (ds, query, cb) -> { + final Predicate p = cb.conjunction(); + if (requestFilter != null) { + final List> expressions = p.getExpressions(); + requestFilter.entrySet().stream() + .forEach(e -> { + switch (FilterName.type(e.getKey())) { + + case exact: + expressions.add(exactSearch(ds, cb, e)); + + break; + case search: + expressions.add(likeSearch(ds, cb, e)); + + break; + case searchOrgs: + // search by case insensitive organization's country + expressions.add( + cb.equal( + cb.lower( + ds.join("organizations").get(FilterName.country.name())), + getValue(e))); + break; + } + }); + } + if (requestSortBy != null) { + if (order != null) { + final Path orderField = ds.get(requestSortBy.name()); + switch (order) { + case ASCENDING: + query.orderBy(cb.asc(orderField)); + break; + case DESCENDING: + query.orderBy(cb.desc(orderField)); + break; + } + } + } + query.distinct(true); + + return p; + }; + } + + public static Specification apiSpec(final RequestFilter requestFilter) { + log.debug(String.format("RequestFilter:'%s'", requestFilter)); + return (api, query, cb) -> { + final Predicate p = cb.conjunction(); + if (requestFilter != null) { + final List> expressions = p.getExpressions(); + requestFilter.entrySet().stream() + .forEach(e -> { + switch (FilterName.type(e.getKey())) { + + case exact: + expressions.add(exactSearch(api, cb, e)); + break; + case search: + + expressions.add(likeSearch(api, cb, e)); + break; + case searchOrgs: + throw new DsmRuntimeException("not implemented"); + } + }); + } + query.distinct(true); + return p; + }; + } + + // HELPERS + + // substring, case insensitive, like based search + private static Predicate likeSearch(final Root r, final CriteriaBuilder cb, final Entry e) { + return cb.like( + cb.lower( + r.get(e.getKey().name())), + WILDCARD + getValue(e) + WILDCARD); + } + + // search by ID, managed. exact match + private static Predicate exactSearch(final Root r, final CriteriaBuilder cb, final Entry e) { + return cb.equal(r.get(e.getKey().name()), getValue(e)); + } + + private static Object getValue(final Entry e) { + if (e.getValue() instanceof String) { + final String s = ((String) e.getValue()); + + if (!e.getKey().equals(FilterName.country)) { + Boolean b = BooleanUtils.toBooleanObject(s); + if (b != null) { + return b; + } + } + return e.getKey().equals(FilterName.id) ? s : StringUtils.lowerCase(s); + } + if (e.getValue() instanceof Boolean) { + return e.getValue(); + } + + return e.getValue(); + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/MongoLoggerClient.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/MongoLoggerClient.java new file mode 100644 index 00000000..511b6240 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/MongoLoggerClient.java @@ -0,0 +1,14 @@ +package eu.dnetlib.openaire.dsm.dao; + +import eu.dnetlib.enabling.datasources.common.AggregationInfo; +import eu.dnetlib.enabling.datasources.common.DsmException; + +import java.util.List; + +public interface MongoLoggerClient { + + List getAggregationHistory(final String dsId) throws DsmException; + + void dropCache(); + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/MongoLoggerClientImpl.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/MongoLoggerClientImpl.java new file mode 100644 index 00000000..978ccefa --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/MongoLoggerClientImpl.java @@ -0,0 +1,234 @@ +package eu.dnetlib.openaire.dsm.dao; + +import java.io.IOException; +import java.time.Instant; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.stream.Collectors; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.cache.*; +import com.google.common.primitives.Ints; +import com.mongodb.BasicDBObject; +import com.mongodb.MongoClient; +import com.mongodb.client.FindIterable; +import com.mongodb.client.MongoCollection; +import eu.dnetlib.OpenaireExporterConfig; +import eu.dnetlib.OpenaireExporterConfig.Datasource; +import eu.dnetlib.enabling.datasources.common.AggregationInfo; +import eu.dnetlib.enabling.datasources.common.AggregationStage; +import eu.dnetlib.enabling.datasources.common.DsmException; +import eu.dnetlib.miscutils.datetime.DateUtils; +import eu.dnetlib.openaire.common.Utils; + +import eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils; +import eu.dnetlib.openaire.dsm.domain.CollectionInfo; +import eu.dnetlib.openaire.dsm.domain.CollectionMode; +import eu.dnetlib.openaire.dsm.domain.TransformationInfo; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.time.DateFormatUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.http.HttpStatus; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.cache.annotation.CacheEvict; +import org.springframework.cache.annotation.Cacheable; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import static com.mongodb.client.model.Filters.*; +import static com.mongodb.client.model.Projections.fields; +/** + * Created by claudio on 20/10/2016. + */ +@Component +@ConditionalOnProperty(value = "openaire.exporter.enable.dsm", havingValue = "true") +public class MongoLoggerClientImpl implements MongoLoggerClient { + + private static final Log log = LogFactory.getLog(MongoLoggerClientImpl.class); + + @Autowired + private MongoClient datasourcePublisherMongoClient; + + @Autowired + private OpenaireExporterConfig config; + + private final static String LOADTIME = "loadtime"; + private final LoadingCache loadingCache = CacheBuilder.newBuilder() + .maximumSize(1) + .expireAfterWrite(60, TimeUnit.MINUTES) + .build(new CacheLoader() { + // The only cached value is associated to "loadtime" + public Instant load(String key) { + final Instant loadTime = getLoadTime(); + log.debug("found load time: " + loadTime.toString()); + return loadTime; + } + }); + + + private static final Bson fields = getFields(); + + private static MongoCollection collection = null; + + @Override + @Cacheable("dsm-aggregationhistory-cache") + public List getAggregationHistory(final String dsId) throws DsmException { + log.warn(String.format("getAggregationHistory(dsId = %s): not using cache", dsId)); + final Datasource conf = config.getDatasource(); + try { + final FindIterable aggregationDocs = getCollection().find(queryForAggregationHistory(dsId, "(collect|transform)")) + .projection(fields) + .limit(conf.getMongoQueryLimit()) + .sort(dbo("system:startHumanDate", -1)); + + final List aggregationInfos = Utils.stream(aggregationDocs.iterator()) + .map(getMapper()) + .filter(ai -> ai.getNumberOfRecords() >= 0 && StringUtils.isNotBlank(ai.getDate())) + .collect(Collectors.toList()); + + final Instant loadTime = loadingCache.get(LOADTIME); + + if (!Objects.equals(Instant.MIN, loadTime)) { + for (final AggregationInfo a : aggregationInfos) { + if (asInstant(a).isBefore(loadTime) && AggregationStage.COLLECT.equals(a.getAggregationStage())) { + a.setIndexedVersion(true); + break; + } + } + } + + return aggregationInfos; + } catch (Throwable e) { + throw new DsmException(HttpStatus.SC_INTERNAL_SERVER_ERROR, String.format("error reading aggregation history for '%s'", dsId), e); + } + } + + private Instant getLoadTime() { + log.warn("querying for metadata load time, not using cache"); + return Optional.ofNullable(getCollection().find(queryForLastMetadataLoad())) + .map(d -> d.sort(dbo("system:startHumanDate", -1)).first()) + .map(d -> (String) d.getOrDefault("system:startHumanDate", "")) + .map(s -> Instant.parse(s.replaceAll("\\+.*", "Z"))) + .orElse(Instant.MIN); + } + + private Instant asInstant(final AggregationInfo a) { + return Instant.parse(a.getDate() + "T00:00:00Z"); + } + + @Override + @CacheEvict(cacheNames = { "dsm-aggregationhistory-cache", "dsm-firstharvestdate-cache" }, allEntries = true) + @Scheduled(fixedDelayString = "${openaire.exporter.cache.ttl}") + public void dropCache() { + log.debug("dropped dsManager aggregation history cache"); + } + + private Function getMapper() { + return new Function() { + + @Override + public AggregationInfo apply(final Document d) { + + AggregationInfo info = null; + final AggregationStage stage = AggregationStage.parse(d.getString("system:wfName")); + switch (stage) { + + case COLLECT: + CollectionInfo cInfo = new CollectionInfo(); + cInfo.setAggregationStage(stage); + cInfo.setCollectionMode(getCollectionMode(d)); + cInfo.setNumberOfRecords(getNumberOfRecords(d)); + cInfo.setDate(getDate(d)); + info = cInfo; + break; + case TRANSFORM: + TransformationInfo tInfo = new TransformationInfo(); + tInfo.setAggregationStage(stage); + tInfo.setNumberOfRecords(getNumberOfRecords(d)); + tInfo.setDate(getDate(d)); + info = tInfo; + break; + } + return info; + } + + private CollectionMode getCollectionMode(Document d) { + return Optional.ofNullable(d.getString("system:node:SELECT_MODE:selection")) + .map(CollectionMode::valueOf) + .orElseGet(() -> + Optional.ofNullable(d.getString("collectionMode")) + .map(CollectionMode::valueOf) + .orElse(null)); + } + + private Integer getNumberOfRecords(final Document d) { + final String sinkSize = d.getString("mainlog:sinkSize"); + final String total = d.getString("mainlog:total"); + + if (StringUtils.isNotBlank(sinkSize)) { + return Ints.tryParse(sinkSize); + } else if (StringUtils.isNotBlank(total)) { + return Ints.tryParse(total); + } else { + return -1; + } + } + }; + } + + private String getDate(final Document d) { + final String dateString = d.getString("system:startHumanDate"); + if (StringUtils.isBlank(dateString)) { return ""; } + return DateFormatUtils.format(new DateUtils().parse(dateString), DsmMappingUtils.DATE_FORMAT); + } + + private static Bson getFields() { + return fields( + eq("system:wfName", 1), + eq("system:node:SELECT_MODE:selection", 1), + eq("collectionMode", 1), + eq("mainlog:sinkSize", 1), + eq("mainlog:writeOps", 1), + eq("mainlog:total", 1), + eq("system:startHumanDate", 1), + eq("system:profileName", 1)); + } + + private static BasicDBObject dbo(final String key, final Object value) { + return new BasicDBObject(key, value); + } + + private Bson queryForAggregationHistory(final String dsId, final String pattern) { + return and( + eq("parentDatasourceId", dsId), + eq("system:profileFamily", "aggregator"), + eq("system:isCompletedSuccessfully", "true"), + regex("system:wfName", pattern, "i")); + } + + private Bson queryForLastMetadataLoad() { + try { + final String contentLoadQuery = config.getContentLoadQuery(); + log.debug("parsing content load query: " + contentLoadQuery); + return new ObjectMapper().readValue(contentLoadQuery, BasicDBObject.class); + } catch (IOException e) { + throw new IllegalArgumentException(e); + } + } + + private synchronized MongoCollection getCollection() { + if (collection == null) { + log.info("inizializing mongodb collection ..."); + final Datasource conf = config.getDatasource(); + collection = datasourcePublisherMongoClient.getDatabase(conf.getMongoDbName()).getCollection(conf.getMongoCollectionName()); + } + return collection; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ObjectStoreClient.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ObjectStoreClient.java new file mode 100644 index 00000000..131fa713 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ObjectStoreClient.java @@ -0,0 +1,9 @@ +package eu.dnetlib.openaire.dsm.dao; + +import eu.dnetlib.enabling.datasources.common.DsmException; + +public interface ObjectStoreClient { + + Long getObjectStoreSize(final String objectStoreId) throws DsmException; + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ObjectStoreClientImpl.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ObjectStoreClientImpl.java new file mode 100644 index 00000000..78763134 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ObjectStoreClientImpl.java @@ -0,0 +1,37 @@ +package eu.dnetlib.openaire.dsm.dao; + +import eu.dnetlib.data.objectstore.rmi.ObjectStoreService; +import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException; +import eu.dnetlib.enabling.datasources.common.DsmException; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +@Component +@ConditionalOnProperty(value = "openaire.exporter.enable.dsm", havingValue = "true") +public class ObjectStoreClientImpl implements ObjectStoreClient { + + private static final Log log = LogFactory.getLog(ObjectStoreClientImpl.class); + + @Autowired + private ObjectStoreService objectStoreService; + + @Override + public Long getObjectStoreSize(final String objectStoreId) throws DsmException { + log.debug("get size for objectStore " + objectStoreId); + if (StringUtils.isBlank(objectStoreId)) { + return 0L; + } + try { + final long size = objectStoreService.getSize(objectStoreId); + log.debug("got objectStore size: " + size); + return size; + } catch (ObjectStoreServiceException e) { + throw new DsmException("unable to get size for objectStore " + objectStoreId); + } + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ResponseUtils.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ResponseUtils.java new file mode 100644 index 00000000..b417a754 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ResponseUtils.java @@ -0,0 +1,53 @@ +package eu.dnetlib.openaire.dsm.dao; + +import java.util.List; +import java.util.Queue; + +import com.google.common.collect.Lists; + +import eu.dnetlib.openaire.dsm.domain.*; + +public class ResponseUtils { + + public static ApiDetailsResponse apiResponse(final List api, final long total) { + final ApiDetailsResponse rsp = new ApiDetailsResponse().setApi(api); + rsp.setHeader(header(total)); + return rsp; + } + + public static DatasourceSnippetResponse snippetResponse(final List snippets, final long total) { + final DatasourceSnippetResponse rsp = new DatasourceSnippetResponse(snippets); + rsp.setHeader(header(total)); + return rsp; + } + + public static DatasourceDetailResponse detailsResponse(final List details, final long total) { + final DatasourceDetailResponse rsp = new DatasourceDetailResponse(details); + rsp.setHeader(header(total)); + return rsp; + } + + public static DatasourceSearchResponse searchResponse(final List infos, final long total) { + final DatasourceSearchResponse rsp = new DatasourceSearchResponse(infos); + rsp.setHeader(header(total)); + return rsp; + } + + + public static Header header(final Queue errors, final long total) { + return Header.newInsance() + .setExceptions(errors) + .setTotal(total); + } + + public static Header header(final long total) { + return header(Lists.newLinkedList(), total); + } + + public static SimpleResponse simpleResponse(final List list) { + final SimpleResponse rsp = new SimpleResponse().setResponse(list);; + rsp.setHeader(header(Lists.newLinkedList(), list.size())); + return rsp; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/VocabularyClient.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/VocabularyClient.java new file mode 100644 index 00000000..2ed99c55 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/VocabularyClient.java @@ -0,0 +1,14 @@ +package eu.dnetlib.openaire.dsm.dao; + +import eu.dnetlib.enabling.datasources.common.DsmException; +import eu.dnetlib.openaire.vocabularies.Vocabulary; + +public interface VocabularyClient { + + Vocabulary getCountries() throws DsmException; + + Vocabulary getDatasourceTypologies() throws DsmException; + + void dropCache(); + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/VocabularyClientImpl.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/VocabularyClientImpl.java new file mode 100644 index 00000000..82fa2fb7 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/VocabularyClientImpl.java @@ -0,0 +1,58 @@ +package eu.dnetlib.openaire.dsm.dao; + +import eu.dnetlib.OpenaireExporterConfig; +import eu.dnetlib.enabling.datasources.common.DsmException; +import eu.dnetlib.openaire.vocabularies.Vocabulary; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cache.annotation.CacheEvict; +import org.springframework.cache.annotation.Cacheable; +import org.springframework.http.ResponseEntity; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; +import org.springframework.web.client.RestTemplate; + +/** + * Created by claudio on 15/09/2017. + */ +@Component +public class VocabularyClientImpl implements VocabularyClient { + + private static final Log log = LogFactory.getLog(VocabularyClientImpl.class); + + @Autowired + private OpenaireExporterConfig config; + + @Override + @Cacheable("vocabularies-cache") + public Vocabulary getCountries() throws DsmException { + return _getVocabulary(config.getVocabularies().getCountriesEndpoint(), Vocabulary.class); + } + + @Override + @Cacheable("vocabularies-cache") + public Vocabulary getDatasourceTypologies() throws DsmException { + return _getVocabulary(config.getVocabularies().getDatasourceTypologiesEndpoint(), Vocabulary.class); + } + + private T _getVocabulary(final String endpoint, Class clazz) throws DsmException { + final RestTemplate rt = new RestTemplate(); + log.info("get vocabulary from " + endpoint); + final ResponseEntity rsp = rt.getForEntity(endpoint, clazz); + + if (!rsp.getStatusCode().is2xxSuccessful()) { + throw new DsmException(rsp.getStatusCodeValue(), "unable to read content from " + endpoint); + } + + return rsp.getBody(); + } + + @Override + @CacheEvict(cacheNames = "vocabularies-cache", allEntries = true) + @Scheduled(fixedDelayString = "${openaire.exporter.cache.ttl}") + public void dropCache() { + log.debug("dropped dsManager vocabulary cache"); + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/utils/DsmMappingUtils.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/utils/DsmMappingUtils.java new file mode 100644 index 00000000..bc65a0a2 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/utils/DsmMappingUtils.java @@ -0,0 +1,312 @@ +package eu.dnetlib.openaire.dsm.dao.utils; + +import static eu.dnetlib.openaire.common.ExporterConstants.ADMIN_INFO; +import static eu.dnetlib.openaire.common.ExporterConstants.ENGLISH_NAME; +import static eu.dnetlib.openaire.common.ExporterConstants.LATITUDE; +import static eu.dnetlib.openaire.common.ExporterConstants.LONGITUDE; +import static eu.dnetlib.openaire.common.ExporterConstants.OFFICIAL_NAME; +import static eu.dnetlib.openaire.common.ExporterConstants.PLATFORM; +import static eu.dnetlib.openaire.common.ExporterConstants.TIMEZONE; +import static eu.dnetlib.openaire.common.ExporterConstants.TYPOLOGY; +import static eu.dnetlib.openaire.common.ExporterConstants.WEBSITE_URL; + +import java.sql.Date; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.commons.lang3.StringUtils; +import org.dom4j.DocumentHelper; +import org.dom4j.Element; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.BeanWrapper; +import org.springframework.beans.BeanWrapperImpl; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Maps; +import com.google.common.xml.XmlEscapers; + +import eu.dnetlib.data.transform.xml.AbstractDNetXsltFunctions; +import eu.dnetlib.miscutils.datetime.DateUtils; +import eu.dnetlib.openaire.dsm.domain.ApiDetails; +import eu.dnetlib.openaire.dsm.domain.DatasourceDetails; +import eu.dnetlib.openaire.dsm.domain.DatasourceDetailsUpdate; +import eu.dnetlib.openaire.dsm.domain.DatasourceSnippet; +import eu.dnetlib.openaire.dsm.domain.DatasourceSnippetExtended; +import eu.dnetlib.openaire.dsm.domain.OrganizationDetails; +import eu.dnetlib.openaire.dsm.domain.db.ApiDbEntry; +import eu.dnetlib.openaire.dsm.domain.db.DatasourceDbEntry; +import eu.dnetlib.openaire.dsm.domain.db.OrganizationDbEntry; + +public class DsmMappingUtils { + + public static final String DATE_FORMAT = "yyyy-MM-dd"; + + public static final String ID_SEPARATOR = "::"; + + public static final String ID_PREFIX = "api_________" + ID_SEPARATOR; + + public static String createId(final ApiDetails api) { + return ID_PREFIX + api.getDatasource() + ID_SEPARATOR + RandomStringUtils.randomAlphanumeric(8); + } + + public static DatasourceDetails asDetails(final DatasourceDbEntry d) { + final DatasourceDetails details = _convert(d, DatasourceDetails.class); + return details.setOpenaireId(asOpenaireId(details.getId())); + } + + @Deprecated + public static DatasourceSnippet asSnippet(final DatasourceDbEntry d) { + final DatasourceSnippet ds = new DatasourceSnippet(); + ds.setId(d.getId()); + ds.setOfficialname(d.getOfficialname()); + ds.setEnglishname(d.getEnglishname()); + return ds; + } + + public static DatasourceSnippetExtended asSnippetExtended(final DatasourceDbEntry d) { + final DatasourceSnippetExtended ds = new DatasourceSnippetExtended(); + ds.setId(d.getId()); + ds.setOfficialname(d.getOfficialname()); + ds.setEnglishname(d.getEnglishname()); + ds.setRegisteredby(d.getRegisteredby()); + ds.setWebsiteurl(d.getWebsiteurl()); + ds.setTypology(d.getTypology()); + ds.setRegistrationdate(d.getRegistrationdate()); + ds.setLogoUrl(d.getLogourl()); + ds.setDescription(d.getDescription()); + ds.setConsentTermsOfUse(d.getConsentTermsOfUse()); + ds.setConsentTermsOfUseDate(d.getConsentTermsOfUseDate()); + ds.setFullTextDownload(d.getFullTextDownload()); + if (d.getOrganizations() != null) { + ds.setOrganizations(d.getOrganizations().stream().map(DsmMappingUtils::asOrganizationDetail).collect(Collectors.toSet())); + } + return ds; + } + + private static OrganizationDetails asOrganizationDetail(final OrganizationDbEntry o) { + return new OrganizationDetails() + .setCountry(o.getCountry()) + .setLegalname(o.getLegalname()) + .setLegalshortname(o.getLegalshortname()) + .setWebsiteurl(o.getWebsiteurl()) + .setLogourl(o.getLogourl()); + } + + public static ApiDetails asDetails(final ApiDbEntry d) { + return _convert(d, ApiDetails.class); + } + + public static ApiDbEntry asDbEntry(final ApiDetails d) { + final ApiDbEntry apiDbEntry = _convert(d, ApiDbEntry.class); + + // Need to complete the references among objects, because you know, referential integrity ... + apiDbEntry.getApiParams().forEach(ap -> ap.getId().setApi(apiDbEntry)); + + return apiDbEntry; + } + + public static DatasourceDbEntry asDbEntry(final DatasourceDetails d) { + final DatasourceDbEntry dbe = _convert(d, DatasourceDbEntry.class); + if (dbe.getOrganizations() != null) { + dbe.getOrganizations().forEach(o -> { + String prefix = StringUtils.isNotBlank(dbe.getNamespaceprefix()) ? dbe.getNamespaceprefix() : dbe.getId(); + o.setId(prefix + ID_SEPARATOR + o.getLegalname()); + if (o.getDateofcollection() == null) { + o.setDateofcollection(new Date(System.currentTimeMillis())); + } + o.setCollectedfrom(dbe.getCollectedfrom()); + }); + } + return dbe; + } + + public static DatasourceDbEntry asDbEntry(final DatasourceDetailsUpdate d) { + return _convert(d, DatasourceDbEntry.class); + } + + public static String asRepositoryProfile(final DatasourceDetails ds) { + final Element root = DocumentHelper.createElement("RESOURCE_PROFILE"); + + final Element header = root.addElement("HEADER"); + header.addElement("RESOURCE_IDENTIFIER").addAttribute("value", ""); + header.addElement("RESOURCE_TYPE").addAttribute("value", "RepositoryServiceResourceType"); + header.addElement("RESOURCE_KIND").addAttribute("value", "RepositoryServiceResources"); + header.addElement("RESOURCE_URI").addAttribute("value", ""); + header.addElement("DATE_OF_CREATION").addAttribute("value", DateUtils.now_ISO8601()); + header.addElement("PROTOCOL"); + + final Element body = root.addElement("BODY"); + final Element conf = body.addElement("CONFIGURATION"); + conf.addElement("DATASOURCE_TYPE").setText(ds.getTypology()); + + final Element origId = conf.addElement("DATASOURCE_ORIGINAL_ID"); + origId.addAttribute("provenance", "D-NET"); + origId.setText(ds.getId()); + + conf.addElement("DATASOURCE_AGGREGATED").setText("false"); + conf.addElement("ENVIRONMENTS").addElement("ENVIRONMENT").setText("OPENAIRE"); + conf.addElement("TYPOLOGY").setText("" + ds.getTypology()); + conf.addElement("MAX_SIZE_OF_DATASTRUCTURE").setText("0"); + conf.addElement("AVAILABLE_DISKSPACE").setText("0"); + conf.addElement("MAX_NUMBER_OF_DATASTRUCTURE").setText("0"); + + final String officialName = ds.getOfficialname(); + conf.addElement("OFFICIAL_NAME").setText(officialName); + final String englishName = ds.getEnglishname(); + conf.addElement("ENGLISH_NAME").setText(StringUtils.isNotBlank(englishName) ? englishName : officialName); + conf.addElement("ICON_URI").setText("" + ds.getLogourl()); + final OrganizationDetails org = getOrganization(ds); + + conf.addElement("COUNTRY").setText(org != null ? org.getCountry() : ""); + + final Element location = conf.addElement("LOCATION"); + location.addElement("LONGITUDE").setText("" + ds.getLongitude()); + location.addElement("LATITUDE").setText("" + ds.getLatitude()); + location.addElement("TIMEZONE").setText("" + ds.getTimezone()); + + conf.addElement("REPOSITORY_WEBPAGE").setText(ds.getWebsiteurl()); + getOrganization(ds); + conf.addElement("REPOSITORY_INSTITUTION").setText(org != null ? org.getLegalname() : ""); + + conf.addElement("ADMIN_INFO").setText(ds.getContactemail()); + + conf.addElement("INTERFACES"); + + final Element extraFields = conf.addElement("EXTRA_FIELDS"); + addExtraField(extraFields, "OpenAireDataSourceId", ds.getId()); + addExtraField(extraFields, "ACTIVATION_ID", ds.getActivationId()); + addExtraField(extraFields, "NamespacePrefix", ds.getNamespaceprefix()); + addExtraField(extraFields, "aggregatorName", ds.getAggregator()); + addExtraField(extraFields, "dateOfCollection", "" + ds.getDateofcollection()); + addExtraField(extraFields, "dateOfValidation", "" + ds.getDateofvalidation()); + conf.addElement("REGISTERED_BY").setText(ds.getRegisteredby()); + + final Element status = body.addElement("STATUS"); + status.addElement("NUMBER_OF_OBJECTS").setText("0"); + status.addElement("LAST_UPDATE").addAttribute("value", DateUtils.now_ISO8601()); + + final Element qos = body.addElement("QOS"); + qos.addElement("AVAILABILITY").setText("0"); + qos.addElement("CAPACITY"); + qos.addElement("THROUGHPUT").setText("0"); + + body.addElement("SECURITY_PARAMETERS"); + body.addElement("BLACKBOARD"); + + return root.asXML(); + } + + public static String asRepositoryInterfce(final ApiDetails api) { + final Element iface = DocumentHelper.createElement("INTERFACE"); + + iface.addAttribute("active", String.valueOf(api.getActive())) + .addAttribute("compliance", api.getCompatibility()) + .addAttribute("contentDescription", api.getContentdescription()) + .addAttribute("id", api.getId()) + .addAttribute("label", String.format("%s (%s)", api.getTypology(), api.getCompatibility())) + .addAttribute("removable", String.valueOf(api.getRemovable())) + .addAttribute("typology", api.getTypology()); + iface.addElement("ACCESS_PROTOCOL").setText(api.getProtocol()); + if (api.getApiParams() != null) { + final Element accessProtocol = (Element) iface.selectSingleNode("./ACCESS_PROTOCOL"); + api.getApiParams().forEach(ap -> { + accessProtocol.addAttribute(ap.getParam(), ap.getValue()); + }); + } + iface.addElement("BASE_URL").setText(api.getBaseurl()); + iface.addElement("INTERFACE_EXTRA_FIELD").addAttribute("name", "last_collection_date"); + iface.addElement("INTERFACE_EXTRA_FIELD").addAttribute("name", "last_collection_mdId"); + iface.addElement("INTERFACE_EXTRA_FIELD").addAttribute("name", "last_collection_total"); + iface.addElement("INTERFACE_EXTRA_FIELD").addAttribute("name", "last_aggregation_date"); + iface.addElement("INTERFACE_EXTRA_FIELD").addAttribute("name", "last_aggregation_mdId"); + iface.addElement("INTERFACE_EXTRA_FIELD").addAttribute("name", "last_aggregation_total"); + + final Element mdPathNode = iface.addElement("INTERFACE_EXTRA_FIELD"); + mdPathNode.addAttribute("name", "metadata_identifier_path"); + if (StringUtils.isNotBlank(api.getMetadataIdentifierPath())) { + mdPathNode.setText(api.getMetadataIdentifierPath()); + } + + return iface.asXML(); + } + + // HELPERS + + private static T _convert(final Object o, final Class clazz) { + final ObjectMapper mapper = new ObjectMapper(); + return mapper.convertValue(o, clazz); + } + + private static OrganizationDetails getOrganization(final DatasourceDetails ds) { + if (ds.getOrganizations() != null && !ds.getOrganizations().isEmpty()) { return ds.getOrganizations().stream().findFirst().get(); } + return null; + } + + private static void addExtraField(final Element extraFields, final String field, final String value) { + final Element f = extraFields.addElement("FIELD"); + f.addElement("key").setText(field); + f.addElement("value").setText(value != null ? value : ""); + } + + private static String asOpenaireId(final String id) { + final String prefix = StringUtils.substringBefore(id, ID_SEPARATOR); + final String md5 = StringUtils.substringAfter(id, ID_SEPARATOR); + + return prefix + ID_SEPARATOR + AbstractDNetXsltFunctions.md5(md5); + } + + public static void copyNonNullProperties(final Object src, final Object target) { + BeanUtils.copyProperties(src, target, getNullPropertyNames(src)); + } + + public static String[] getNullPropertyNames(final Object source) { + final BeanWrapper src = new BeanWrapperImpl(source); + final java.beans.PropertyDescriptor[] pds = src.getPropertyDescriptors(); + + final Set emptyNames = new HashSet<>(); + for (final java.beans.PropertyDescriptor pd : pds) { + final Object srcValue = src.getPropertyValue(pd.getName()); + if (srcValue == null) { + emptyNames.add(pd.getName()); + } + } + final String[] result = new String[emptyNames.size()]; + return emptyNames.toArray(result); + } + + public static Map asMapOfChanges(final DatasourceDetailsUpdate d) { + final Map changes = Maps.newHashMap(); + + if (d.getContactemail() != null) { + changes.put(ADMIN_INFO, XmlEscapers.xmlContentEscaper().escape(d.getContactemail())); + } + if (d.getEnglishname() != null) { + changes.put(ENGLISH_NAME, XmlEscapers.xmlContentEscaper().escape(d.getEnglishname())); + } + if (d.getOfficialname() != null) { + changes.put(OFFICIAL_NAME, XmlEscapers.xmlContentEscaper().escape(d.getOfficialname())); + } + if (d.getTimezone() != null) { + changes.put(TIMEZONE, XmlEscapers.xmlContentEscaper().escape(d.getTimezone())); + } + if (d.getLatitude() != null) { + changes.put(LATITUDE, XmlEscapers.xmlContentEscaper().escape(String.valueOf(d.getLatitude()))); + } + if (d.getLongitude() != null) { + changes.put(LONGITUDE, XmlEscapers.xmlContentEscaper().escape(String.valueOf(d.getLongitude()))); + } + if (d.getPlatform() != null) { + changes.put(PLATFORM, XmlEscapers.xmlContentEscaper().escape(d.getPlatform())); + } + if (d.getTypology() != null) { + changes.put(TYPOLOGY, XmlEscapers.xmlContentEscaper().escape(d.getTypology())); + } + if (d.getWebsiteurl() != null) { + changes.put(WEBSITE_URL, XmlEscapers.xmlContentEscaper().escape(d.getWebsiteurl())); + } + return changes; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/utils/IndexDsInfo.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/utils/IndexDsInfo.java new file mode 100644 index 00000000..effb9491 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/utils/IndexDsInfo.java @@ -0,0 +1,36 @@ +package eu.dnetlib.openaire.dsm.dao.utils; + +/** + * Created by claudio on 20/10/2016. + */ +public class IndexDsInfo { + + private final String indexBaseUrl; + private final String indexDsId; + private final String format; + private final String coll; + + public IndexDsInfo(final String indexBaseUrl, final String indexDsId, final String format, final String coll) { + this.indexBaseUrl = indexBaseUrl; + this.indexDsId = indexDsId; + this.format = format; + this.coll = coll; + } + + public String getIndexBaseUrl() { + return indexBaseUrl; + } + + public String getIndexDsId() { + return indexDsId; + } + + public String getFormat() { + return format; + } + + public String getColl() { + return coll; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/utils/IndexRecordsInfo.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/utils/IndexRecordsInfo.java new file mode 100644 index 00000000..5f3e1256 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/utils/IndexRecordsInfo.java @@ -0,0 +1,49 @@ +package eu.dnetlib.openaire.dsm.dao.utils; + +/** + * Created by claudio on 21/10/2016. + */ +public class IndexRecordsInfo { + + private long total; + + private long funded; + + private String date; + + public IndexRecordsInfo() {} + + public IndexRecordsInfo(final long total, final long funded, final String date) { + this.total = total; + this.funded = funded; + this.date = date; + } + + public long getTotal() { + return total; + } + + public IndexRecordsInfo setTotal(final long total) { + this.total = total; + return this; + } + + public long getFunded() { + return funded; + } + + public IndexRecordsInfo setFunded(final long funded) { + this.funded = funded; + return this; + } + + public String getDate() { + return date; + } + + public IndexRecordsInfo setDate(final String date) { + this.date = date; + return this; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/AggregationHistoryResponse.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/AggregationHistoryResponse.java new file mode 100644 index 00000000..eeddbc94 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/AggregationHistoryResponse.java @@ -0,0 +1,29 @@ +package eu.dnetlib.openaire.dsm.domain; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import eu.dnetlib.enabling.datasources.common.AggregationInfo; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +import java.util.List; + +@ApiModel +@JsonAutoDetect +public class AggregationHistoryResponse extends Response { + + @ApiModelProperty(position = 1) + private List aggregationInfo; + + public AggregationHistoryResponse(List aggregationInfo) { + super(); + this.aggregationInfo = aggregationInfo; + } + + public List getAggregationInfo() { + return aggregationInfo; + } + + public void setAggregationInfo(List aggregationInfo) { + this.aggregationInfo = aggregationInfo; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/ApiDetails.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/ApiDetails.java new file mode 100644 index 00000000..5a51df3d --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/ApiDetails.java @@ -0,0 +1,214 @@ +package eu.dnetlib.openaire.dsm.domain; + +import java.sql.Date; +import java.util.Set; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +@JsonAutoDetect +@ApiModel(value = "Api model", description = "provides information about the datasource API") +public class ApiDetails extends ApiIgnoredProperties { + + @ApiModelProperty(position = 0) + private String id = null; + + @ApiModelProperty(position = 1) + private String protocol = null; + + @ApiModelProperty(position = 2) + private String datasource = null; + + @ApiModelProperty(position = 3) + private String contentdescription = null; + + @ApiModelProperty(position = 4) + private String typology = null; + + @ApiModelProperty(position = 5) + private String compatibility; + + @ApiModelProperty(position = 7) + private String compatibilityOverride; + + @ApiModelProperty(position = 8) + private Integer lastCollectionTotal; + + @ApiModelProperty(position = 9) + private Date lastCollectionDate; + + @ApiModelProperty(position = 10) + private Integer lastAggregationTotal; + + @ApiModelProperty(position = 11) + private Date lastAggregationDate; + + @ApiModelProperty(position = 12) + private Integer lastDownloadTotal; + + @ApiModelProperty(position = 13) + private Date lastDownloadDate; + + @ApiModelProperty(position = 14) + private String baseurl; + + @ApiModelProperty(position = 15) + protected Boolean removable = false; + + @ApiModelProperty(position = 16) + private Set apiParams; + + @ApiModelProperty(position = 17) + private String metadataIdentifierPath = ""; + + public String getId() { + return id; + } + + public String getProtocol() { + return protocol; + } + + public String getDatasource() { + return datasource; + } + + public String getContentdescription() { + return contentdescription; + } + + public String getTypology() { + return typology; + } + + public String getCompatibility() { + return compatibility; + } + + public Integer getLastCollectionTotal() { + return lastCollectionTotal; + } + + public Date getLastCollectionDate() { + return lastCollectionDate; + } + + public Integer getLastAggregationTotal() { + return lastAggregationTotal; + } + + public Date getLastAggregationDate() { + return lastAggregationDate; + } + + public Integer getLastDownloadTotal() { + return lastDownloadTotal; + } + + public Date getLastDownloadDate() { + return lastDownloadDate; + } + + public String getBaseurl() { + return baseurl; + } + + public ApiDetails setId(final String id) { + this.id = id; + return this; + } + + public ApiDetails setProtocol(final String protocol) { + this.protocol = protocol; + return this; + } + + public ApiDetails setDatasource(final String datasource) { + this.datasource = datasource; + return this; + } + + public ApiDetails setContentdescription(final String contentdescription) { + this.contentdescription = contentdescription; + return this; + } + + public ApiDetails setTypology(final String typology) { + this.typology = typology; + return this; + } + + public ApiDetails setCompatibility(final String compatibility) { + this.compatibility = compatibility; + return this; + } + + public ApiDetails setLastCollectionTotal(final Integer lastCollectionTotal) { + this.lastCollectionTotal = lastCollectionTotal; + return this; + } + + public ApiDetails setLastCollectionDate(final Date lastCollectionDate) { + this.lastCollectionDate = lastCollectionDate; + return this; + } + + public ApiDetails setLastAggregationTotal(final Integer lastAggregationTotal) { + this.lastAggregationTotal = lastAggregationTotal; + return this; + } + + public ApiDetails setLastAggregationDate(final Date lastAggregationDate) { + this.lastAggregationDate = lastAggregationDate; + return this; + } + + public ApiDetails setLastDownloadTotal(final Integer lastDownloadTotal) { + this.lastDownloadTotal = lastDownloadTotal; + return this; + } + + public ApiDetails setLastDownloadDate(final Date lastDownloadDate) { + this.lastDownloadDate = lastDownloadDate; + return this; + } + + public ApiDetails setBaseurl(final String baseurl) { + this.baseurl = baseurl; + return this; + } + + public Set getApiParams() { + return apiParams; + } + + public void setApiParams(final Set apiParams) { + this.apiParams = apiParams; + } + + public String getCompatibilityOverride() { + return compatibilityOverride; + } + + public void setCompatibilityOverride(final String compatibilityOverride) { + this.compatibilityOverride = compatibilityOverride; + } + + public Boolean getRemovable() { + return removable; + } + + public void setRemovable(final Boolean removable) { + this.removable = removable; + } + + public String getMetadataIdentifierPath() { + return metadataIdentifierPath; + } + + public void setMetadataIdentifierPath(final String metadataIdentifierPath) { + this.metadataIdentifierPath = metadataIdentifierPath; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/ApiDetailsResponse.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/ApiDetailsResponse.java new file mode 100644 index 00000000..495dc908 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/ApiDetailsResponse.java @@ -0,0 +1,24 @@ +package eu.dnetlib.openaire.dsm.domain; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +@ApiModel +@JsonAutoDetect +public class ApiDetailsResponse extends Response { + + @ApiModelProperty(position = 1) + private List api; + + public List getApi() { + return api; + } + + public ApiDetailsResponse setApi(final List api) { + this.api = api; + return this; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/ApiIgnoredProperties.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/ApiIgnoredProperties.java new file mode 100644 index 00000000..d674f260 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/ApiIgnoredProperties.java @@ -0,0 +1,72 @@ +package eu.dnetlib.openaire.dsm.domain; + +import com.fasterxml.jackson.annotation.JsonIgnore; + +public abstract class ApiIgnoredProperties { + + @JsonIgnore + protected Boolean active = false; + + @JsonIgnore + protected String lastCollectionMdid; + + @JsonIgnore + protected String lastAggregationMdid; + + @JsonIgnore + protected String lastDownloadObjid; + + @JsonIgnore + protected String lastValidationJob; + + @JsonIgnore + protected boolean compatibilityOverrided; + + public Boolean getActive() { + return active; + } + + public void setActive(final Boolean active) { + this.active = active; + } + + public String getLastCollectionMdid() { + return lastCollectionMdid; + } + + public void setLastCollectionMdid(final String lastCollectionMdid) { + this.lastCollectionMdid = lastCollectionMdid; + } + + public String getLastAggregationMdid() { + return lastAggregationMdid; + } + + public void setLastAggregationMdid(final String lastAggregationMdid) { + this.lastAggregationMdid = lastAggregationMdid; + } + + public String getLastDownloadObjid() { + return lastDownloadObjid; + } + + public void setLastDownloadObjid(final String lastDownloadObjid) { + this.lastDownloadObjid = lastDownloadObjid; + } + + public String getLastValidationJob() { + return lastValidationJob; + } + + public void setLastValidationJob(final String lastValidationJob) { + this.lastValidationJob = lastValidationJob; + } + + public boolean isCompatibilityOverrided() { + return compatibilityOverrided; + } + + public void setCompatibilityOverrided(final boolean compatibilityOverrided) { + this.compatibilityOverrided = compatibilityOverrided; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/ApiParamDetails.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/ApiParamDetails.java new file mode 100644 index 00000000..158bd72a --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/ApiParamDetails.java @@ -0,0 +1,24 @@ +package eu.dnetlib.openaire.dsm.domain; + +public class ApiParamDetails { + + protected String param; + + protected String value; + + public String getParam() { + return param; + } + + public void setParam(final String param) { + this.param = param; + } + + public String getValue() { + return value; + } + + public void setValue(final String value) { + this.value = value; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/CollectionInfo.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/CollectionInfo.java new file mode 100644 index 00000000..60df410d --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/CollectionInfo.java @@ -0,0 +1,25 @@ +package eu.dnetlib.openaire.dsm.domain; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +import eu.dnetlib.enabling.datasources.common.AggregationInfo; +import io.swagger.annotations.ApiModel; + +/** + * Created by claudio on 29/11/2016. + */ +@ApiModel +@JsonAutoDetect +public class CollectionInfo extends AggregationInfo { + + private CollectionMode collectionMode; + + public CollectionMode getCollectionMode() { + return collectionMode; + } + + public void setCollectionMode(final CollectionMode collectionMode) { + this.collectionMode = collectionMode; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/CollectionMode.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/CollectionMode.java new file mode 100644 index 00000000..996bd6bf --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/CollectionMode.java @@ -0,0 +1,13 @@ +package eu.dnetlib.openaire.dsm.domain; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import io.swagger.annotations.ApiModel; + +/** + * Created by claudio on 12/09/16. + */ +@ApiModel +@JsonAutoDetect +public enum CollectionMode { + REFRESH, INCREMENTAL +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetailResponse.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetailResponse.java new file mode 100644 index 00000000..75ca070b --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetailResponse.java @@ -0,0 +1,28 @@ +package eu.dnetlib.openaire.dsm.domain; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +import java.util.List; + +@ApiModel +@JsonAutoDetect +public class DatasourceDetailResponse extends Response { + + @ApiModelProperty(position = 1) + private List datasourceInfo; + + public DatasourceDetailResponse(List datasourceInfo) { + super(); + this.datasourceInfo = datasourceInfo; + } + + public List getDatasourceInfo() { + return datasourceInfo; + } + + public void setDatasourceInfo(List datasourceInfo) { + this.datasourceInfo = datasourceInfo; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetails.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetails.java new file mode 100644 index 00000000..deb52543 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetails.java @@ -0,0 +1,402 @@ +package eu.dnetlib.openaire.dsm.domain; + +import java.sql.Date; +import java.util.Set; + +import javax.persistence.Transient; +import javax.validation.constraints.Email; +import javax.validation.constraints.NotBlank; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +/** + * Created by claudio on 12/09/16. + */ +@JsonAutoDetect +@ApiModel(value = "Datasource model", description = "provides information about the datasource") +public class DatasourceDetails extends DatasourceIgnoredProperties { + + @NotBlank + @ApiModelProperty(position = 0) + private String id; + + @Transient + @ApiModelProperty(position = 1) + private String openaireId; + + @NotBlank + @ApiModelProperty(position = 2) + private String officialname; + + @NotBlank + @ApiModelProperty(position = 3) + private String englishname; + + @ApiModelProperty(position = 4) + private String websiteurl; + + @ApiModelProperty(position = 5) + private String logourl; + + @Email + @ApiModelProperty(position = 6) + private String contactemail; + + @ApiModelProperty(position = 7) + private Double latitude; + + @ApiModelProperty(position = 8) + private Double longitude; + + @ApiModelProperty(position = 9) + private String timezone; + + @NotBlank + @ApiModelProperty(position = 10) + private String namespaceprefix; + + @ApiModelProperty(position = 11) + private String languages; + + @ApiModelProperty(position = 12) + private Date dateofvalidation; + + @NotBlank + @ApiModelProperty(position = 13) + private String typology; + + @ApiModelProperty(position = 14) + private Date dateofcollection; + + @ApiModelProperty(position = 15) + private String platform; + + @ApiModelProperty(position = 16) + private String activationId; + + @ApiModelProperty(position = 17) + private String description; + + @ApiModelProperty(position = 18) + private String issn; + + @ApiModelProperty(position = 19) + private String eissn; + + @ApiModelProperty(position = 20) + private String lissn; + + @Email + @ApiModelProperty(position = 21) + private String registeredby; + + @ApiModelProperty(position = 22) + private String subjects; + + @ApiModelProperty(position = 23) + protected String aggregator = "OPENAIRE"; + + @ApiModelProperty(position = 24) + protected String collectedfrom; + + @ApiModelProperty(position = 25) + private Boolean managed; + + @ApiModelProperty(position = 28) + private Boolean consentTermsOfUse; + + @ApiModelProperty(position = 29) + private Boolean fullTextDownload; + + @ApiModelProperty(position = 30) + private Date consentTermsOfUseDate; + + @ApiModelProperty(position = 26) + private Set organizations; + + @ApiModelProperty(position = 27) + private Set identities; + + public String getId() { + return id; + } + + public String getOpenaireId() { + return openaireId; + } + + public String getOfficialname() { + return officialname; + } + + public String getEnglishname() { + return englishname; + } + + public String getWebsiteurl() { + return websiteurl; + } + + public String getLogourl() { + return logourl; + } + + public String getContactemail() { + return contactemail; + } + + public Double getLatitude() { + return latitude; + } + + public Double getLongitude() { + return longitude; + } + + public String getTimezone() { + return timezone; + } + + public String getLanguages() { + return languages; + } + + public String getNamespaceprefix() { + return namespaceprefix; + } + + public Date getDateofvalidation() { + return dateofvalidation; + } + + public String getTypology() { + return typology; + } + + public Date getDateofcollection() { + return dateofcollection; + } + + public String getPlatform() { + return platform; + } + + public String getActivationId() { + return activationId; + } + + public String getDescription() { + return description; + } + + public String getIssn() { + return issn; + } + + public String getEissn() { + return eissn; + } + + public String getLissn() { + return lissn; + } + + public String getRegisteredby() { + return registeredby; + } + + public String getSubjects() { + return subjects; + } + + public String getAggregator() { + return aggregator; + } + + public String getCollectedfrom() { + return collectedfrom; + } + + public Boolean getManaged() { + return managed; + } + + public Boolean getConsentTermsOfUse() { + return consentTermsOfUse; + } + + public Boolean getFullTextDownload() { + return fullTextDownload; + } + + public Set getOrganizations() { + return organizations; + } + + public Set getIdentities() { + return identities; + } + + public DatasourceDetails setId(final String id) { + this.id = id; + return this; + } + + public DatasourceDetails setOpenaireId(final String openaireId) { + this.openaireId = openaireId; + return this; + } + + public DatasourceDetails setOfficialname(final String officialname) { + this.officialname = officialname; + return this; + } + + public DatasourceDetails setEnglishname(final String englishname) { + this.englishname = englishname; + return this; + } + + public DatasourceDetails setWebsiteurl(final String websiteurl) { + this.websiteurl = websiteurl; + return this; + } + + public DatasourceDetails setLogourl(final String logourl) { + this.logourl = logourl; + return this; + } + + public DatasourceDetails setContactemail(final String contactemail) { + this.contactemail = contactemail; + return this; + } + + public DatasourceDetails setLatitude(final Double latitude) { + this.latitude = latitude; + return this; + } + + public DatasourceDetails setLongitude(final Double longitude) { + this.longitude = longitude; + return this; + } + + public DatasourceDetails setTimezone(final String timezone) { + this.timezone = timezone; + return this; + } + + public DatasourceDetails setLanguages(final String languages) { + this.languages = languages; + return this; + } + + public DatasourceDetails setNamespaceprefix(final String namespaceprefix) { + this.namespaceprefix = namespaceprefix; + return this; + } + + public DatasourceDetails setDateofvalidation(final Date dateofvalidation) { + this.dateofvalidation = dateofvalidation; + return this; + } + + public DatasourceDetails setTypology(final String typology) { + this.typology = typology; + return this; + } + + public DatasourceDetails setDateofcollection(final Date dateofcollection) { + this.dateofcollection = dateofcollection; + return this; + } + + public DatasourceDetails setPlatform(final String platform) { + this.platform = platform; + return this; + } + + public DatasourceDetails setActivationId(final String activationId) { + this.activationId = activationId; + return this; + } + + public DatasourceDetails setDescription(final String description) { + this.description = description; + return this; + } + + public DatasourceDetails setIssn(final String issn) { + this.issn = issn; + return this; + } + + public DatasourceDetails setEissn(final String eissn) { + this.eissn = eissn; + return this; + } + + public DatasourceDetails setLissn(final String lissn) { + this.lissn = lissn; + return this; + } + + public DatasourceDetails setRegisteredby(final String registeredby) { + this.registeredby = registeredby; + return this; + } + + public DatasourceDetails setSubjects(final String subjects) { + this.subjects = subjects; + return this; + } + + public DatasourceDetails setAggregator(final String aggregator) { + this.aggregator = aggregator; + return this; + } + + public DatasourceDetails setCollectedfrom(final String collectedfrom) { + this.collectedfrom = collectedfrom; + return this; + } + + public DatasourceDetails setManaged(final Boolean managed) { + this.managed = managed; + return this; + } + + public DatasourceDetails setOrganizations(final Set organizations) { + this.organizations = organizations; + return this; + } + + public DatasourceDetails setIdentities(final Set identities) { + this.identities = identities; + return this; + } + + public DatasourceDetails setConsentTermsOfUse(final Boolean consentTermsOfUse) { + this.consentTermsOfUse = consentTermsOfUse; + return this; + } + + public DatasourceDetails setFullTextDownload(final Boolean fullTextDownload) { + this.fullTextDownload = fullTextDownload; + return this; + } + + public Date getConsentTermsOfUseDate() { + return consentTermsOfUseDate; + } + + public DatasourceDetails setConsentTermsOfUseDate(final Date consentTermsOfUseDate) { + this.consentTermsOfUseDate = consentTermsOfUseDate; + return this; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetailsUpdate.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetailsUpdate.java new file mode 100644 index 00000000..c3ddcb88 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetailsUpdate.java @@ -0,0 +1,240 @@ +package eu.dnetlib.openaire.dsm.domain; + +import java.sql.Date; +import java.util.Set; + +import javax.validation.constraints.Email; +import javax.validation.constraints.NotBlank; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +/** + * Created by claudio on 12/09/16. + */ +@JsonAutoDetect +@ApiModel(value = "Datasource updatable fields model", description = "provides information about the datasource field that can be updated") +public class DatasourceDetailsUpdate { + + @NotBlank + @ApiModelProperty(position = 0) + private String id; + + @NotBlank + @ApiModelProperty(position = 2) + private String officialname; + + @NotBlank + @ApiModelProperty(position = 3) + private String englishname; + + @ApiModelProperty(position = 4) + private String websiteurl; + + @ApiModelProperty(position = 5) + private String logourl; + + @Email + @ApiModelProperty(position = 6) + private String contactemail; + + @ApiModelProperty(position = 7) + private Double latitude; + + @ApiModelProperty(position = 8) + private Double longitude; + + @ApiModelProperty(position = 9) + private String timezone; + + @NotBlank + @ApiModelProperty(position = 13) + private String typology; + + @ApiModelProperty(position = 15) + private String platform; + + @ApiModelProperty(position = 17) + private String description; + + @Email + @ApiModelProperty(position = 21) + private String registeredby; + + @ApiModelProperty(position = 25) + private Boolean managed; + + @ApiModelProperty(position = 27) + private Set identities; + + @ApiModelProperty(position = 28) + private Boolean consentTermsOfUse; + + @ApiModelProperty(position = 29) + private Date consentTermsOfUseDate; + + @ApiModelProperty(position = 30) + private Boolean fullTextDownload; + + public String getId() { + return id; + } + + public String getOfficialname() { + return officialname; + } + + public String getEnglishname() { + return englishname; + } + + public String getWebsiteurl() { + return websiteurl; + } + + public String getLogourl() { + return logourl; + } + + public String getContactemail() { + return contactemail; + } + + public Double getLatitude() { + return latitude; + } + + public Double getLongitude() { + return longitude; + } + + public String getTimezone() { + return timezone; + } + + public String getTypology() { + return typology; + } + + public String getPlatform() { + return platform; + } + + public String getDescription() { + return description; + } + + public String getRegisteredby() { + return registeredby; + } + + public Boolean getManaged() { + return managed; + } + + public Set getIdentities() { + return identities; + } + + public DatasourceDetailsUpdate setId(final String id) { + this.id = id; + return this; + } + + public DatasourceDetailsUpdate setOfficialname(final String officialname) { + this.officialname = officialname; + return this; + } + + public DatasourceDetailsUpdate setEnglishname(final String englishname) { + this.englishname = englishname; + return this; + } + + public DatasourceDetailsUpdate setWebsiteurl(final String websiteurl) { + this.websiteurl = websiteurl; + return this; + } + + public DatasourceDetailsUpdate setLogourl(final String logourl) { + this.logourl = logourl; + return this; + } + + public DatasourceDetailsUpdate setContactemail(final String contactemail) { + this.contactemail = contactemail; + return this; + } + + public DatasourceDetailsUpdate setLatitude(final Double latitude) { + this.latitude = latitude; + return this; + } + + public DatasourceDetailsUpdate setLongitude(final Double longitude) { + this.longitude = longitude; + return this; + } + + public DatasourceDetailsUpdate setTimezone(final String timezone) { + this.timezone = timezone; + return this; + } + + public DatasourceDetailsUpdate setTypology(final String typology) { + this.typology = typology; + return this; + } + + public DatasourceDetailsUpdate setPlatform(final String platform) { + this.platform = platform; + return this; + } + + public DatasourceDetailsUpdate setDescription(final String description) { + this.description = description; + return this; + } + + public DatasourceDetailsUpdate setRegisteredby(final String registeredby) { + this.registeredby = registeredby; + return this; + } + + public DatasourceDetailsUpdate setManaged(final Boolean managed) { + this.managed = managed; + return this; + } + + public DatasourceDetailsUpdate setIdentities(final Set identities) { + this.identities = identities; + return this; + } + + public Boolean getConsentTermsOfUse() { + return consentTermsOfUse; + } + + public void setConsentTermsOfUse(final Boolean consentTermsOfUse) { + this.consentTermsOfUse = consentTermsOfUse; + } + + public Date getConsentTermsOfUseDate() { + return consentTermsOfUseDate; + } + + public void setConsentTermsOfUseDate(final Date consentTermsOfUseDate) { + this.consentTermsOfUseDate = consentTermsOfUseDate; + } + + public Boolean getFullTextDownload() { + return fullTextDownload; + } + + public void setFullTextDownload(final Boolean fullTextDownload) { + this.fullTextDownload = fullTextDownload; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceIgnoredProperties.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceIgnoredProperties.java new file mode 100644 index 00000000..94374a5f --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceIgnoredProperties.java @@ -0,0 +1,196 @@ +package eu.dnetlib.openaire.dsm.domain; + +import java.sql.Date; + +import com.fasterxml.jackson.annotation.JsonIgnore; + +public abstract class DatasourceIgnoredProperties { + + @JsonIgnore + protected String od_contenttypes; + + @JsonIgnore + protected String provenanceaction; + + @JsonIgnore + protected Date releasestartdate; + + @JsonIgnore + protected Date releaseenddate; + + @JsonIgnore + protected String missionstatementurl; + + @JsonIgnore + protected Boolean dataprovider; + + @JsonIgnore + protected Boolean serviceprovider; + + @JsonIgnore + protected String databaseaccesstype; + + @JsonIgnore + protected String datauploadtype; + + @JsonIgnore + protected String databaseaccessrestriction; + + @JsonIgnore + protected String datauploadrestriction; + + @JsonIgnore + protected Boolean versioning; + + @JsonIgnore + protected String citationguidelineurl; + + @JsonIgnore + protected String qualitymanagementkind; + + @JsonIgnore + protected String pidsystems; + + @JsonIgnore + protected String certificates; + + @JsonIgnore + protected Date registrationdate; + + public String getOd_contenttypes() { + return od_contenttypes; + } + + public void setOd_contenttypes(final String od_contenttypes) { + this.od_contenttypes = od_contenttypes; + } + + public String getProvenanceaction() { + return provenanceaction; + } + + public void setProvenanceaction(final String provenanceaction) { + this.provenanceaction = provenanceaction; + } + + public Date getReleasestartdate() { + return releasestartdate; + } + + public void setReleasestartdate(final Date releasestartdate) { + this.releasestartdate = releasestartdate; + } + + public Date getReleaseenddate() { + return releaseenddate; + } + + public void setReleaseenddate(final Date releaseenddate) { + this.releaseenddate = releaseenddate; + } + + public String getMissionstatementurl() { + return missionstatementurl; + } + + public void setMissionstatementurl(final String missionstatementurl) { + this.missionstatementurl = missionstatementurl; + } + + public Boolean getDataprovider() { + return dataprovider; + } + + public void setDataprovider(final Boolean dataprovider) { + this.dataprovider = dataprovider; + } + + public Boolean getServiceprovider() { + return serviceprovider; + } + + public void setServiceprovider(final Boolean serviceprovider) { + this.serviceprovider = serviceprovider; + } + + public String getDatabaseaccesstype() { + return databaseaccesstype; + } + + public void setDatabaseaccesstype(final String databaseaccesstype) { + this.databaseaccesstype = databaseaccesstype; + } + + public String getDatauploadtype() { + return datauploadtype; + } + + public void setDatauploadtype(final String datauploadtype) { + this.datauploadtype = datauploadtype; + } + + public String getDatabaseaccessrestriction() { + return databaseaccessrestriction; + } + + public void setDatabaseaccessrestriction(final String databaseaccessrestriction) { + this.databaseaccessrestriction = databaseaccessrestriction; + } + + public String getDatauploadrestriction() { + return datauploadrestriction; + } + + public void setDatauploadrestriction(final String datauploadrestriction) { + this.datauploadrestriction = datauploadrestriction; + } + + public Boolean getVersioning() { + return versioning; + } + + public void setVersioning(final Boolean versioning) { + this.versioning = versioning; + } + + public String getCitationguidelineurl() { + return citationguidelineurl; + } + + public void setCitationguidelineurl(final String citationguidelineurl) { + this.citationguidelineurl = citationguidelineurl; + } + + public String getQualitymanagementkind() { + return qualitymanagementkind; + } + + public void setQualitymanagementkind(final String qualitymanagementkind) { + this.qualitymanagementkind = qualitymanagementkind; + } + + public String getPidsystems() { + return pidsystems; + } + + public void setPidsystems(final String pidsystems) { + this.pidsystems = pidsystems; + } + + public String getCertificates() { + return certificates; + } + + public void setCertificates(final String certificates) { + this.certificates = certificates; + } + + public Date getRegistrationdate() { + return registrationdate; + } + + public void setRegistrationdate(final Date registrationdate) { + this.registrationdate = registrationdate; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceInfo.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceInfo.java new file mode 100644 index 00000000..74c8f7cb --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceInfo.java @@ -0,0 +1,124 @@ +package eu.dnetlib.openaire.dsm.domain; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import eu.dnetlib.enabling.datasources.common.AggregationInfo; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +@JsonAutoDetect +@ApiModel(value = "Datasource info model", description = "provides information about the datasource and its aggregation status") +public class DatasourceInfo { + + @ApiModelProperty(position = 0) + private long indexRecords; + + @ApiModelProperty(position = 1) + private long fundedContent; + + @ApiModelProperty(position = 2) + private long fulltexts; + + @ApiModelProperty(position = 3) + private String lastIndexingDate; + + @ApiModelProperty(position = 4) + private String firstHarvestDate; + + @ApiModelProperty(position = 5) + private DatasourceDetails datasource; + + @ApiModelProperty(position = 6) + private AggregationInfo lastCollection; + + @ApiModelProperty(position = 7) + private AggregationInfo lastTransformation; + + @ApiModelProperty(position = 8) + private List aggregationHistory; + + public DatasourceInfo() { + super(); + } + + public DatasourceInfo setIndexRecords(final long indexRecords) { + this.indexRecords = indexRecords; + return this; + } + + public DatasourceInfo setFundedContent(final long fundedContent) { + this.fundedContent = fundedContent; + return this; + } + + public DatasourceInfo setFulltexts(final long fulltexts) { + this.fulltexts = fulltexts; + return this; + } + + public DatasourceInfo setLastIndexingDate(final String lastIndexingDate) { + this.lastIndexingDate = lastIndexingDate; + return this; + } + + public DatasourceInfo setAggregationHistory(final List aggregationHistory) { + this.aggregationHistory = aggregationHistory; + return this; + } + + public DatasourceInfo setLastCollection(final AggregationInfo lastCollection) { + this.lastCollection = lastCollection; + return this; + } + + public DatasourceInfo setLastTransformation(final AggregationInfo lastTransformation) { + this.lastTransformation = lastTransformation; + return this; + } + + public long getIndexRecords() { + return indexRecords; + } + + public long getFundedContent() { + return fundedContent; + } + + public long getFulltexts() { + return fulltexts; + } + + public String getLastIndexingDate() { + return lastIndexingDate; + } + + public List getAggregationHistory() { + return aggregationHistory; + } + + public AggregationInfo getLastCollection() { + return lastCollection; + } + + public AggregationInfo getLastTransformation() { + return lastTransformation; + } + + public DatasourceDetails getDatasource() { + return datasource; + } + + public DatasourceInfo setDatasource(final DatasourceDetails datasource) { + this.datasource = datasource; + return this; + } + + public String getFirstHarvestDate() { + return firstHarvestDate; + } + + public void setFirstHarvestDate(final String firstHarvestDate) { + this.firstHarvestDate = firstHarvestDate; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceResponse.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceResponse.java new file mode 100644 index 00000000..daae0a07 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceResponse.java @@ -0,0 +1,29 @@ +package eu.dnetlib.openaire.dsm.domain; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.google.common.collect.Lists; +import io.swagger.annotations.ApiModel; + +@ApiModel +@JsonAutoDetect +public class DatasourceResponse extends Response { + + private List datasourceInfo = Lists.newArrayList(); + + public DatasourceResponse addDatasourceInfo(final T datasourceInfo) { + getDatasourceInfo().add(datasourceInfo); + return this; + } + + public List getDatasourceInfo() { + return datasourceInfo; + } + + public DatasourceResponse setDatasourceInfo(final List datasourceInfo) { + this.datasourceInfo = datasourceInfo; + return this; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceSearchResponse.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceSearchResponse.java new file mode 100644 index 00000000..ecf9e77f --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceSearchResponse.java @@ -0,0 +1,28 @@ +package eu.dnetlib.openaire.dsm.domain; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +import java.util.List; + +@ApiModel +@JsonAutoDetect +public class DatasourceSearchResponse extends Response { + + @ApiModelProperty(position = 1) + private List datasourceInfo; + + public DatasourceSearchResponse(List datasourceInfo) { + super(); + this.datasourceInfo = datasourceInfo; + } + + public List getDatasourceInfo() { + return datasourceInfo; + } + + public void setDatasourceInfo(List datasourceInfo) { + this.datasourceInfo = datasourceInfo; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceSnippet.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceSnippet.java new file mode 100644 index 00000000..34fda827 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceSnippet.java @@ -0,0 +1,49 @@ +package eu.dnetlib.openaire.dsm.domain; + +import javax.validation.constraints.NotBlank; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +@JsonAutoDetect +@ApiModel(value = "Datasource model", description = "provides information about the datasource") +public class DatasourceSnippet { + + @NotBlank + @ApiModelProperty(position = 0) + private String id; + + @NotBlank + @ApiModelProperty(position = 2) + private String officialname; + + @NotBlank + @ApiModelProperty(position = 3) + private String englishname; + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + public String getOfficialname() { + return officialname; + } + + public void setOfficialname(final String officialname) { + this.officialname = officialname; + } + + public String getEnglishname() { + return englishname; + } + + public void setEnglishname(final String englishname) { + this.englishname = englishname; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceSnippetExtended.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceSnippetExtended.java new file mode 100644 index 00000000..902e2c01 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceSnippetExtended.java @@ -0,0 +1,170 @@ +package eu.dnetlib.openaire.dsm.domain; + +import java.util.Date; +import java.util.Set; + +import javax.validation.constraints.Email; +import javax.validation.constraints.NotBlank; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +@JsonAutoDetect +@ApiModel(value = "Datasource model", description = "provides extended information about the datasource") +public class DatasourceSnippetExtended { + + @NotBlank + @ApiModelProperty(position = 0) + private String id; + + @NotBlank + @ApiModelProperty(position = 2) + private String officialname; + + @NotBlank + @ApiModelProperty(position = 3) + private String englishname; + + @ApiModelProperty(position = 4) + private String websiteurl; + + @Email + @ApiModelProperty(position = 5) + private String registeredby; + + @ApiModelProperty(position = 6) + private Date registrationdate; + + @ApiModelProperty(position = 7) + private String typology; + + @ApiModelProperty(position = 8) + private String logoUrl; + + @ApiModelProperty(position = 9) + private String description; + + @ApiModelProperty(position = 10) + private Boolean consentTermsOfUse; + + @ApiModelProperty(position = 11) + private Date consentTermsOfUseDate; + + @ApiModelProperty(position = 12) + private Boolean fullTextDownload; + + @ApiModelProperty(position = 13) + private Set organizations; + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + public String getOfficialname() { + return officialname; + } + + public void setOfficialname(final String officialname) { + this.officialname = officialname; + } + + public String getEnglishname() { + return englishname; + } + + public void setEnglishname(final String englishname) { + this.englishname = englishname; + } + + public String getWebsiteurl() { + return websiteurl; + } + + public void setWebsiteurl(final String websiteurl) { + this.websiteurl = websiteurl; + } + + public String getRegisteredby() { + return registeredby; + } + + public void setRegisteredby(final String registeredby) { + this.registeredby = registeredby; + } + + public Date getRegistrationdate() { + return registrationdate; + } + + public void setRegistrationdate(final Date registrationdate) { + this.registrationdate = registrationdate; + } + + public String getTypology() { + return typology; + } + + public void setTypology(final String typology) { + this.typology = typology; + } + + public String getLogoUrl() { + return logoUrl; + } + + public Boolean getConsentTermsOfUse() { + return consentTermsOfUse; + } + + public DatasourceSnippetExtended setConsentTermsOfUse(final Boolean consentTermsOfUse) { + this.consentTermsOfUse = consentTermsOfUse; + return this; + } + + public Date getConsentTermsOfUseDate() { + return consentTermsOfUseDate; + } + + public DatasourceSnippetExtended setConsentTermsOfUseDate(final Date consentTermsOfUseDate) { + this.consentTermsOfUseDate = consentTermsOfUseDate; + return this; + } + + public Boolean getFullTextDownload() { + return fullTextDownload; + } + + public DatasourceSnippetExtended setFullTextDownload(final Boolean fullTextDownload) { + this.fullTextDownload = fullTextDownload; + return this; + } + + public DatasourceSnippetExtended setLogoUrl(final String logoUrl) { + this.logoUrl = logoUrl; + return this; + } + + public String getDescription() { + return description; + } + + public DatasourceSnippetExtended setDescription(final String description) { + this.description = description; + return this; + } + + public Set getOrganizations() { + return organizations; + } + + public void setOrganizations(final Set organizations) { + this.organizations = organizations; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceSnippetResponse.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceSnippetResponse.java new file mode 100644 index 00000000..a5636b65 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceSnippetResponse.java @@ -0,0 +1,28 @@ +package eu.dnetlib.openaire.dsm.domain; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +import java.util.List; + +@ApiModel +@JsonAutoDetect +public class DatasourceSnippetResponse extends Response { + + @ApiModelProperty(position = 1) + private List datasourceInfo; + + public DatasourceSnippetResponse(List datasourceInfo) { + super(); + this.datasourceInfo = datasourceInfo; + } + + public List getDatasourceInfo() { + return datasourceInfo; + } + + public void setDatasourceInfo(List datasourceInfo) { + this.datasourceInfo = datasourceInfo; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/FilterName.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/FilterName.java new file mode 100644 index 00000000..7e6cb524 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/FilterName.java @@ -0,0 +1,35 @@ +package eu.dnetlib.openaire.dsm.domain; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import io.swagger.annotations.ApiModel; + +@JsonAutoDetect +@ApiModel(value = "Filter name", description = "List of the field names used to filter datasources") +public enum +FilterName { + id, managed, collectedfrom, // exact match + officialname, englishname, websiteurl, contactemail, registeredby, typology, platform, // like match + country; // exact match on related organization + + public static FilterType type(FilterName filterName) { + switch (filterName) { + case id: + case managed: + case collectedfrom: + return FilterType.exact; + case officialname: + case englishname: + case websiteurl: + case contactemail: + case registeredby: + case typology: + case platform: + return FilterType.search; + case country: + return FilterType.searchOrgs; + default: + throw new IllegalStateException("unmapped filter type for: " + filterName); + } + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/FilterType.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/FilterType.java new file mode 100644 index 00000000..5d15adfd --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/FilterType.java @@ -0,0 +1,5 @@ +package eu.dnetlib.openaire.dsm.domain; + +public enum FilterType { + exact, search, searchOrgs +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/Header.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/Header.java new file mode 100644 index 00000000..81dc2d67 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/Header.java @@ -0,0 +1,115 @@ +package eu.dnetlib.openaire.dsm.domain; + +import java.util.List; +import java.util.Queue; +import java.util.stream.Collectors; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.google.common.collect.Lists; +import com.google.gson.GsonBuilder; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +@ApiModel +@JsonAutoDetect +public class Header { + + @ApiModelProperty(position = 0) + private long total; + + @ApiModelProperty(position = 1) + private int page; + + @ApiModelProperty(position = 2) + private int size; + + @ApiModelProperty(position = 3) + private long time; + + @ApiModelProperty(position = 4) + private int statusCode; + + @ApiModelProperty(position = 5) + private List errors = Lists.newArrayList(); + + @JsonIgnore + private Queue exceptions = Lists.newLinkedList(); + + public static Header newInsance() { + return new Header(); + } + + public Header() { + } + + public long getTime() { + return time; + } + + public Header setTime(final long time) { + this.time = time; + return this; + } + + public int getStatusCode() { + return statusCode; + } + + public Header setStatusCode(final int statusCode) { + this.statusCode = statusCode; + return this; + } + + public long getTotal() { + return total; + } + + public int getPage() { + return page; + } + + public int getSize() { + return size; + } + + public Header setPage(final int page) { + this.page = page; + return this; + } + + public Header setSize(final int size) { + this.size = size; + return this; + } + + public Header setTotal(final long total) { + this.total = total; + return this; + } + + public Queue getExceptions() { + return exceptions; + } + + public Header setExceptions(final Queue exceptions) { + this.exceptions = exceptions; + return this; + } + + public List getErrors() { + return getExceptions().stream() + .map(Throwable::getMessage) + .collect(Collectors.toList()); + } + + public Header setErrors(final List errors) { + this.errors = errors; + return this; + } + + public String toJson() { + return new GsonBuilder().setPrettyPrinting().create().toJson(this); + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/IdentitiesDetails.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/IdentitiesDetails.java new file mode 100644 index 00000000..ff389b38 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/IdentitiesDetails.java @@ -0,0 +1,32 @@ +package eu.dnetlib.openaire.dsm.domain; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import io.swagger.annotations.ApiModel; + +@ApiModel +@JsonAutoDetect +public class IdentitiesDetails { + + private String pid; + + private String issuertype; + + public String getPid() { + return pid; + } + + public String getIssuertype() { + return issuertype; + } + + public IdentitiesDetails setPid(final String pid) { + this.pid = pid; + return this; + } + + public IdentitiesDetails setIssuertype(final String issuertype) { + this.issuertype = issuertype; + return this; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/OrganizationDetails.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/OrganizationDetails.java new file mode 100644 index 00000000..79dd5a3c --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/OrganizationDetails.java @@ -0,0 +1,76 @@ +package eu.dnetlib.openaire.dsm.domain; + +import javax.validation.constraints.NotBlank; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +@JsonAutoDetect +@ApiModel(value = "Organization info model", description = "provides information about the organization") +public class OrganizationDetails extends OrganizationIgnoredProperties { + + @ApiModelProperty(position = 0) + private String legalshortname; + + @NotBlank + @ApiModelProperty(position = 1) + private String legalname; + + @ApiModelProperty(position = 2) + private String websiteurl; + + @ApiModelProperty(position = 3) + private String logourl; + + @NotBlank + @ApiModelProperty(position = 4) + private String country; + + public String getLegalshortname() { + return legalshortname; + } + + public String getLegalname() { + return legalname; + } + + public String getWebsiteurl() { + return websiteurl; + } + + public String getLogourl() { + return logourl; + } + + public String getCountry() { + return country; + } + + public OrganizationDetails setLegalshortname(final String legalshortname) { + this.legalshortname = legalshortname; + return this; + } + + public OrganizationDetails setLegalname(final String legalname) { + this.legalname = legalname; + return this; + } + + public OrganizationDetails setWebsiteurl(final String websiteurl) { + this.websiteurl = websiteurl; + return this; + } + + public OrganizationDetails setLogourl(final String logourl) { + this.logourl = logourl; + return this; + } + + public OrganizationDetails setCountry(final String country) { + this.country = country; + return this; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/OrganizationIgnoredProperties.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/OrganizationIgnoredProperties.java new file mode 100644 index 00000000..3212fb66 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/OrganizationIgnoredProperties.java @@ -0,0 +1,64 @@ +package eu.dnetlib.openaire.dsm.domain; + +import java.util.Date; +import java.util.Set; + +import com.fasterxml.jackson.annotation.JsonIgnore; + +public class OrganizationIgnoredProperties { + + @JsonIgnore + protected String id; + + @JsonIgnore + protected String collectedfrom; + + @JsonIgnore + protected Date dateofcollection; + + @JsonIgnore + protected String provenanceaction; + + @JsonIgnore + protected Set datasources; + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + public String getCollectedfrom() { + return collectedfrom; + } + + public void setCollectedfrom(final String collectedfrom) { + this.collectedfrom = collectedfrom; + } + + public Date getDateofcollection() { + return dateofcollection; + } + + public void setDateofcollection(final Date dateofcollection) { + this.dateofcollection = dateofcollection; + } + + public String getProvenanceaction() { + return provenanceaction; + } + + public void setProvenanceaction(final String provenanceaction) { + this.provenanceaction = provenanceaction; + } + + public Set getDatasources() { + return datasources; + } + + public void setDatasources(final Set datasources) { + this.datasources = datasources; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/RegisteredDatasourceInfo.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/RegisteredDatasourceInfo.java new file mode 100644 index 00000000..31e5e6b5 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/RegisteredDatasourceInfo.java @@ -0,0 +1,95 @@ +package eu.dnetlib.openaire.dsm.domain; + +public class RegisteredDatasourceInfo { + + private String id; + private String officialName; + private String englishName; + private String organization; + private String typology; + private String registeredBy; + private String registrationDate; + private String compatibility; + private String lastCollectionDate; + private long lastCollectionTotal; + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + public String getOfficialName() { + return officialName; + } + + public void setOfficialName(final String officialName) { + this.officialName = officialName; + } + + public String getEnglishName() { + return englishName; + } + + public void setEnglishName(final String englishName) { + this.englishName = englishName; + } + + public String getOrganization() { + return organization; + } + + public void setOrganization(final String organization) { + this.organization = organization; + } + + public String getTypology() { + return typology; + } + + public void setTypology(final String typology) { + this.typology = typology; + } + + public String getRegisteredBy() { + return registeredBy; + } + + public void setRegisteredBy(final String registeredBy) { + this.registeredBy = registeredBy; + } + + public String getRegistrationDate() { + return registrationDate; + } + + public void setRegistrationDate(final String registrationDate) { + this.registrationDate = registrationDate; + } + + public String getCompatibility() { + return compatibility; + } + + public void setCompatibility(final String compatibility) { + this.compatibility = compatibility; + } + + public String getLastCollectionDate() { + return lastCollectionDate; + } + + public void setLastCollectionDate(final String lastCollectionDate) { + this.lastCollectionDate = lastCollectionDate; + } + + public long getLastCollectionTotal() { + return lastCollectionTotal; + } + + public void setLastCollectionTotal(final long lastCollectionTotal) { + this.lastCollectionTotal = lastCollectionTotal; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/RequestFilter.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/RequestFilter.java new file mode 100644 index 00000000..51858e94 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/RequestFilter.java @@ -0,0 +1,14 @@ +package eu.dnetlib.openaire.dsm.domain; + +import java.util.HashMap; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import io.swagger.annotations.ApiModel; + +@JsonAutoDetect +@ApiModel(value = "Request filter", description = "field name and value pairs") +public class RequestFilter extends HashMap { + + public RequestFilter() {} + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/RequestSort.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/RequestSort.java new file mode 100644 index 00000000..bf0c0cbd --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/RequestSort.java @@ -0,0 +1,8 @@ +package eu.dnetlib.openaire.dsm.domain; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +@JsonAutoDetect +public enum RequestSort { + id, officialname, dateofvalidation, registrationdate +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/RequestSortOrder.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/RequestSortOrder.java new file mode 100644 index 00000000..2423a991 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/RequestSortOrder.java @@ -0,0 +1,8 @@ +package eu.dnetlib.openaire.dsm.domain; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +@JsonAutoDetect +public enum RequestSortOrder { + ASCENDING, DESCENDING +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/Response.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/Response.java new file mode 100644 index 00000000..d52b499a --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/Response.java @@ -0,0 +1,29 @@ +package eu.dnetlib.openaire.dsm.domain; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +@JsonAutoDetect +@ApiModel( + value = "Api response model", + description = "Api response model, provides a response header") +public class Response { + + @ApiModelProperty(position = 0) + private Header header; + + public Response() { + this.header = new Header(); + } + + public Header getHeader() { + return header; + } + + public Response setHeader(final Header header) { + this.header = header; + return this; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/SimpleResponse.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/SimpleResponse.java new file mode 100644 index 00000000..f9a1d782 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/SimpleResponse.java @@ -0,0 +1,25 @@ +package eu.dnetlib.openaire.dsm.domain; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +@ApiModel +@JsonAutoDetect +public class SimpleResponse extends Response { + + @ApiModelProperty(position = 1) + private List response; + + public List getResponse() { + return response; + } + + public SimpleResponse setResponse(final List response) { + this.response = response; + return this; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/TransformationInfo.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/TransformationInfo.java new file mode 100644 index 00000000..b4ccc38b --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/TransformationInfo.java @@ -0,0 +1,15 @@ +package eu.dnetlib.openaire.dsm.domain; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +import eu.dnetlib.enabling.datasources.common.AggregationInfo; +import io.swagger.annotations.ApiModel; + +/** + * Created by claudio on 29/11/2016. + */ +@ApiModel +@JsonAutoDetect +public class TransformationInfo extends AggregationInfo { + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/ApiDbEntry.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/ApiDbEntry.java new file mode 100755 index 00000000..0f64a3e4 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/ApiDbEntry.java @@ -0,0 +1,28 @@ +package eu.dnetlib.openaire.dsm.domain.db; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Table; + +import eu.dnetlib.enabling.datasources.common.Api; + +/** + * Api + */ +@Entity +@Table(name = "dsm_api") +public class ApiDbEntry extends Api { + + @Column(name = "compatibility_override") + protected String compatibilityOverride; + + public String getCompatibilityOverride() { + return compatibilityOverride; + } + + public Api setCompatibilityOverride(final String compatibilityOverride) { + this.compatibilityOverride = compatibilityOverride; + return this; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/ApiParamDbEntry.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/ApiParamDbEntry.java new file mode 100644 index 00000000..59a90314 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/ApiParamDbEntry.java @@ -0,0 +1,58 @@ +package eu.dnetlib.openaire.dsm.domain.db; + +import javax.persistence.EmbeddedId; +import javax.persistence.Entity; +import javax.persistence.Table; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import eu.dnetlib.enabling.datasources.common.ApiParam; +import io.swagger.annotations.ApiModel; + +/** + * Created by claudio on 13/04/2017. + */ +@Entity +@Table(name = "dsm_apiparams") +@JsonIgnoreProperties(ignoreUnknown = true) +@ApiModel(value = "Datasource Api params model", description = "describes the datasource api params") +public class ApiParamDbEntry implements ApiParam { + + @EmbeddedId + protected ApiParamKeyDbEntry id; + + protected String value; + + public ApiParamDbEntry() {} + + public ApiParam setId(final ApiParamKeyDbEntry id) { + this.id = id; + return this; + } + + @JsonIgnore + public ApiParamKeyDbEntry getId() { + return id; + } + + public String getValue() { + return value; + } + + public void setValue(final String value) { + this.value = value; + } + + @Override + public String getParam() { + return id.getParam(); + } + + @Override + public void setParam(final String param) { + final ApiParamKeyDbEntry apk = new ApiParamKeyDbEntry(); + apk.setParam(param); + setId(apk); + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/ApiParamKeyDbEntry.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/ApiParamKeyDbEntry.java new file mode 100644 index 00000000..bbb51391 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/ApiParamKeyDbEntry.java @@ -0,0 +1,22 @@ +package eu.dnetlib.openaire.dsm.domain.db; + +import javax.persistence.Embeddable; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import eu.dnetlib.enabling.datasources.common.ApiParamKey; + +/** + * Created by claudio on 13/04/2017. + */ +@Embeddable +@JsonIgnoreProperties(ignoreUnknown = true) +public class ApiParamKeyDbEntry extends ApiParamKey { + + @Override + @JsonIgnore + public ApiDbEntry getApi() { + return super.getApi(); + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/CountryTerm.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/CountryTerm.java new file mode 100644 index 00000000..87b78ed8 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/CountryTerm.java @@ -0,0 +1,48 @@ +package eu.dnetlib.openaire.dsm.domain.db; + +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Table; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +import eu.dnetlib.enabling.datasources.common.BrowseTerm; +import io.swagger.annotations.ApiModel; + +/** + * Created by claudio on 20/04/2017. + */ +@Entity +@Table(name = "browse_countries") +@ApiModel +@JsonAutoDetect +public class CountryTerm implements Comparable, BrowseTerm { + + @Id + private String term; + private long total; + + @Override + public String getTerm() { + return term; + } + + public void setTerm(final String term) { + this.term = term; + } + + @Override + public long getTotal() { + return total; + } + + public void setTotal(final long total) { + this.total = total; + } + + @Override + public int compareTo(final CountryTerm o) { + return getTerm().compareTo(o.getTerm()); + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/DatasourceApiDbEntry.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/DatasourceApiDbEntry.java new file mode 100644 index 00000000..618072c6 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/DatasourceApiDbEntry.java @@ -0,0 +1,175 @@ +package eu.dnetlib.openaire.dsm.domain.db; + +import javax.persistence.*; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonIgnore; +import io.swagger.annotations.ApiModel; + +@Entity +@JsonAutoDetect +@Table(name = "dsm_datasource_api") +@ApiModel(value = "DatasourceApi model", description = "describes a joint view between datasources and their API (1:N)") +public class DatasourceApiDbEntry { + + @Id + @JsonIgnore + private Long rowid; + + private String id; + private String officialname; + private String englishname; + private String websiteurl; + private String contactemail; + private String collectedfrom; + private String registeredby; + private String typology; + private String platform; + private Boolean managed; + + protected String protocol = null; + protected String contentdescription = null; + protected Boolean active = false; + protected Boolean removable = false; + protected String apitypology = null; + protected String compatibility; + private String baseurl; + + public DatasourceApiDbEntry() {} + + public Long getRowid() { + return rowid; + } + + public void setRowid(final Long rowid) { + this.rowid = rowid; + } + + public String getBaseurl() { + return baseurl; + } + + public void setBaseurl(final String baseurl) { + this.baseurl = baseurl; + } + + public String getContactemail() { + return contactemail; + } + + public void setContactemail(final String contactemail) { + this.contactemail = contactemail; + } + + public String getRegisteredby() { + return registeredby; + } + + public void setRegisteredby(final String registeredby) { + this.registeredby = registeredby; + } + + public String getOfficialname() { + return officialname; + } + + public void setOfficialname(final String officialname) { + this.officialname = officialname; + } + + public String getEnglishname() { + return englishname; + } + + public void setEnglishname(final String englishname) { + this.englishname = englishname; + } + + public String getWebsiteurl() { + return websiteurl; + } + + public void setWebsiteurl(final String websiteurl) { + this.websiteurl = websiteurl; + } + + public String getTypology() { + return typology; + } + + public void setTypology(final String typology) { + this.typology = typology; + } + + public String getPlatform() { + return platform; + } + + public void setPlatform(final String platform) { + this.platform = platform; + } + + public Boolean getManaged() { + return managed; + } + + public void setManaged(final Boolean managed) { + this.managed = managed; + } + + public String getProtocol() { + return protocol; + } + + public void setProtocol(final String protocol) { + this.protocol = protocol; + } + + public String getContentdescription() { + return contentdescription; + } + + public void setContentdescription(final String contentdescription) { + this.contentdescription = contentdescription; + } + + public Boolean getActive() { + return active; + } + + public void setActive(final Boolean active) { + this.active = active; + } + + public Boolean getRemovable() { + return removable; + } + + public void setRemovable(final Boolean removable) { + this.removable = removable; + } + + public String getApitypology() { + return apitypology; + } + + public void setApitypology(final String apitypology) { + this.apitypology = apitypology; + } + + public String getCompatibility() { + return compatibility; + } + + public void setCompatibility(final String compatibility) { + this.compatibility = compatibility; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } +} \ No newline at end of file diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/DatasourceDbEntry.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/DatasourceDbEntry.java new file mode 100755 index 00000000..490c7af4 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/DatasourceDbEntry.java @@ -0,0 +1,30 @@ +package eu.dnetlib.openaire.dsm.domain.db; + +import javax.persistence.Entity; +import javax.persistence.Table; +import javax.persistence.Transient; + +import eu.dnetlib.enabling.datasources.common.Datasource; +import org.hibernate.annotations.DynamicUpdate; +import org.hibernate.annotations.SelectBeforeUpdate; + +/** + * Datasource + */ +@Entity +@DynamicUpdate +@SelectBeforeUpdate +@Table(name = "dsm_datasources") +public class DatasourceDbEntry extends Datasource { + + @Transient + private String openaireId; + + public String getOpenaireId() { + return openaireId; + } + + public void setOpenaireId(final String openaireId) { + this.openaireId = openaireId; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/IdentityDbEntry.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/IdentityDbEntry.java new file mode 100644 index 00000000..032a4aa3 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/IdentityDbEntry.java @@ -0,0 +1,20 @@ +package eu.dnetlib.openaire.dsm.domain.db; + +import javax.persistence.Entity; +import javax.persistence.Table; + +import eu.dnetlib.enabling.datasources.common.Identity; +import org.hibernate.annotations.DynamicUpdate; +import org.hibernate.annotations.SelectBeforeUpdate; + +/** + * Created by claudio on 13/04/2017. + */ + +@Entity +@DynamicUpdate +@SelectBeforeUpdate +@Table(name = "dsm_identities") +public class IdentityDbEntry extends Identity { + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/OrganizationDbEntry.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/OrganizationDbEntry.java new file mode 100644 index 00000000..66e0fec8 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/OrganizationDbEntry.java @@ -0,0 +1,27 @@ +package eu.dnetlib.openaire.dsm.domain.db; + +import java.util.Set; +import javax.persistence.Entity; +import javax.persistence.Table; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import eu.dnetlib.enabling.datasources.common.Organization; +import org.hibernate.annotations.DynamicUpdate; +import org.hibernate.annotations.SelectBeforeUpdate; + +/** + * Created by claudio on 13/04/2017. + */ +@Entity +@DynamicUpdate +@SelectBeforeUpdate +@Table(name = "dsm_organizations") +public class OrganizationDbEntry extends Organization { + + @Override + @JsonIgnore + public Set getDatasources() { + return super.getDatasources(); + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderDao.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderDao.java new file mode 100644 index 00000000..31f65b4d --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderDao.java @@ -0,0 +1,45 @@ +package eu.dnetlib.openaire.funders; + +import java.util.List; +import java.util.stream.Collectors; + +import eu.dnetlib.openaire.funders.domain.ConversionUtils; +import eu.dnetlib.openaire.funders.domain.ExtendedFunderDetails; +import eu.dnetlib.openaire.funders.domain.FunderDetails; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.domain.PageRequest; +import org.springframework.stereotype.Component; + +@Component +@ConditionalOnProperty(value = "openaire.exporter.enable.funders", havingValue = "true") +public class FunderDao { + + private static final Log log = LogFactory.getLog(FunderDao.class); + + @Autowired + private FunderRepository funderRepository; + + public ExtendedFunderDetails getExtendedFunderDetails(final String funderId) throws FundersApiException { + return ConversionUtils.asExtendedFunderDetails(funderRepository.getOne(funderId)); + } + + public List listFunderDetails(final int page, final int size) throws FundersApiException { + return funderRepository.findAll(PageRequest.of(page, size)) + .getContent() + .stream() + .map(ConversionUtils::asFunderDetails) + .collect(Collectors.toList()); + } + + public List listFunderIds(final int page, final int size) throws FundersApiException { + return funderRepository.findAll(PageRequest.of(page, size)) + .getContent() + .stream() + .map(f -> f.getId()) + .collect(Collectors.toList()); + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderRepository.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderRepository.java new file mode 100644 index 00000000..39bec085 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderRepository.java @@ -0,0 +1,10 @@ +package eu.dnetlib.openaire.funders; + +import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface FunderRepository extends JpaRepository { + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiController.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiController.java new file mode 100644 index 00000000..6a3f72b8 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiController.java @@ -0,0 +1,68 @@ +package eu.dnetlib.openaire.funders; + +import java.util.List; + +import eu.dnetlib.openaire.common.AbstractExporterController; +import eu.dnetlib.openaire.funders.domain.ExtendedFunderDetails; +import eu.dnetlib.openaire.funders.domain.FunderDetails; +import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiResponse; +import io.swagger.annotations.ApiResponses; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.web.bind.annotation.*; + +@RestController +@CrossOrigin(origins = { "*" }) +@ConditionalOnProperty(value = "openaire.exporter.enable.funders", havingValue = "true") +@io.swagger.annotations.Api(tags = "OpenAIRE funders API", description = "the OpenAIRE funders API") +public class FundersApiController extends AbstractExporterController { + + private static final Log log = LogFactory.getLog(FundersApiController.class); + + @Autowired + private FunderDao fDao; + + @RequestMapping(value = "/funders", produces = { "application/json" }, method = RequestMethod.GET) + @ApiOperation( + value = "get basic information about funders", + notes = "basic information about funders: id, name, shortname, last update date, registration date", + response = FunderDetails[].class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = FunderDetails[].class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + public List getFunders( + @PathVariable final int page, + @PathVariable final int size) throws FundersApiException { + + return fDao.listFunderDetails(page, size); + } + + @RequestMapping(value = "/funder/{id}", produces = { "application/json" }, method = RequestMethod.GET) + @ApiOperation(value = "get the funder details", notes = "complete funder information", response = FunderDbEntry.class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = FunderDbEntry.class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + public ExtendedFunderDetails getFunderDetails( + @PathVariable final String id) throws FundersApiException { + + return fDao.getExtendedFunderDetails(id); + } + + @RequestMapping(value = "/funder/ids", produces = { "application/json" }, method = RequestMethod.GET) + @ApiOperation(value = "get the list of funder ids", notes = "get the list of funder ids", response = String[].class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = String[].class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + public List getFunderIds( + @PathVariable final int page, + @PathVariable final int size + ) throws FundersApiException { + + return fDao.listFunderIds(page, size); + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiException.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiException.java new file mode 100644 index 00000000..62103670 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiException.java @@ -0,0 +1,19 @@ +package eu.dnetlib.openaire.funders; + +public class FundersApiException extends Exception { + + public FundersApiException() { + } + + public FundersApiException(final String message) { + super(message); + } + + public FundersApiException(final String message, final Throwable cause) { + super(message, cause); + } + + public FundersApiException(final Throwable cause) { + super(cause); + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/ConversionUtils.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/ConversionUtils.java new file mode 100644 index 00000000..75aae566 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/ConversionUtils.java @@ -0,0 +1,88 @@ +package eu.dnetlib.openaire.funders.domain; + +import java.io.StringReader; +import java.util.stream.Collectors; + +import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry; +import eu.dnetlib.openaire.funders.domain.db.FundingPathDbEntry; +import org.dom4j.Document; +import org.dom4j.DocumentException; +import org.dom4j.io.SAXReader; + +import static org.apache.commons.lang3.StringUtils.*; + +public class ConversionUtils { + + public static final String SEPARATOR = "::"; + + public static FunderDetails asFunderDetails(final FunderDbEntry fdb) { + final FunderDetails f = new FunderDetails(); + + f.setId(fdb.getId()); + f.setName(fdb.getName()); + f.setShortname(fdb.getShortname()); + f.setJurisdiction(fdb.getJurisdiction()); + f.setRegistrationDate(fdb.getRegistrationdate()); + f.setLastUpdateDate(fdb.getLastupdatedate()); + + return f; + } + + public static ExtendedFunderDetails asExtendedFunderDetails(final FunderDbEntry fdb) { + final ExtendedFunderDetails f = new ExtendedFunderDetails(asFunderDetails(fdb)); + + if (fdb.getFundingpaths() != null) { + f.setFundingStreams( + fdb.getFundingpaths().stream() + .map(ConversionUtils::asFundingStream) + .collect(Collectors.toList())); + } + + return f; + } + + private static FundingStream asFundingStream(final FundingPathDbEntry pathDbEntry) { + final FundingStream f = new FundingStream(); + + try { + final Document xml = new SAXReader().read(new StringReader(pathDbEntry.getPath())); + + for(int i=2;i>=0;i--) { + if (hasFundingLevel(i, xml)) { + f.setId(getId(i, xml)); + f.setName(getName(i, xml)); + break; + } + } + + if (isBlank(f.getId()) && isNoneBlank(xml.valueOf("//funder/id/text()"))) { + f.setId(xml.valueOf("//funder/shortname/text()")); + f.setName(xml.valueOf("//funder/name/text()")); + } + + if (isBlank(f.getId())) { + throw new IllegalStateException("invalid funding path:\n" + xml.asXML()); + } + + return f; + + } catch (DocumentException e) { + throw new IllegalStateException("unable to parse funding path:\n" + pathDbEntry.getPath()); + } + } + + private static String getName(int level, final Document d) { + return d.valueOf(String.format("//funding_level_%s/description/text()", level)); + } + + private static String getId(int level, final Document d) { + return substringAfter( + d.valueOf( + String.format("//funding_level_%s/id/text()", level) + ), SEPARATOR); + } + + private static boolean hasFundingLevel(int level, Document d) { + return isNotBlank(getId(level, d)); + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/ExtendedFunderDetails.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/ExtendedFunderDetails.java new file mode 100644 index 00000000..d6cc54d2 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/ExtendedFunderDetails.java @@ -0,0 +1,34 @@ +package eu.dnetlib.openaire.funders.domain; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +@JsonAutoDetect +public class ExtendedFunderDetails extends FunderDetails { + + private List fundingStreams; + + public ExtendedFunderDetails() {} + + public ExtendedFunderDetails(final FunderDetails fd) { + super(); + setId(fd.getId()); + setName(fd.getName()); + setShortname(fd.getShortname()); + setJurisdiction(fd.getJurisdiction()); + setRegistrationDate(fd.getRegistrationDate()); + setLastUpdateDate(fd.getLastUpdateDate()); + } + + public List getFundingStreams() { + return fundingStreams; + } + + public ExtendedFunderDetails setFundingStreams(final List fundingStreams) { + this.fundingStreams = fundingStreams; + return this; + } + +} + diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/FunderDetails.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/FunderDetails.java new file mode 100644 index 00000000..04335b09 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/FunderDetails.java @@ -0,0 +1,76 @@ +package eu.dnetlib.openaire.funders.domain; + +import java.util.Date; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +@JsonAutoDetect +public class FunderDetails { + + private String id; + + private String name; + + private String shortname; + + private String jurisdiction; + + private Date registrationDate; + + private Date lastUpdateDate; + + public String getId() { + return id; + } + + public String getName() { + return name; + } + + public String getShortname() { + return shortname; + } + + public String getJurisdiction() { + return jurisdiction; + } + + public Date getRegistrationDate() { + return registrationDate; + } + + public Date getLastUpdateDate() { + return lastUpdateDate; + } + + public FunderDetails setId(final String id) { + this.id = id; + return this; + } + + public FunderDetails setName(final String name) { + this.name = name; + return this; + } + + public FunderDetails setShortname(final String shortname) { + this.shortname = shortname; + return this; + } + + public FunderDetails setJurisdiction(final String jurisdiction) { + this.jurisdiction = jurisdiction; + return this; + } + + public FunderDetails setRegistrationDate(final Date registrationDate) { + this.registrationDate = registrationDate; + return this; + } + + public FunderDetails setLastUpdateDate(final Date lastUpdateDate) { + this.lastUpdateDate = lastUpdateDate; + return this; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/FundingStream.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/FundingStream.java new file mode 100644 index 00000000..1efd0aa7 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/FundingStream.java @@ -0,0 +1,27 @@ +package eu.dnetlib.openaire.funders.domain; + +public class FundingStream { + + private String id; + + private String name; + + public String getId() { + return id; + } + + public String getName() { + return name; + } + + public FundingStream setId(final String id) { + this.id = id; + return this; + } + + public FundingStream setName(final String name) { + this.name = name; + return this; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java new file mode 100644 index 00000000..a47c8d51 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java @@ -0,0 +1,116 @@ +package eu.dnetlib.openaire.funders.domain.db; + +import java.sql.Date; +import java.util.Set; +import javax.persistence.*; + +import eu.dnetlib.openaire.dsm.domain.db.IdentityDbEntry; + +@Entity +@Table(name = "funders") +public class FunderDbEntry { + + @Id + private String id; + private String name; + private String shortname; + private String jurisdiction; + private String websiteurl; + private String policy; + private Date registrationdate; + private Date lastupdatedate; + + @OneToMany( + cascade = { CascadeType.PERSIST, CascadeType.MERGE }, + fetch = FetchType.LAZY ) + @JoinTable( + name = "funder_identity", + joinColumns = @JoinColumn(name = "funder"), + inverseJoinColumns = @JoinColumn(name = "pid")) + private Set pids; + + @OneToMany(mappedBy = "funderid", cascade = CascadeType.ALL, fetch = FetchType.LAZY) + private Set fundingpaths; + + public FunderDbEntry() {} + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + public String getShortname() { + return shortname; + } + + public void setShortname(final String shortname) { + this.shortname = shortname; + } + + public String getJurisdiction() { + return jurisdiction; + } + + public void setJurisdiction(final String jurisdiction) { + this.jurisdiction = jurisdiction; + } + + public String getWebsiteurl() { + return websiteurl; + } + + public void setWebsiteurl(final String websiteurl) { + this.websiteurl = websiteurl; + } + + public String getPolicy() { + return policy; + } + + public void setPolicy(final String policy) { + this.policy = policy; + } + + public Date getRegistrationdate() { + return registrationdate; + } + + public void setRegistrationdate(final Date registrationdate) { + this.registrationdate = registrationdate; + } + + public Date getLastupdatedate() { + return lastupdatedate; + } + + public void setLastupdatedate(final Date lastupdatedate) { + this.lastupdatedate = lastupdatedate; + } + + public Set getPids() { + return pids; + } + + public void setPids(final Set pids) { + this.pids = pids; + } + + public Set getFundingpaths() { + return fundingpaths; + } + + public void setFundingpaths(final Set fundingpaths) { + this.fundingpaths = fundingpaths; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FundingPathDbEntry.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FundingPathDbEntry.java new file mode 100644 index 00000000..2f7dc534 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FundingPathDbEntry.java @@ -0,0 +1,104 @@ +package eu.dnetlib.openaire.funders.domain.db; + +import javax.persistence.*; + +import com.fasterxml.jackson.annotation.JsonIgnore; + +@Entity +@Table(name = "fundingpaths") +public class FundingPathDbEntry { + + @JsonIgnore + @Transient + @Column(name = "_dnet_resource_identifier_") + private String dnetresourceidentifier; + + @Id + private String id; + private String path; + + @Column(name = "funder") + @Transient + @JsonIgnore + private String orgId; + private String jurisdiction; + private String description; + private String optional1; + private String optional2; + + private String funderid; + + public FundingPathDbEntry() {} + + public String getDnetresourceidentifier() { + return dnetresourceidentifier; + } + + public void setDnetresourceidentifier(final String dnetresourceidentifier) { + this.dnetresourceidentifier = dnetresourceidentifier; + } + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + public String getPath() { + return path; + } + + public void setPath(final String path) { + this.path = path; + } + + public String getOrgId() { + return orgId; + } + + public void setOrgId(final String orgId) { + this.orgId = orgId; + } + + public String getJurisdiction() { + return jurisdiction; + } + + public void setJurisdiction(final String jurisdiction) { + this.jurisdiction = jurisdiction; + } + + public String getDescription() { + return description; + } + + public void setDescription(final String description) { + this.description = description; + } + + public String getOptional1() { + return optional1; + } + + public void setOptional1(final String optional1) { + this.optional1 = optional1; + } + + public String getOptional2() { + return optional2; + } + + public void setOptional2(final String optional2) { + this.optional2 = optional2; + } + + public String getFunderid() { + return funderid; + } + + public void setFunderid(final String funderid) { + this.funderid = funderid; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/info/InfoController.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/info/InfoController.java new file mode 100644 index 00000000..e7fc9eb8 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/info/InfoController.java @@ -0,0 +1,80 @@ +package eu.dnetlib.openaire.info; + + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import eu.dnetlib.openaire.common.AbstractExporterController; +import eu.dnetlib.openaire.common.ExporterConstants; +import io.swagger.annotations.*; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.format.annotation.DateTimeFormat; +import org.springframework.web.bind.annotation.*; +import java.time.LocalDate; +import java.util.List; +import java.util.Map; + + +@RestController +@CrossOrigin(origins = { "*" }) +@ConditionalOnProperty(value = "openaire.exporter.enable.info", havingValue = "true") +@io.swagger.annotations.Api(tags = "OpenAIRE Info API", description = "the OpenAIRE info API") +public class InfoController extends AbstractExporterController { + + private static final Log log = LogFactory.getLog(InfoController.class); // NOPMD by marko on 11/24/08 5:02 PM + + public final static String UTF8 = "UTF-8"; + + @Autowired + private JdbcInfoDao jdbcInfoDao; + + @RequestMapping(value = "/info/{infoKey}", produces = { "application/json" }, method = RequestMethod.GET) + @ApiOperation(value = "get info date", notes = "get info date", tags = { ExporterConstants.R }, response = LocalDate.class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = LocalDate.class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + public LocalDate getDate(@PathVariable final String infoKey){ + JdbcInfoDao.DATE_INFO info = JdbcInfoDao.DATE_INFO.valueOf(infoKey); + if(info == null) throw new RuntimeException(infoKey + " not recognized"); + return jdbcInfoDao.getDate(info); + } + + @RequestMapping(value = "/info", produces = { "application/json" }, method = RequestMethod.GET) + @ApiOperation(value = "get all the info date", notes = "get all the info date", tags = { ExporterConstants.R }, response = Map.class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = LocalDate.class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + public Map listInfo(){ + Map map = Maps.newHashMap(); + for(JdbcInfoDao.DATE_INFO dateInfo : JdbcInfoDao.DATE_INFO.values()){ + map.put(dateInfo.name(), jdbcInfoDao.getDate(dateInfo)); + } + return map; + } + + @RequestMapping(value = "/info/keys", produces = { "application/json" }, method = RequestMethod.GET) + @ApiOperation(value = "get the available keys", notes = "get the available keys", tags = { ExporterConstants.R }, + response = String.class, responseContainer = "List") + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = LocalDate.class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + public List listInfoKeys(){ + List keys = Lists.newArrayList(); + for(JdbcInfoDao.DATE_INFO dateInfo : JdbcInfoDao.DATE_INFO.values()){ + keys.add(dateInfo.name()); + } + return keys; + } + + + @RequestMapping(value = "/info/dropCache", produces = { "application/json" }, method = RequestMethod.GET) + @ApiOperation(value = "Drops the info cache", notes = "Drops the info cache", tags = { ExporterConstants.R }) + public void dropCache(){ + jdbcInfoDao.dropCache(); + } + + + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/info/JdbcInfoDao.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/info/JdbcInfoDao.java new file mode 100644 index 00000000..1e3a6abc --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/info/JdbcInfoDao.java @@ -0,0 +1,32 @@ +package eu.dnetlib.openaire.info; + +import eu.dnetlib.openaire.project.dao.ValueCleaner; +import org.antlr.stringtemplate.StringTemplate; + +import java.io.IOException; +import java.io.OutputStream; +import java.sql.SQLException; +import java.time.LocalDate; +import java.util.Date; +import java.util.Map; +import java.util.zip.ZipOutputStream; + +public interface JdbcInfoDao { + + enum DATE_INFO { + claim_load_date, + oaf_load_date, + odf_load_date, + inference_date, + stats_update_date, + crossref_update_date, + orcid_update_date, + unpaywall_update_date, + mag_update_date + } + + LocalDate getDate(final DATE_INFO dateInfo); + + void dropCache(); + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/info/JdbcInfoDaoImpl.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/info/JdbcInfoDaoImpl.java new file mode 100644 index 00000000..63142558 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/info/JdbcInfoDaoImpl.java @@ -0,0 +1,111 @@ +package eu.dnetlib.openaire.info; + +import com.google.common.base.Joiner; +import com.google.common.base.Splitter; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import eu.dnetlib.OpenaireExporterConfig; +import eu.dnetlib.openaire.project.dao.JdbcApiDao; +import eu.dnetlib.openaire.project.dao.ProjectTsvRepository; +import eu.dnetlib.openaire.project.dao.ValueCleaner; +import eu.dnetlib.openaire.project.domain.Project; +import eu.dnetlib.openaire.project.domain.db.ProjectDetails; +import eu.dnetlib.openaire.project.domain.db.ProjectTsv; +import org.antlr.stringtemplate.StringTemplate; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.time.DateFormatUtils; +import org.apache.commons.lang3.time.DateUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.cache.annotation.CacheEvict; +import org.springframework.cache.annotation.Cacheable; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import javax.sql.DataSource; +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.Charset; +import java.sql.*; +import java.time.Duration; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.FormatStyle; +import java.util.Arrays; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.zip.GZIPOutputStream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +/** + * Created by alessia on 29/04/2020 + * + * Get and set info dates via JDBC. Dates are expected to be in a table named 'info' with two columns: + * key - see JdbcInfoDao.DATE_INFO enum + * value - the date (LocalDate, no time) + * + */ +@Component +@ConditionalOnProperty(value = "openaire.exporter.enable.info", havingValue = "true") +public class JdbcInfoDaoImpl implements JdbcInfoDao { + + private static final Log log = LogFactory.getLog(JdbcInfoDaoImpl.class); + + @Autowired + private OpenaireExporterConfig config; + + @Autowired + private DataSource dataSource; + + @Override + @Cacheable("info") + public LocalDate getDate(final DATE_INFO dateInfo) { + final String sql = "SELECT value FROM info WHERE key=?"; + + LocalDate date = null; + try (final Connection con = getConn(); final PreparedStatement stm = getStm(sql, con, dateInfo.name()); final ResultSet rs = getRs(stm)) { + log.info("loading info "+dateInfo+" Query: "+stm.toString()); + if(rs.next()) { + date = rs.getObject("value", LocalDate.class); + } + } catch (SQLException e) { + throw new RuntimeException(e); + } + + return date; + } + + @Override + @CacheEvict(cacheNames = { "info" }, allEntries = true) + @Scheduled(fixedDelayString = "${openaire.exporter.cache.ttl}") + public void dropCache() { + log.debug("dropped info cache"); + } + + private Connection getConn() throws SQLException { + final Connection connection = dataSource.getConnection(); + connection.setAutoCommit(false); + return connection; + } + + private PreparedStatement getStm(final String sql, final Connection con, final String param) throws SQLException { + final PreparedStatement stm = con.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY); + stm.setString(1, param); + stm.setFetchSize(config.getJdbc().getMaxRows()); + return stm; + } + + private ResultSet getRs(final PreparedStatement stm) throws SQLException { + final ResultSet rs = stm.executeQuery(); + rs.setFetchSize(config.getJdbc().getMaxRows()); + return rs; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/ProjectQueryParams.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/ProjectQueryParams.java new file mode 100644 index 00000000..16ff1d5a --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/ProjectQueryParams.java @@ -0,0 +1,86 @@ +package eu.dnetlib.openaire.project; + +import java.util.regex.Pattern; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +public class ProjectQueryParams { + + private final Pattern patternFundingStream = Pattern.compile("(\\w*(::|%| )*)*"); + + private final Pattern patternDate = Pattern.compile("\\d\\d\\d\\d-\\d\\d-\\d\\d"); + + private String fundingProgramme = null; + /** Whatever is following the fundingProgramme **/ + private String fundingPath = null; + + private String startFrom = null; + private String startUntil = null; + private String endFrom = null; + private String endUntil = null; + + private static final Log log = LogFactory.getLog(ProjectQueryParams.class); // NOPMD by marko on 11/24/08 5:02 PM + + public String getFundingProgramme() { + return fundingProgramme; + } + + public void setFundingProgramme(final String fundingProgramme) { + this.fundingProgramme = verifyParam(fundingProgramme); + } + + public String getFundingPath() { + return fundingPath; + } + + public void setFundingPath(final String fundingPath) { + this.fundingPath = verifyParam(fundingPath); + } + + public String getStartFrom() { + return startFrom; + } + + public void setStartFrom(final String startFrom) { + this.startFrom = verifyDateParam(startFrom); + } + + public String getStartUntil() { + return startUntil; + } + + public void setStartUntil(final String startUntil) { + this.startUntil = verifyDateParam(startUntil); + } + + public String getEndFrom() { + return endFrom; + } + + public void setEndFrom(final String endFrom) { + this.endFrom = verifyDateParam(endFrom); + } + + public String getEndUntil() { + return endUntil; + } + + public void setEndUntil(final String endUntil) { + this.endUntil = verifyDateParam(endUntil); + } + + protected String verifyParam(final String p) { + if ((p != null) && !patternFundingStream.matcher(p).matches()) { + throw new IllegalArgumentException(String.format("Parameter '%s' contains an invalid character", p)); + } + return p; + } + + protected String verifyDateParam(final String date) { + if ((date != null) && !patternDate.matcher(date).matches()) { + throw new IllegalArgumentException(String.format("Parameter date '%s' contains an invalid character. Accepted pattern is %s", date, patternDate.toString())); + } + return date; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/ProjectQueryParamsFactory.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/ProjectQueryParamsFactory.java new file mode 100644 index 00000000..8108b9db --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/ProjectQueryParamsFactory.java @@ -0,0 +1,47 @@ +package eu.dnetlib.openaire.project; + +import javax.servlet.http.HttpServletRequest; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +@Component +@ConditionalOnProperty(value = "openaire.exporter.enable.project", havingValue = "true") +public class ProjectQueryParamsFactory { + + private static final String BASE_PATH = "/export/"; + private static final String NO_FILTER = "ALL"; + + public ProjectQueryParams generateParams(final HttpServletRequest request, + final String startFrom, + final String startUntil, + final String endFrom, + final String endUntil) { + ProjectQueryParams params = new ProjectQueryParams(); + + String[] arr = request.getServletPath().replace(BASE_PATH, "").split("\\/"); + if (arr.length != 5) throw new IllegalArgumentException("Invalid url"); + + params.setFundingProgramme(arr[0]); + String stream = NO_FILTER.equals(arr[1]) ? null : arr[1]; + String substream = NO_FILTER.equals(arr[2]) ? null : arr[2]; + if (substream == null) { + params.setFundingPath(stream); + } else { + if (stream == null) { + stream = "%"; + } + params.setFundingPath(stream + "::" + substream); + } + // params.setSpecificProgramme(NO_FILTER.equals(arr[1]) ? null : arr[1]); + // params.setSubdivision(NO_FILTER.equals(arr[2]) ? null : arr[2]); + // NB: arr[3] should be 'projects' + // NB: arr[4] should be '[file].do' + params.setStartFrom(startFrom); + params.setStartUntil(startUntil); + params.setEndFrom(endFrom); + params.setEndUntil(endUntil); + + return params; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/ProjectsController.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/ProjectsController.java new file mode 100644 index 00000000..7c8070b3 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/ProjectsController.java @@ -0,0 +1,229 @@ +package eu.dnetlib.openaire.project; + +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.sql.SQLException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Map; +import java.util.zip.ZipOutputStream; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import com.google.common.xml.XmlEscapers; +import eu.dnetlib.OpenaireExporterConfig; +import eu.dnetlib.OpenaireExporterConfig.Project; +import eu.dnetlib.openaire.common.AbstractExporterController; +import eu.dnetlib.openaire.common.ExporterConstants; +import eu.dnetlib.openaire.project.domain.db.ProjectTsv; +import eu.dnetlib.openaire.project.domain.db.ProjectDetails; +import eu.dnetlib.openaire.project.dao.JdbcApiDao; +import eu.dnetlib.openaire.project.dao.ValueCleaner; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiResponse; +import io.swagger.annotations.ApiResponses; +import org.antlr.stringtemplate.StringTemplate; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.core.io.Resource; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.CrossOrigin; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; + +@Controller +@CrossOrigin(origins = { "*" }) +@ConditionalOnProperty(value = "openaire.exporter.enable.project", havingValue = "true") +@io.swagger.annotations.Api(tags = "OpenAIRE projects API", description = "the OpenAIRE projects API") +public class ProjectsController extends AbstractExporterController { + + private static final Log log = LogFactory.getLog(ProjectsController.class); // NOPMD by marko on 11/24/08 5:02 PM + + public final static String UTF8 = "UTF-8"; + + @Autowired + private OpenaireExporterConfig config; + + @Autowired + private JdbcApiDao dao; + + @Autowired + private ProjectQueryParamsFactory projectQueryParamsFactory; + + @RequestMapping(value = "/export/**/project/dspace.do", method = RequestMethod.GET) + @ApiOperation( + value = "DSpace", + notes = "return project information in compatible with the OpenAIRE plugin for DSpace", + tags = { ExporterConstants.DSPACE }, + response = String.class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = String.class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + public void processDspace(final HttpServletRequest request, final ServletResponse response, + @RequestParam(value = "startFrom", required = false) final String startFrom, + @RequestParam(value = "startUntil", required = false) final String startUntil, + @RequestParam(value = "endFrom", required = false) final String endFrom, + @RequestParam(value = "endUntil", required = false) final String endUntil) throws Exception { + + final Project conf = config.getProject(); + + final ProjectQueryParams params = projectQueryParamsFactory.generateParams(request, startFrom, startUntil, endFrom, endUntil); + final StringTemplate headSt = new StringTemplate(IOUtils.toString(conf.getDspaceHeadTemplate().getInputStream(), UTF8)); + + headSt.setAttribute("fundingProgramme", params.getFundingProgramme()); + + final StringTemplate tailSt = new StringTemplate(IOUtils.toString(conf.getDspaceTailTemplate().getInputStream(), UTF8)); + + response.setContentType("text/xml"); + doProcess(response, params, headSt.toString(), conf.getDspaceTemplate(), tailSt.toString(), s -> XmlEscapers.xmlContentEscaper().escape(oneLiner(s))); + } + + @RequestMapping(value = "/export/**/project/eprints.do", method = RequestMethod.GET) + @ApiOperation( + value = "EPrints", + notes = "return project information in compatible with the OpenAIRE plugin for Eprints", + tags = { ExporterConstants.EPRINT }, + response = String.class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = String.class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + public void processEprints(final HttpServletRequest request, final ServletResponse response, + @RequestParam(value = "startFrom", required = false) final String startFrom, + @RequestParam(value = "startUntil", required = false) final String startUntil, + @RequestParam(value = "endFrom", required = false) final String endFrom, + @RequestParam(value = "endUntil", required = false) final String endUntil) throws Exception { + + final ProjectQueryParams params = projectQueryParamsFactory.generateParams(request, startFrom, startUntil, endFrom, endUntil); + response.setContentType("text/html"); + doProcess(response, params, null, config.getProject().getEprintsTemplate(), null, this::oneLiner); + } + + private String oneLiner(final String s) { + return StringUtils.isNotBlank(s) ? s.replaceAll("\\n", " ").trim() : ""; + } + + private void doProcess( + final ServletResponse response, + final ProjectQueryParams params, + final String head, final Resource projectTemplate, final String tail, + final ValueCleaner cleaner) throws IOException, SQLException { + + final StringTemplate st = new StringTemplate(IOUtils.toString(projectTemplate.getInputStream(), UTF8)); + try(final OutputStream out = new BufferedOutputStream(response.getOutputStream())) { + dao.streamProjects(obtainQuery(params), out, head, st, tail, cleaner); + } + } + + @RequestMapping(value = "/noads/project2tsv.do", method = RequestMethod.GET) + @ApiOperation( + value = "TSV", + notes = "download project information in TSV format", + tags = { ExporterConstants.TSV }, + response = ProjectTsv[].class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = ProjectTsv[].class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + public void processTsv(final HttpServletResponse response, + @RequestParam(value = "funding", required = true) final String funding, + @RequestParam(value = "article293", required = false) final Boolean article293) throws Exception { + + final String fundingPrefix = getFundingPrefix(funding, null); + + final String date = new SimpleDateFormat("yyyyMMdd").format(new Date()); + final String filename = "projects_" + funding + "_" + date + ".tsv"; + response.setContentType("text/tab-separated-values"); + response.setHeader("Content-Disposition", "attachment; filename=\"" + filename + ".zip\""); + try(final ZipOutputStream out = new ZipOutputStream(new BufferedOutputStream(response.getOutputStream()))) { + dao.processTsvRequest(out, article293, fundingPrefix, filename); + } catch (Throwable e) { + throw new RuntimeException("Error processing the request", e); + } + } + + @RequestMapping(value = "/export/streamProjectDetails.do", method = RequestMethod.GET) + @ApiOperation( + value = "Stream projects", + notes = "stream project information", + tags = { ExporterConstants.STREAMING }, + response = ProjectDetails[].class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = ProjectDetails[].class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + public void streamProjectDetails(final HttpServletResponse response, + @RequestParam(value = "format", required = true) final String format, + @RequestParam(value = "compress", required = false) final Boolean compress) throws IOException, SQLException { + + if (compress != null && compress) { + response.setHeader("Content-Encoding", "gzip"); + } + switch (format) { + case "csv": + response.setContentType("text/csv"); + break; + case "json": + response.setContentType("text/plain"); + break; + default: throw new IllegalArgumentException("unsupported format: " + format); + } + + dao.processProjectDetails(response.getOutputStream(), format, compress); + } + + /** + * Creates the query on the fundingProgramme specified in the given parameters. + * + * @param params + * request parameters + * @return the query string + * @throws IllegalArgumentException + * if the funding program is not recognized + * @throws IOException + * if there are problem loading the query temlate + * @throws IllegalArgumentException + * if the funding program is not recognized + */ + protected String obtainQuery(final ProjectQueryParams params) throws IllegalArgumentException, IOException { + String funding = params.getFundingProgramme(); + String suffix = params.getFundingPath(); + + final StringTemplate st = new StringTemplate(IOUtils.toString(config.getProject().getProjectsFundingQueryTemplate().getInputStream(), UTF8)); + st.setAttribute("fundingprefix", getFundingPrefix(funding, suffix)); + String theQuery = setDateParameters(st.toString(), params); + log.debug("Generated query: " + theQuery); + return theQuery; + } + + private String getFundingPrefix(final String funding, final String suffix) { + final Map fundingIds = dao.readFundingpathIds(); + if (!fundingIds.containsKey(funding.toUpperCase())) { + throw new IllegalArgumentException("invalid funding " + funding); + } + String fundingPrefix = fundingIds.get(funding.toUpperCase()); + return StringUtils.isBlank(suffix) ? fundingPrefix : fundingPrefix + "::" + suffix.toUpperCase(); + } + + private String setDateParameters(final String query, final ProjectQueryParams params) { + String queryWithDates = query; + if (params.getStartFrom() != null) { + queryWithDates += " AND startdate >= '" + params.getStartFrom() + "'"; + } + if (params.getStartUntil() != null) { + queryWithDates += " AND startdate <= '" + params.getStartUntil() + "'"; + } + if (params.getEndFrom() != null) { + queryWithDates += " AND enddate >= '" + params.getEndFrom() + "'"; + } + if (params.getEndUntil() != null) { + queryWithDates += " AND enddate <= '" + params.getEndUntil() + "'"; + } + return queryWithDates; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/dao/JdbcApiDao.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/dao/JdbcApiDao.java new file mode 100644 index 00000000..277c5c29 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/dao/JdbcApiDao.java @@ -0,0 +1,29 @@ +package eu.dnetlib.openaire.project.dao; + +import java.io.IOException; +import java.io.OutputStream; +import java.sql.SQLException; +import java.util.Map; +import java.util.zip.ZipOutputStream; + +import org.antlr.stringtemplate.StringTemplate; + +public interface JdbcApiDao { + + Map readFundingpathIds(); + + void processProjectDetails(final OutputStream outputStream, String format, Boolean compress) throws IOException; + + void processTsvRequest(final ZipOutputStream out, final Boolean article293, final String fundingPrefix, final String filename) throws IOException; + + void streamProjects( + final String sql, + final OutputStream out, + final String head, + final StringTemplate projectTemplate, + final String tail, + final ValueCleaner cleaner) throws IOException, SQLException; + + void dropCache(); + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/dao/JdbcApiDaoImpl.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/dao/JdbcApiDaoImpl.java new file mode 100644 index 00000000..f2f2b258 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/dao/JdbcApiDaoImpl.java @@ -0,0 +1,238 @@ +package eu.dnetlib.openaire.project.dao; + +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.Charset; +import java.sql.*; +import java.time.Duration; +import java.time.LocalDateTime; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.zip.GZIPOutputStream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +import javax.sql.DataSource; + +import com.google.common.base.Joiner; +import com.google.common.base.Splitter; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import eu.dnetlib.OpenaireExporterConfig; +import eu.dnetlib.openaire.project.domain.Project; +import eu.dnetlib.openaire.project.domain.db.ProjectDetails; +import eu.dnetlib.openaire.project.domain.db.ProjectTsv; +import org.antlr.stringtemplate.StringTemplate; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.cache.annotation.CacheEvict; +import org.springframework.cache.annotation.Cacheable; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +/** + * Created by claudio on 20/09/16. + */ +@Component +@ConditionalOnProperty(value = "openaire.exporter.enable.project", havingValue = "true") +public class JdbcApiDaoImpl implements JdbcApiDao { + + public static final Charset UTF8 = Charset.forName("UTF-8"); + + private static final Log log = LogFactory.getLog(JdbcApiDaoImpl.class); + + @Autowired + private OpenaireExporterConfig config; + + @Autowired + private DataSource dataSource; + + @Autowired + private ProjectTsvRepository projectTsvRepository; + + @Override + @Cacheable("fundingpath-ids") + public Map readFundingpathIds() { + + log.debug("loading funding ids"); + final String sql = "SELECT id FROM fundingpaths"; + final Set ids = Sets.newHashSet(); + try (final Connection con = getConn(); final PreparedStatement stm = getStm(sql, con); final ResultSet rs = getRs(stm)) { + while (rs.next()) { + ids.add(rs.getString("id")); + } + } catch (SQLException e) { + throw new RuntimeException(e); + } + log.debug(String.format("loaded %s funding ids", ids.size())); + + final Map res = Maps.newHashMap(); + final Splitter sp = Splitter.on("::").trimResults(); + ids.stream() + .filter(s -> sp.splitToList(s).size() < 3) + .forEach(s -> res.put(StringUtils.substringAfterLast(s, "::").toUpperCase(), s)); + + res.put("FP7", "ec__________::EC::FP7"); + res.put("H2020", "ec__________::EC::H2020"); + log.debug(String.format("processed %s funding ids", res.size())); + res.forEach((k,v) -> log.debug(String.format("%s : '%s'", k, v))); + return res; + } + + @Override + public void processProjectDetails(final OutputStream outputStream, String format, Boolean compress) throws IOException { + final OutputStream out = getOutputStream(new BufferedOutputStream(outputStream), compress); + try { + final String sql = "SELECT * FROM project_details"; + try (final Connection con = getConn(); final PreparedStatement stm = getStm(sql, con); final ResultSet rs = getRs(stm)) { + while (rs.next()) { + try { + switch (format) { + case "csv": + out.write(getProjectDetails(rs).asCSV().getBytes(UTF8)); + break; + case "json": + out.write(getProjectDetails(rs).asJson().getBytes(UTF8)); + break; + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } catch (SQLException e) { + throw new RuntimeException(e); + } + } finally { + if (out instanceof GZIPOutputStream) { + ((GZIPOutputStream) out).finish(); + } + out.close(); + } + } + + private OutputStream getOutputStream(final OutputStream outputStream, final Boolean compress) throws IOException { + if (compress != null && compress) { + return new GZIPOutputStream(outputStream); + } + return outputStream; + } + + private ProjectDetails getProjectDetails(final ResultSet rs) throws SQLException { + return new ProjectDetails() + .setProjectId(rs.getString("projectid")) + .setAcronym(rs.getString("acronym")) + .setCode(rs.getString("code")) + .setJsonextrainfo(rs.getString("jsonextrainfo")) + .setFundingPath(asList(rs.getArray("fundingpath"))); + } + + private String[] asList(final Array value) throws SQLException { + if (value != null) { + final List list = Arrays.asList((Object[]) value.getArray()); + return list.stream() + .map(o -> o != null ? o.toString() : null) + .toArray(String[]::new); + } + return new String[0]; + } + + @Override + public void processTsvRequest(final ZipOutputStream out, final Boolean article293, final String fundingPrefix, final String filename) throws IOException { + out.putNextEntry(new ZipEntry(filename)); + writeTsvLine(out, Splitter.on(",").trimResults().splitToList(config.getProject().getTsvFields())); + queryForTsv(fundingPrefix, article293).forEach(p -> { + try { + writeTsvLine(out, p.asList()); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + + private void writeTsvLine(final ZipOutputStream out, final List s) throws IOException { + out.write(Joiner.on('\t').useForNull("").join(s).getBytes(UTF8)); + out.write('\n'); + } + + private Iterable queryForTsv(final String fundingPrefix, final Boolean article293) { + log.debug(String.format("fundingPrefix:'%s' and oa_mandate_for_datasets:'%s'", fundingPrefix, article293)); + if (article293 != null) { + return projectTsvRepository.findByFundingpathidStartingWithAndOaMandateForDatasetsOrderByAcronym(fundingPrefix, article293); + } else { + return projectTsvRepository.findByFundingpathidStartingWithOrderByAcronym(fundingPrefix); + } + } + + @Override + public void streamProjects(final String sql, final OutputStream out, + final String head, final StringTemplate projectTemplate, final String tail, + final ValueCleaner cleaner) throws IOException, SQLException { + + if (log.isDebugEnabled()) { + log.debug("Thread " + Thread.currentThread().getId() + " begin"); + } + final LocalDateTime start = LocalDateTime.now(); + + if (StringUtils.isNotBlank(head)) { + out.write(head.getBytes(UTF8)); + } + + try (final Connection con = getConn(); final PreparedStatement stm = getStm(sql, con); final ResultSet rs = getRs(stm)) { + while (rs.next()) { + final Project p = new Project() + .setFunder(cleaner.clean(rs.getString("funder"))) + .setJurisdiction(cleaner.clean(rs.getString("jurisdiction"))) + .setFundingpathid(cleaner.clean(rs.getString("fundingpathid"))) + .setAcronym(cleaner.clean(rs.getString("acronym"))) + .setTitle(cleaner.clean(rs.getString("title"))) + .setCode(cleaner.clean(rs.getString("code"))) + .setStartdate(cleaner.clean(rs.getString("startdate"))) + .setEnddate(cleaner.clean(rs.getString("enddate"))); + + projectTemplate.reset(); + projectTemplate.setAttribute("p", p); + out.write(projectTemplate.toString().getBytes(UTF8)); + } + if (StringUtils.isNotBlank(tail)) { + out.write(tail.getBytes(UTF8)); + } + final LocalDateTime end = LocalDateTime.now(); + if (log.isDebugEnabled()) { + log.debug("Thread " + Thread.currentThread().getId() + " ends, took: " + Duration.between(start, end).toMillis() + " ms"); + } + } + } + + @Override + @CacheEvict(cacheNames = { "fundingpath-ids" }, allEntries = true) + @Scheduled(fixedDelayString = "${openaire.exporter.cache.ttl}") + public void dropCache() { + log.debug("dropped fundingpath ids cache"); + } + + private Connection getConn() throws SQLException { + final Connection connection = dataSource.getConnection(); + connection.setAutoCommit(false); + return connection; + } + + private PreparedStatement getStm(final String sql, final Connection con) throws SQLException { + final PreparedStatement stm = con.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY); + stm.setFetchSize(config.getJdbc().getMaxRows()); + + return stm; + } + + private ResultSet getRs(final PreparedStatement stm) throws SQLException { + final ResultSet rs = stm.executeQuery(); + rs.setFetchSize(config.getJdbc().getMaxRows()); + return rs; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/dao/ProjectApiRepository.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/dao/ProjectApiRepository.java new file mode 100644 index 00000000..a3e6f733 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/dao/ProjectApiRepository.java @@ -0,0 +1,18 @@ +package eu.dnetlib.openaire.project.dao; + +import eu.dnetlib.openaire.project.domain.db.ProjectApi; +import eu.dnetlib.openaire.project.domain.db.ProjectTsv; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.repository.PagingAndSortingRepository; +import org.springframework.stereotype.Repository; + +/** + * Created by claudio on 06/07/2017. + */ +@Repository +@ConditionalOnProperty(value = "openaire.exporter.enable.project", havingValue = "true") +public interface ProjectApiRepository extends PagingAndSortingRepository { + + Iterable findByFundingpathidStartingWith(String fundingpathid); + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/dao/ProjectTsvRepository.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/dao/ProjectTsvRepository.java new file mode 100644 index 00000000..f8328735 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/dao/ProjectTsvRepository.java @@ -0,0 +1,19 @@ +package eu.dnetlib.openaire.project.dao; + +import eu.dnetlib.openaire.project.domain.db.ProjectTsv; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.repository.PagingAndSortingRepository; +import org.springframework.stereotype.Repository; + +/** + * Created by claudio on 04/07/2017. + */ +@Repository +@ConditionalOnProperty(value = "openaire.exporter.enable.project", havingValue = "true") +public interface ProjectTsvRepository extends PagingAndSortingRepository { + + Iterable findByFundingpathidStartingWithOrderByAcronym(String fundingpathid); + + Iterable findByFundingpathidStartingWithAndOaMandateForDatasetsOrderByAcronym(String fundingpathid, boolean article293); + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/dao/ValueCleaner.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/dao/ValueCleaner.java new file mode 100644 index 00000000..db84257c --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/dao/ValueCleaner.java @@ -0,0 +1,10 @@ +package eu.dnetlib.openaire.project.dao; + +/** + * Created by claudio on 23/09/2016. + */ +public interface ValueCleaner { + + String clean(String s); + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/domain/Project.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/domain/Project.java new file mode 100644 index 00000000..509351ff --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/domain/Project.java @@ -0,0 +1,264 @@ +package eu.dnetlib.openaire.project.domain; + +import java.util.ArrayList; +import java.util.List; + +import com.google.common.base.Splitter; +import com.google.common.collect.Lists; +import org.apache.commons.lang3.StringUtils; + +/** + * Created by claudio on 20/09/16. + */ +public class Project { + + public static final String INFO_EU_REPO_GRANT_AGREEMENT = "info:eu-repo/grantAgreement/"; + private String code; + private String acronym; + private String title; + private String callIdentifier; + private String startdate; + private String enddate; + private boolean oaMandateForPublications; + private boolean oaMandateForDatasets; + private String fundingpathid; + private String description; + private String funder; + private String jurisdiction; + private String orgLegalname; + private String orgCountry; + private String orgRole; + private String firstname; + private String secondnames; + private String email; + + public Project() { + } + + public String getIdnamespace() { + String res = INFO_EU_REPO_GRANT_AGREEMENT + getFunder()+"/"; + final String fundingProgram = asFundingProgram(getFundingpathid()); + if (StringUtils.isNotBlank(fundingProgram)) { + res += fundingProgram; + } + res += "/" + escapeCode(getCode()); + if (StringUtils.isNotBlank(getJurisdiction())) { + res += "/" + getJurisdiction(); + } + return res; + } + + public String getListLabel() { + return String.format("for:value:component:_%s_project_id", asFunder(getFunder())); + } + + private String asFunder(final String legalshortname) { + switch (legalshortname.toLowerCase()) { + case "ec": + return asFundingProgram(getFundingpathid()).toLowerCase(); + default: + return legalshortname.toLowerCase(); + } + } + + public List asList() { + return Lists.newArrayList( + clean(getCode()), + clean(getAcronym()), + clean(getTitle()), + clean(getCallIdentifier()), + clean(getStartdate()), + clean(getEnddate()), + String.valueOf(isOaMandateForPublications()), + String.valueOf(isOaMandateForDatasets()), + clean(getDescription()), + clean(getOrgLegalname()), + clean(getOrgCountry()), + clean(getOrgRole()), + clean(getFirstname()), + clean(getSecondnames()), + clean(getEmail())); + } + + private String clean(final String s) { + return StringUtils.isNotBlank(s) ? "\"" + s.replaceAll("\\n|\\t|\\s+", " ").replace("\"","\"\"").trim() + "\"" : ""; + } + + private String escapeCode(final String code) { + return replaceSlash(code); + } + + private String asFundingProgram(final String fundingpathid) { + final ArrayList strings = Lists.newArrayList(Splitter.on("::").split(fundingpathid)); + if(strings.size() <= 1) throw new IllegalStateException("Unexpected funding id: "+fundingpathid); + if(strings.size() == 2) return ""; + else return replaceSlash(strings.get(2)); + } + + private String replaceSlash(final String s) { + return s.replaceAll("/", "%2F"); + } + + public String getCode() { + return code; + } + + public Project setCode(final String code) { + this.code = code; + return this; + } + + public String getAcronym() { + return acronym; + } + + public Project setAcronym(final String acronym) { + this.acronym = acronym; + return this; + } + + public String getTitle() { + return title; + } + + public Project setTitle(final String title) { + this.title = title; + return this; + } + + public String getCallIdentifier() { + return callIdentifier; + } + + public Project setCallIdentifier(final String call_identifier) { + this.callIdentifier = call_identifier; + return this; + } + + public String getStartdate() { + return startdate; + } + + public Project setStartdate(final String startdate) { + this.startdate = startdate; + return this; + } + + public String getEnddate() { + return enddate; + } + + public Project setEnddate(final String enddate) { + this.enddate = enddate; + return this; + } + + public boolean isOaMandateForPublications() { + return oaMandateForPublications; + } + + public Project setOaMandateForPublications(final boolean oaMandateForPublications) { + this.oaMandateForPublications = oaMandateForPublications; + return this; + } + + public boolean isOaMandateForDatasets() { + return oaMandateForDatasets; + } + + public Project setOaMandateForDatasets(final boolean oaMandateForDatasets) { + this.oaMandateForDatasets = oaMandateForDatasets; + return this; + } + + + public String getFundingpathid() { + return fundingpathid; + } + + public Project setFundingpathid(final String fundingpathid) { + this.fundingpathid = fundingpathid; + return this; + } + + public String getDescription() { + return description; + } + + public Project setDescription(final String description) { + this.description = description; + return this; + } + + public String getJurisdiction() { + return jurisdiction; + } + + public Project setJurisdiction(final String jurisdiction) { + this.jurisdiction = jurisdiction; + return this; + } + + public String getOrgLegalname() { + return orgLegalname; + } + + public Project setOrgLegalname(final String legalname) { + this.orgLegalname = legalname; + return this; + } + + public String getOrgCountry() { + return orgCountry; + } + + public Project setOrgCountry(final String country) { + this.orgCountry = country; + return this; + } + + public String getOrgRole() { + return orgRole; + } + + public Project setOrgRole(final String role) { + this.orgRole = role; + return this; + } + + public String getFirstname() { + return firstname; + } + + public Project setFirstname(final String firstname) { + this.firstname = firstname; + return this; + } + + public String getSecondnames() { + return secondnames; + } + + public Project setSecondnames(final String secondnames) { + this.secondnames = secondnames; + return this; + } + + public String getEmail() { + return email; + } + + public Project setEmail(final String email) { + this.email = email; + return this; + } + + public String getFunder() { + return funder; + } + + public Project setFunder(final String funder) { + this.funder = funder; + return this; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/domain/db/ProjectApi.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/domain/db/ProjectApi.java new file mode 100644 index 00000000..361f6534 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/domain/db/ProjectApi.java @@ -0,0 +1,152 @@ +package eu.dnetlib.openaire.project.domain.db; + +import java.sql.Date; +import java.util.ArrayList; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Table; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.google.common.base.Splitter; +import com.google.common.collect.Lists; +import io.swagger.annotations.ApiModel; +import org.apache.commons.lang3.StringUtils; + +/** + * Created by claudio on 20/09/16. + */ +@Entity +@Table(name = "projects_api") +@ApiModel(value = "Project api model", description = "Project api model used by DSpace and Eprints exporter") +public class ProjectApi { + + public static final String INFO_EU_REPO_GRANT_AGREEMENT = "info:eu-repo/grantAgreement/"; + + @Id + @JsonIgnore + private String id; + + private String code; + private String acronym; + private String title; + private String funder; + private String jurisdiction; + private Date startdate; + private Date enddate; + private String fundingpathid; + + public ProjectApi() { } + + public String getIdnamespace() { + String res = INFO_EU_REPO_GRANT_AGREEMENT + getFunder()+"/"; + final String fundingProgram = asFundingProgram(getFundingpathid()); + if (StringUtils.isNotBlank(fundingProgram)) { + res += fundingProgram; + } + res += "/" + escapeCode(getCode()); + if (StringUtils.isNotBlank(getJurisdiction())) { + res += "/" + getJurisdiction(); + } + return res; + } + + public String getListLabel() { + return String.format("for:value:component:_%s_project_id", asFunder(getFunder())); + } + + private String asFunder(final String legalshortname) { + switch (legalshortname.toLowerCase()) { + case "ec": + return asFundingProgram(getFundingpathid()).toLowerCase(); + default: + return legalshortname.toLowerCase(); + } + } + + private String escapeCode(final String code) { + return replaceSlash(code); + } + + private String asFundingProgram(final String fundingpathid) { + final ArrayList strings = Lists.newArrayList(Splitter.on("::").split(fundingpathid)); + if(strings.size() <= 1) throw new IllegalStateException("Unexpected funding id: "+fundingpathid); + if(strings.size() == 2) return ""; + else return replaceSlash(strings.get(2)); + } + + private String replaceSlash(final String s) { + return s.replaceAll("/", "%2F"); + } + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + public String getCode() { + return code; + } + + public void setCode(final String code) { + this.code = code; + } + + public String getAcronym() { + return acronym; + } + + public void setAcronym(final String acronym) { + this.acronym = acronym; + } + + public String getTitle() { + return title; + } + + public void setTitle(final String title) { + this.title = title; + } + + public String getFunder() { + return funder; + } + + public void setFunder(final String funder) { + this.funder = funder; + } + + public String getJurisdiction() { + return jurisdiction; + } + + public void setJurisdiction(final String jurisdiction) { + this.jurisdiction = jurisdiction; + } + + public Date getStartdate() { + return startdate; + } + + public void setStartdate(final Date startdate) { + this.startdate = startdate; + } + + public Date getEnddate() { + return enddate; + } + + public void setEnddate(final Date enddate) { + this.enddate = enddate; + } + + public String getFundingpathid() { + return fundingpathid; + } + + public void setFundingpathid(final String fundingpathid) { + this.fundingpathid = fundingpathid; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/domain/db/ProjectDetails.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/domain/db/ProjectDetails.java new file mode 100644 index 00000000..e4bb8ad3 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/domain/db/ProjectDetails.java @@ -0,0 +1,157 @@ +package eu.dnetlib.openaire.project.domain.db; + +import java.io.IOException; +import java.io.StringReader; +import java.io.StringWriter; +import java.lang.reflect.Field; +import java.util.Arrays; +import java.util.List; +import javax.persistence.*; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.google.gson.Gson; +import io.swagger.annotations.ApiModel; +import org.hibernate.annotations.Type; +import org.supercsv.cellprocessor.Optional; +import org.supercsv.cellprocessor.ift.CellProcessor; +import org.supercsv.cellprocessor.ift.StringCellProcessor; +import org.supercsv.io.CsvBeanReader; +import org.supercsv.io.CsvBeanWriter; +import org.supercsv.io.ICsvBeanReader; +import org.supercsv.io.ICsvBeanWriter; +import org.supercsv.prefs.CsvPreference; +import org.supercsv.util.CsvContext; + +/** + * Created by claudio on 04/07/2017. + */ +@Entity +@Table(name = "project_details") +@ApiModel(value = "Project details model", description = "provides project details") +public class ProjectDetails { + + @Transient + @JsonIgnore + private transient static final String[] FIELDS = Arrays.stream(ProjectDetails.class.getDeclaredFields()) + .map(Field::getName) + .filter(s -> !s.equals("FIELDS")) + .filter(s -> !s.startsWith("optional")) + .toArray(String[]::new); + + @Id + @Column(name = "projectid") + private String projectId; + private String acronym; + private String code; + + private String jsonextrainfo; + + @Type(type = "eu.dnetlib.openaire.common.GenericArrayUserType") + @Column(name = "fundingpath", columnDefinition = "text[]") + private String[] fundingPath; + + public ProjectDetails() { + } + + public String getProjectId() { + return projectId; + } + + public String getAcronym() { + return acronym; + } + + public String getCode() { + return code; + } + + public String getJsonextrainfo() { + return jsonextrainfo; + } + + public String[] getFundingPath() { + return fundingPath; + } + + public void setFundingPath(final List fundingPath) { + if(fundingPath != null && !fundingPath.isEmpty()) { + this.fundingPath = fundingPath.toArray(new String[fundingPath.size()]); + } + } + + public static ProjectDetails fromJson(final String json) { + return new Gson().fromJson(json, ProjectDetails.class); + } + + public static ProjectDetails fromCSV(final String csv) throws IOException { + try (ICsvBeanReader beanReader = new CsvBeanReader(new StringReader(csv), CsvPreference.STANDARD_PREFERENCE)) { + return beanReader.read(ProjectDetails.class, FIELDS, getProcessors(new StringCellProcessor() { + @Override + public Object execute(final Object value, final CsvContext context) { + return new Gson().fromJson(value.toString(), List.class); + } + })); + } + } + + /** + * Sets up the processors used for the examples. There are 10 CSV columns, so 10 processors are defined. Empty + * columns are read as null (hence the NotNull() for mandatory columns). + * + * @return the cell processors + */ + private static CellProcessor[] getProcessors(final CellProcessor fundingPathProcessor) { + return new CellProcessor[] { + new Optional(), // projectId + new Optional(), // acronym + new Optional(), // code + new Optional(), // jsonextrainfo + fundingPathProcessor + }; + } + + public String asJson() { + return new Gson().toJson(this) + '\n'; + } + + public String asCSV() throws IOException { + final StringWriter sb = new StringWriter(); + try (ICsvBeanWriter beanWriter = new CsvBeanWriter(sb, CsvPreference.STANDARD_PREFERENCE)) { + beanWriter.write(this, FIELDS, getProcessors(new StringCellProcessor() { + @Override + public Object execute(final Object value, final CsvContext context) { + return new Gson().toJson(value); + } + })); + beanWriter.flush(); + } + + return sb.toString(); + } + + public ProjectDetails setProjectId(final String projectId) { + this.projectId = projectId; + return this; + } + + public ProjectDetails setAcronym(final String acronym) { + this.acronym = acronym; + return this; + } + + public ProjectDetails setCode(final String code) { + this.code = code; + return this; + } + + public ProjectDetails setJsonextrainfo(final String jsonextrainfo) { + this.jsonextrainfo = jsonextrainfo; + return this; + } + + public ProjectDetails setFundingPath(final String[] fundingPath) { + this.fundingPath = fundingPath; + return this; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/domain/db/ProjectTsv.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/domain/db/ProjectTsv.java new file mode 100644 index 00000000..fdf7bd05 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/domain/db/ProjectTsv.java @@ -0,0 +1,210 @@ +package eu.dnetlib.openaire.project.domain.db; + +import java.sql.Date; +import java.util.List; +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Table; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.google.common.collect.Lists; +import io.swagger.annotations.ApiModel; +import org.apache.commons.lang3.StringUtils; + +/** + * Created by claudio on 05/07/2017. + */ +@Entity +@Table(name = "projects_tsv") +@ApiModel(value = "Project TSV model", description = "project TSV model description") +public class ProjectTsv { + + @Id + @JsonIgnore + private long rowid; + private String code; + private String acronym; + private String title; + @Column(name = "call_identifier") + private String callIdentifier; + private Date startdate; + private Date enddate; + @Column(name = "ec_sc39") + private Boolean ecSc39; + @Column(name = "oa_mandate_for_publications") + private Boolean oaMandateForPublications; + @Column(name = "oa_mandate_for_datasets") + private Boolean oaMandateForDatasets; + @JsonIgnore + private String fundingpathid; + private String description; + @Column(name = "legalname") + private String orgLegalname; + @Column(name = "country") + private String orgCountry; + @Column(name = "role") + private String orgRole; + private String contactfullname; + private String contactemail; + + public ProjectTsv() {} + + public List asList() { + return Lists.newArrayList( + clean(getCode()), + clean(getAcronym()), + clean(getTitle()), + clean(getCallIdentifier()), + clean(getStartdate() != null ? getStartdate().toString() : ""), + clean(getEnddate() != null ? getEnddate().toString() : ""), + clean(String.valueOf(isOaMandateForPublications())), + clean(String.valueOf(isOaMandateForDatasets())), + clean(getDescription()), + clean(getOrgLegalname()), + clean(getOrgCountry()), + clean(getOrgRole()), + clean(getContactfullname()), + clean(getContactemail())); + } + + private String clean(final String s) { + return StringUtils.isNotBlank(s) ? "\"" + s.replaceAll("\\n|\\t|\\s+", " ").replace("\"","\"\"").trim() + "\"" : ""; + } + + public long getRowid() { + return rowid; + } + + public void setRowid(final long rowid) { + this.rowid = rowid; + } + + public String getCode() { + return code; + } + + public void setCode(final String code) { + this.code = code; + } + + public String getAcronym() { + return acronym; + } + + public void setAcronym(final String acronym) { + this.acronym = acronym; + } + + public String getTitle() { + return title; + } + + public void setTitle(final String title) { + this.title = title; + } + + public String getCallIdentifier() { + return callIdentifier; + } + + public void setCallIdentifier(final String callIdentifier) { + this.callIdentifier = callIdentifier; + } + + public Date getStartdate() { + return startdate; + } + + public void setStartdate(final Date startdate) { + this.startdate = startdate; + } + + public Date getEnddate() { + return enddate; + } + + public void setEnddate(final Date enddate) { + this.enddate = enddate; + } + + public Boolean isEcSc39() { + return ecSc39; + } + + public void setEcSc39(final Boolean ecSc39) { + this.ecSc39 = ecSc39; + } + + public Boolean isOaMandateForPublications() { + return oaMandateForPublications; + } + + public void setOaMandateForPublications(final Boolean oaMandateForPublications) { + this.oaMandateForPublications = oaMandateForPublications; + } + + public Boolean isOaMandateForDatasets() { + return oaMandateForDatasets; + } + + public void setOaMandateForDatasets(final Boolean oaMandateForDatasets) { + this.oaMandateForDatasets = oaMandateForDatasets; + } + + public String getFundingpathid() { + return fundingpathid; + } + + public void setFundingpathid(final String fundingpathid) { + this.fundingpathid = fundingpathid; + } + + public String getDescription() { + return description; + } + + public void setDescription(final String description) { + this.description = description; + } + + public String getOrgLegalname() { + return orgLegalname; + } + + public void setOrgLegalname(final String orgLegalname) { + this.orgLegalname = orgLegalname; + } + + public String getOrgCountry() { + return orgCountry; + } + + public void setOrgCountry(final String orgCountry) { + this.orgCountry = orgCountry; + } + + public String getOrgRole() { + return orgRole; + } + + public void setOrgRole(final String orgRole) { + this.orgRole = orgRole; + } + + public String getContactfullname() { + return contactfullname; + } + + public void setContactfullname(final String contactfullname) { + this.contactfullname = contactfullname; + } + + public String getContactemail() { + return contactemail; + } + + public void setContactemail(final String contactemail) { + this.contactemail = contactemail; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/vocabularies/Country.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/vocabularies/Country.java new file mode 100644 index 00000000..73c54763 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/vocabularies/Country.java @@ -0,0 +1,35 @@ +package eu.dnetlib.openaire.vocabularies; + +/** + * Created by claudio on 15/09/2017. + */ +public class Country { + + private String code; + + private String name; + + public Country() { + } + + public Country(final String code, final String name) { + this.code = code; + this.name = name; + } + + public String getCode() { + return code; + } + + public void setCode(final String code) { + this.code = code; + } + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/vocabularies/Term.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/vocabularies/Term.java new file mode 100644 index 00000000..30adc410 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/vocabularies/Term.java @@ -0,0 +1,29 @@ +package eu.dnetlib.openaire.vocabularies; + +/** + * Created by claudio on 15/09/2017. + */ +public class Term { + private String englishName; + private String code; + + public Term() { + } + + public String getEnglishName() { + return englishName; + } + + public void setEnglishName(final String englishName) { + this.englishName = englishName; + } + + public String getCode() { + return code; + } + + public void setCode(final String code) { + this.code = code; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/vocabularies/Vocabulary.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/vocabularies/Vocabulary.java new file mode 100644 index 00000000..54c6e090 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/vocabularies/Vocabulary.java @@ -0,0 +1,50 @@ +package eu.dnetlib.openaire.vocabularies; + +import java.util.List; +import java.util.Map; + +import com.google.common.collect.Maps; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** + * Created by claudio on 15/09/2017. + */ +public class Vocabulary { + + + private static final Log log = LogFactory.getLog(Vocabulary.class); + + private List terms; + + public Vocabulary() { + } + + private static final Map byCode = Maps.newConcurrentMap(); + + public String getEnglishName(final String code) { + if (byCode.isEmpty()) { + hashByCode(); + } + final Term term = byCode.get(code); + return term != null ? term.getEnglishName() : null; + } + + public boolean hasCode(final String code) { + return getEnglishName(code) != null; + } + + private void hashByCode() { + log.info("hashing vocabulary by code ..."); + getTerms().forEach(term -> byCode.put(term.getCode(), term)); + log.info("hashing vocabulary by code ... done!"); + } + + public List getTerms() { + return terms; + } + + public void setTerms(final List terms) { + this.terms = terms; + } +} diff --git a/apps/dnet-exporter-api/src/main/resources/application.properties b/apps/dnet-exporter-api/src/main/resources/application.properties new file mode 100644 index 00000000..9c4e8b67 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/application.properties @@ -0,0 +1,34 @@ +# COMMON +server.servlet.context-path = /openaire +server.port = 8080 + +spring.datasource.driverClassName = org.postgresql.Driver +spring.jpa.database-platform = org.hibernate.dialect.PostgreSQL9Dialect +spring.jpa.show-sql = false +spring.jpa.properties.hibernate.format_sql = true +spring.jpa.hibernate.ddl-auto = validate +#spring.mvc.dispatch-options-request = true + +# SWAGGER +spring.jackson.date-format = eu.dnetlib.openaire.common.RFC3339DateFormat +spring.jackson.serialization.WRITE_DATES_AS_TIMESTAMPS = false + +management.endpoints.web.exposure.include = prometheus,health + +management.endpoints.web.base-path = / +management.endpoints.web.path-mapping.prometheus = metrics +management.endpoints.web.path-mapping.health = health + + +# ENABLE / DISABLE CONTROLLERS +openaire.exporter.enable.dsm = true +openaire.exporter.enable.community = true +openaire.exporter.enable.context = true +openaire.exporter.enable.funders = false +openaire.exporter.enable.project = true +openaire.exporter.enable.info = true + +# CACHE TTL, 12h +openaire.exporter.cache.ttl = 43200000 + + diff --git a/apps/dnet-exporter-api/src/main/resources/dnet_dsm.data.sql b/apps/dnet-exporter-api/src/main/resources/dnet_dsm.data.sql new file mode 100644 index 00000000..ee3ec218 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/dnet_dsm.data.sql @@ -0,0 +1,176 @@ +INSERT INTO dsm_identities(pid, issuertype) VALUES ('77', 'roar'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('680', 'roar'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('260', 'roar'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('637', 'roar'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('528', 'roar'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('577', 'roar'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('566', 'roar'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('241', 'roar'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('491', 'roar'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('324', 'roar'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('375', 'roar'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('152', 'roar'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('1025', 'roar'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('10.13039/100004440', 'doi'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('10.13039/100000001', 'doi'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('10.13039/100000002', 'doi'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('10.13039/501100007601', 'doi'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('10.13039/501100001871', 'doi'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('10.13039/501100008982', 'doi'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('10.13039/501100000925', 'doi'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('10.13039/501100000923', 'doi'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('10.13039/501100001602', 'doi'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('10.13039/501100002428', 'doi'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('10.13039/501100000690', 'doi'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('10.13039/501100002341', 'doi'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 2242 8989', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0004 5900 900X', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 2169 9189', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 2181 2823', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0004 0427 7672', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 1958 7073', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 2169 1945', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 2154 0709', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 2176 1982', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 0789 9694', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 2228 3249', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 2176 7727', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0004 0611 9213', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0004 0609 4140', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0004 4663 8325', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 1092 7772', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0004 0452 5752', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 2297 5165', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 0672 3101', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 1957 0992', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 1091 8438', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 0507 0997', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 0665 7300', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 0685 2712', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0004 0647 6886', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 0943 9683', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 2096 9829', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 1957 9997', 'isni'); + +INSERT INTO funder_identity(funder, pid) VALUES ('H2020', '680'); +INSERT INTO funder_identity(funder, pid) VALUES ('H2020', '10.13039/501100007601'); +INSERT INTO funder_identity(funder, pid) VALUES ('H2020', '0000 0001 2242 8989'); +INSERT INTO funder_identity(funder, pid) VALUES ('FP7', '0000 0004 5900 900X'); +INSERT INTO funder_identity(funder, pid) VALUES ('FCT', '260'); +INSERT INTO funder_identity(funder, pid) VALUES ('FCT', '10.13039/501100001871'); +INSERT INTO funder_identity(funder, pid) VALUES ('FCT', '0000 0001 2169 9189'); +INSERT INTO funder_identity(funder, pid) VALUES ('FCT', '0000 0001 2181 2823'); +INSERT INTO funder_identity(funder, pid) VALUES ('WT', '637'); +INSERT INTO funder_identity(funder, pid) VALUES ('WT', '10.13039/100004440'); +INSERT INTO funder_identity(funder, pid) VALUES ('WT', '0000 0004 0427 7672'); +INSERT INTO funder_identity(funder, pid) VALUES ('NSF', '528'); +INSERT INTO funder_identity(funder, pid) VALUES ('NSF', '10.13039/501100008982'); +INSERT INTO funder_identity(funder, pid) VALUES ('NSF', '10.13039/100000001'); +INSERT INTO funder_identity(funder, pid) VALUES ('NSF', '0000 0001 1958 7073'); +INSERT INTO funder_identity(funder, pid) VALUES ('NSF', '0000 0001 2169 1945'); +INSERT INTO funder_identity(funder, pid) VALUES ('NSF', '0000 0001 2154 0709'); +INSERT INTO funder_identity(funder, pid) VALUES ('NHMRC', '577'); +INSERT INTO funder_identity(funder, pid) VALUES ('NHMRC', '10.13039/501100000925'); +INSERT INTO funder_identity(funder, pid) VALUES ('NHMRC', '0000 0001 2176 1982'); +INSERT INTO funder_identity(funder, pid) VALUES ('NHMRC', '0000 0001 0789 9694'); +INSERT INTO funder_identity(funder, pid) VALUES ('NHMRC', '0000 0001 2228 3249'); +INSERT INTO funder_identity(funder, pid) VALUES ('NHMRC', '0000 0001 2176 7727'); +INSERT INTO funder_identity(funder, pid) VALUES ('NHMRC', '0000 0001 0789 9694'); +INSERT INTO funder_identity(funder, pid) VALUES ('NHMRC', '0000 0001 2228 3249'); +INSERT INTO funder_identity(funder, pid) VALUES ('ARC', '566'); +INSERT INTO funder_identity(funder, pid) VALUES ('ARC', '10.13039/501100000923'); +INSERT INTO funder_identity(funder, pid) VALUES ('ARC', '0000 0004 0611 9213'); +INSERT INTO funder_identity(funder, pid) VALUES ('MSES', '0000 0004 0609 4140'); +INSERT INTO funder_identity(funder, pid) VALUES ('CSF', '0000 0004 4663 8325'); +INSERT INTO funder_identity(funder, pid) VALUES ('NWO', '241'); +INSERT INTO funder_identity(funder, pid) VALUES ('NWO', '0000 0001 1092 7772'); +INSERT INTO funder_identity(funder, pid) VALUES ('SFI', '10.13039/501100001602'); +INSERT INTO funder_identity(funder, pid) VALUES ('SFI', '0000 0004 0452 5752'); +INSERT INTO funder_identity(funder, pid) VALUES ('NIH', '10.13039/100000002'); +INSERT INTO funder_identity(funder, pid) VALUES ('NIH', '0000 0001 2297 5165'); +INSERT INTO funder_identity(funder, pid) VALUES ('NIH', '491'); +INSERT INTO funder_identity(funder, pid) VALUES ('SNSF', '324'); +INSERT INTO funder_identity(funder, pid) VALUES ('SNSF', '0000 0001 0672 3101'); +INSERT INTO funder_identity(funder, pid) VALUES ('SNSF', '0000 0001 1957 0992'); +INSERT INTO funder_identity(funder, pid) VALUES ('FWF', '77'); +INSERT INTO funder_identity(funder, pid) VALUES ('FWF', '10.13039/501100002428'); +INSERT INTO funder_identity(funder, pid) VALUES ('FWF', '0000 0001 1091 8438'); +INSERT INTO funder_identity(funder, pid) VALUES ('RCUK', '375'); +INSERT INTO funder_identity(funder, pid) VALUES ('RCUK', '10.13039/501100000690'); +INSERT INTO funder_identity(funder, pid) VALUES ('RCUK', '0000 0001 0507 0997'); +INSERT INTO funder_identity(funder, pid) VALUES ('TBT', '0000 0001 0665 7300'); +INSERT INTO funder_identity(funder, pid) VALUES ('TBT', '0000 0001 0685 2712'); +INSERT INTO funder_identity(funder, pid) VALUES ('AFF', '1025'); +INSERT INTO funder_identity(funder, pid) VALUES ('AFF', '10.13039/501100002341'); +INSERT INTO funder_identity(funder, pid) VALUES ('AFF', '0000 0004 0647 6886'); +INSERT INTO funder_identity(funder, pid) VALUES ('CONICYT', '0000 0001 0943 9683'); +INSERT INTO funder_identity(funder, pid) VALUES ('DFG', '152'); +INSERT INTO funder_identity(funder, pid) VALUES ('DFG', '0000 0001 2096 9829'); +INSERT INTO funder_identity(funder, pid) VALUES ('DFG', '0000 0001 1957 9997'); + + +INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) +VALUES ('H2020', 'European Commission - Horizon 2020', 'H2020', 'EU', 'http://ec.europa.eu/research/participants/data/ref/h2020/grants_manual/hi/oa_pilot/h2020-hi-oa-pilot-guide_en.pdf', 'OA mandate for publications; OA to research data by default', '2015-08-24'); + +INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) +VALUES ('FP7', 'European Commission - 7th Framework program', 'FP7', 'EU', 'http://ec.europa.eu/research/fp7', 'OA advised for publications; 2012 – 2013 (FP7)Special Clause 39 ERC on Open Access applies', '2013-05-07'); + +INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) +VALUES ('FCT', 'Portuguese Foundation for Science and Technology', 'FCT', 'PT', 'https://www.fct.pt/acessoaberto/index.phtml.en', 'OA mandate for publications; OA to research data advised', '2015-02-27'); + +INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) +VALUES ('WT', 'Wellcome Trust', 'WT', '', 'https://wellcome.ac.uk/funding/managing-grant/open-access-policy', 'OA mandate for articles, monographs and book chapters; CC-BY licence required', '2013-05-07'); + +INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) +VALUES ('NSF', 'National Science Foundation', 'NSF', 'US', 'https://www.nsf.gov/news/special_reports/public_access', 'Version of record or AAM of articles, conferences and data deposited into the repository; persistent identifier to full text on publisher’s website', '2016-03-04'); + +INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) +VALUES ('NHMRC', 'National Health and Medical Research Council', 'NHMRC', 'AU', 'https://www.nhmrc.gov.au/grants-funding/policy/nhmrc-open-access-policy', 'OA mandate for publications', '2015-08-24'); + +INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) +VALUES ('ARC', 'Australian Research Council', 'ARC', 'AU', 'http://www.arc.gov.au/arc-open-access-policy', 'OA mandate for research outputs, except for research data', '2015-08-24'); + +INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) +VALUES ('MESTD', 'Ministry of Education, Science and Technological Development', 'MESTD', 'RS', 'NOAD', 'NOAD', '2017-01-23'); + +INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) +VALUES ('MSES', 'Ministry of Science Education and Sport', 'MSES/MZOS', 'HR', 'NOAD', 'NOAD', '2015-09-14'); + +INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) +VALUES ('CSF', 'Croatian Science Foundation', 'CSF/HRZZ', 'HR', 'NOAD', 'NOAD', '2015-09-14'); + +INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) +VALUES ('NWO', 'Netherlands Organisation for Scientific Research', 'NWO', 'NL', 'NOAD', 'NOAD', '2016-06-23'); + +INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) +VALUES ('SFI', 'Science Foundation Ireland', 'SFI', 'IE', 'http://www.sfi.ie/resources/open-access-dec-10.pdf', 'OA mandate to publications, conference proceedings and technical reports', '2015-07-21'); + +INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) +VALUES ('NIH', 'National Institute of Health', 'NIH', 'US', 'https://publicaccess.nih.gov/policy.htm', 'OA to articles', '2017-09-21'); + +INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) +VALUES ('SNSF', 'Swiss National Science Foundation', 'SNSF', 'CH', 'NOAD', 'NOAD', '2016-11-16'); + +INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) +VALUES ('FWF', 'Austrian Science Fund', 'FWF', 'AT', 'NOAD', 'NOAD', '2017-09-21'); + +INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) +VALUES ('RCUK', 'Research Council UK', 'RCUK', 'UK', 'NOAD', 'NOAD', '2017-11-02'); + +INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) +VALUES ('TBT', 'Scientific and Technological Research Council of Turkey', 'Tubitak', 'TR', 'NOAD', 'NOAD', '2017-11-02'); + +INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) +VALUES ('DFG', 'German Research Foundation', 'DFG', 'DE', 'NOAD', 'NOAD', '2017-09-29'); + +INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) +VALUES ('TARA', 'Tara Expedition Foundation', 'TARA', 'FR', 'NOAD', 'NOAD', '2016-12-02'); + +INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) +VALUES ('AFF', 'Academy of Finland', 'AFF', 'FI', 'NOAD', 'NOAD', '2017-09-21'); + +INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) +VALUES ('CONICYT', 'Comisión Nacional de Investigación Científica y Tecnológica', 'CONICYT', 'CL', 'http://www.lareferencia.info', '', '2017-09-08'); + +INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) +VALUES ('SGOV', 'Ministry of Economy, Industry and Competitiveness', 'SGOV', 'ES', 'NOAD', 'NOAD', '2017-09-14'); diff --git a/apps/dnet-exporter-api/src/main/resources/dnet_dsm.sql b/apps/dnet-exporter-api/src/main/resources/dnet_dsm.sql new file mode 100644 index 00000000..b40ee542 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/dnet_dsm.sql @@ -0,0 +1,510 @@ + +CREATE TABLE dsm_api ( + id character varying(255) NOT NULL, + protocol character varying(255), + datasource character varying(255), + contentdescription character varying(255) DEFAULT 'metadata'::character varying, + active boolean DEFAULT false, + removable boolean DEFAULT false, + typology character varying(255) DEFAULT 'UNKNOWN'::character varying, + compatibility character varying(255) DEFAULT 'UNKNOWN'::character varying, + metadata_identifier_path character varying(512) DEFAULT NULL::character varying, + last_collection_total integer, + last_collection_date timestamp without time zone, + last_collection_mdid character varying(255) DEFAULT NULL::character varying, + last_aggregation_total integer, + last_aggregation_date timestamp without time zone, + last_aggregation_mdid character varying(255) DEFAULT NULL::character varying, + last_download_total integer, + last_download_date timestamp without time zone, + last_download_objid character varying(255) DEFAULT NULL::character varying, + last_validation_job character varying(255) DEFAULT NULL::character varying, + baseurl text, + _dnet_resource_identifier_ character varying(2048) DEFAULT ((('temp_'::text || md5((clock_timestamp())::text)) || '_'::text) || md5((random())::text)), + compatibility_override character varying(255) DEFAULT NULL::character varying +); + + +ALTER TABLE public.dsm_api OWNER TO dnetapi; + + +CREATE TABLE dsm_datasource_organization ( + datasource character varying(255) NOT NULL, + organization character varying(255) NOT NULL, + _dnet_resource_identifier_ character varying(2048) DEFAULT ((('temp_'::text || md5((clock_timestamp())::text)) || '_'::text) || md5((random())::text)) +); + + +ALTER TABLE public.dsm_datasource_organization OWNER TO dnetapi; + +-- +-- Name: dsm_datasources; Type: TABLE; Schema: public; Owner: dnetapi; Tablespace: +-- + +CREATE TABLE dsm_datasources ( + id character varying(255) NOT NULL, + officialname character varying(512) NOT NULL, + englishname character varying(512), + websiteurl character varying(255), + logourl character varying(255), + contactemail character varying(255), + latitude double precision DEFAULT 0.0, + longitude double precision DEFAULT 0.0, + timezone character varying(10) DEFAULT '0.0'::character varying, + namespaceprefix character(12) NOT NULL, + languages text, + od_contenttypes text, + collectedfrom character varying(255), + dateofvalidation date, + optional1 character varying(255), + optional2 character varying(255), + typology character varying(255) NOT NULL, + provenanceaction character varying(255) DEFAULT 'UNKNOWN'::character varying, + dateofcollection date DEFAULT ('now'::text)::date NOT NULL, + platform character varying(255), + activationid character varying(255), + description text, + releasestartdate date, + releaseenddate date, + missionstatementurl character varying(512), + dataprovider boolean, + serviceprovider boolean, + databaseaccesstype character varying(64), + datauploadtype character varying(64), + databaseaccessrestriction character varying(64), + datauploadrestriction character varying(64), + versioning boolean, + citationguidelineurl character varying(512), + qualitymanagementkind character varying(64), + pidsystems text, + certificates text, + aggregator character varying(64) DEFAULT 'OPENAIRE'::character varying NOT NULL, + issn character varying(20), + eissn character varying(20), + lissn character varying(20), + registeredby character varying(255), + subjects text, + managed boolean DEFAULT false, + registrationdate date, + consentTermsOfUse boolean default null, + fullTextDownload boolean default null, + _dnet_resource_identifier_ character varying(2048) DEFAULT ((('temp_'::text || md5((clock_timestamp())::text)) || '_'::text) || md5((random())::text)) +); + + +ALTER TABLE public.dsm_datasources OWNER TO dnetapi; + +-- +-- Name: dsm_organizations; Type: TABLE; Schema: public; Owner: dnetapi; Tablespace: +-- + +CREATE TABLE dsm_organizations ( + id character varying(255) NOT NULL, + legalshortname character varying(255), + legalname character varying(255), + websiteurl text, + logourl character varying(255), + ec_legalbody boolean DEFAULT false, + ec_legalperson boolean DEFAULT false, + ec_nonprofit boolean DEFAULT false, + ec_researchorganization boolean DEFAULT false, + ec_highereducation boolean DEFAULT false, + ec_internationalorganizationeurinterests boolean DEFAULT false, + ec_internationalorganization boolean DEFAULT false, + ec_enterprise boolean DEFAULT false, + ec_smevalidated boolean DEFAULT false, + ec_nutscode boolean DEFAULT false, + country character varying(255), + collectedfrom character varying(255), + optional1 character varying(255), + optional2 character varying(255), + dateofcollection date DEFAULT ('now'::text)::date NOT NULL, + provenanceaction character varying(255) DEFAULT 'UNKNOWN'::character varying, + _dnet_resource_identifier_ character varying(2048) DEFAULT ((('temp_'::text || md5((clock_timestamp())::text)) || '_'::text) || md5((random())::text)), + lastupdate date DEFAULT ('now'::text)::date NOT NULL, + trust double precision DEFAULT 0.9 +); + + +ALTER TABLE public.dsm_organizations OWNER TO dnetapi; + +-- +-- Name: browse_countries; Type: VIEW; Schema: public; Owner: dnet +-- + +CREATE VIEW browse_countries AS + SELECT o.country AS term, + count(*) AS total + FROM (((dsm_api a + LEFT JOIN dsm_datasources d ON (((a.datasource)::text = (d.id)::text))) + LEFT JOIN dsm_datasource_organization dao ON (((d.id)::text = (dao.datasource)::text))) + LEFT JOIN dsm_organizations o ON (((dao.organization)::text = (o.id)::text))) + GROUP BY o.country + ORDER BY count(*) DESC; + + +ALTER TABLE public.browse_countries OWNER TO dnet; + + + +CREATE TABLE dsm_apiparams ( + param character varying(255) NOT NULL, + value text DEFAULT ''::character varying NOT NULL, + api character varying(255) NOT NULL, + _dnet_resource_identifier_ character varying(2048) DEFAULT ((('temp_'::text || md5((clock_timestamp())::text)) || '_'::text) || md5((random())::text)) +); + + +ALTER TABLE public.dsm_apiparams OWNER TO dnetapi; + +-- +-- Name: dsm_datasource_api; Type: VIEW; Schema: public; Owner: dnet +-- + +CREATE OR REPLACE VIEW dsm_datasource_api AS + SELECT + row_number() OVER (ORDER BY a.id) AS rowid, + d.id, + d.officialname, + d.englishname, + d.websiteurl, + d.contactemail, + d.collectedfrom, + d.typology, + d.platform, + d.registeredby, + d.managed, + a.protocol, + a.contentdescription, + a.active, + a.removable, + a.typology AS apitypology, + a.compatibility, + a.baseurl + FROM (dsm_datasources d + LEFT JOIN dsm_api a ON (((d.id)::text = (a.datasource)::text))); + +-- +-- Name: dsm_datasourcepids; Type: TABLE; Schema: public; Owner: dnetapi; Tablespace: +-- + +CREATE TABLE dsm_datasourcepids ( + datasource character varying(255) NOT NULL, + pid character varying(255) NOT NULL, + _dnet_resource_identifier_ character varying(2048) DEFAULT ((('temp_'::text || md5((clock_timestamp())::text)) || '_'::text) || md5((random())::text)) +); + + +ALTER TABLE public.dsm_datasourcepids OWNER TO dnetapi; + +-- +-- Name: dsm_identities; Type: TABLE; Schema: public; Owner: dnetapi; Tablespace: +-- + +CREATE TABLE dsm_identities ( + pid character varying(255) NOT NULL, + issuertype character varying(255), + _dnet_resource_identifier_ character varying(2048) DEFAULT ((('temp_'::text || md5((clock_timestamp())::text)) || '_'::text) || md5((random())::text)) +); + + +ALTER TABLE public.dsm_identities OWNER TO dnetapi; + +-- +-- Name: dsm_organizationpids; Type: TABLE; Schema: public; Owner: dnetapi; Tablespace: +-- + +CREATE TABLE dsm_organizationpids ( + organization character varying(255) NOT NULL, + pid character varying(255) NOT NULL, + _dnet_resource_identifier_ character varying(2048) DEFAULT ((('temp_'::text || md5((clock_timestamp())::text)) || '_'::text) || md5((random())::text)) +); + + +ALTER TABLE public.dsm_organizationpids OWNER TO dnetapi; + +-- +-- Name: funder_identity; Type: TABLE; Schema: public; Owner: dnet; Tablespace: +-- + +CREATE TABLE funder_identity ( + funder character varying(255), + pid character varying(255) +); + + +ALTER TABLE public.funder_identity OWNER TO dnet; + +-- +-- Name: funders; Type: TABLE; Schema: public; Owner: dnet; Tablespace: +-- + +CREATE TABLE funders ( + id character varying(255) NOT NULL, + name character varying(255), + shortname character varying(255), + jurisdiction character varying(255), + websiteurl text, + policy character varying(255), + registrationdate date DEFAULT ('now'::text)::date NOT NULL, + lastupdatedate date +); + + +ALTER TABLE public.funders OWNER TO dnet; + +-- +-- Name: fundingpaths; Type: TABLE; Schema: public; Owner: dnet; Tablespace: +-- + + + + + + + + + + + + + + +-- +-- Name: project_organization; Type: TABLE; Schema: public; Owner: dnet; Tablespace: +-- + +--CREATE TABLE project_organization ( +-- participantnumber integer, +-- project character varying(255) NOT NULL, +-- resporganization character varying(255) NOT NULL, +-- semanticclass character varying(255) DEFAULT 'UNKNOWN'::character varying, +-- trust double precision DEFAULT 0.9, +-- _dnet_resource_identifier_ character varying(2048) DEFAULT ((('temp_'::text || md5((clock_timestamp())::text)) || '_'::text) || md5((random())::text)) +--); + + +--ALTER TABLE public.project_organization OWNER TO dnet; + + + + +-- +-- Name: dsm_api_pkey; Type: CONSTRAINT; Schema: public; Owner: dnetapi; Tablespace: +-- + +ALTER TABLE ONLY dsm_api + ADD CONSTRAINT dsm_api_pkey PRIMARY KEY (id); + + +-- +-- Name: dsm_apicollection_pkey; Type: CONSTRAINT; Schema: public; Owner: dnetapi; Tablespace: +-- + +ALTER TABLE ONLY dsm_apiparams + ADD CONSTRAINT dsm_apicollection_pkey PRIMARY KEY (api, param); + + +-- +-- Name: dsm_datasourcepids_pkey; Type: CONSTRAINT; Schema: public; Owner: dnetapi; Tablespace: +-- + +ALTER TABLE ONLY dsm_datasourcepids + ADD CONSTRAINT dsm_datasourcepids_pkey PRIMARY KEY (datasource, pid); + + +-- +-- Name: dsm_datasources_namespaceprefix_key; Type: CONSTRAINT; Schema: public; Owner: dnetapi; Tablespace: +-- + +ALTER TABLE ONLY dsm_datasources + ADD CONSTRAINT dsm_datasources_namespaceprefix_key UNIQUE (namespaceprefix); + + +-- +-- Name: dsm_datasources_pkey; Type: CONSTRAINT; Schema: public; Owner: dnetapi; Tablespace: +-- + +ALTER TABLE ONLY dsm_datasources + ADD CONSTRAINT dsm_datasources_pkey PRIMARY KEY (id); + + +-- +-- Name: dsm_identities_pkey; Type: CONSTRAINT; Schema: public; Owner: dnetapi; Tablespace: +-- + +ALTER TABLE ONLY dsm_identities + ADD CONSTRAINT dsm_identities_pkey PRIMARY KEY (pid); + + +-- +-- Name: dsm_organization_datasource_pkey; Type: CONSTRAINT; Schema: public; Owner: dnetapi; Tablespace: +-- + +ALTER TABLE ONLY dsm_datasource_organization + ADD CONSTRAINT dsm_organization_datasource_pkey PRIMARY KEY (datasource, organization); + + +-- +-- Name: dsm_organizationpids_pkey; Type: CONSTRAINT; Schema: public; Owner: dnetapi; Tablespace: +-- + +ALTER TABLE ONLY dsm_organizationpids + ADD CONSTRAINT dsm_organizationpids_pkey PRIMARY KEY (organization, pid); + + +-- +-- Name: dsm_organizations_pkey; Type: CONSTRAINT; Schema: public; Owner: dnetapi; Tablespace: +-- + +ALTER TABLE ONLY dsm_organizations + ADD CONSTRAINT dsm_organizations_pkey PRIMARY KEY (id); + + + + + + +-- +-- Name: dsm_datasources_contactemail_idx; Type: INDEX; Schema: public; Owner: dnetapi; Tablespace: +-- + +CREATE INDEX dsm_datasources_contactemail_idx ON dsm_datasources USING btree (contactemail); + + +-- +-- Name: dsm_datasources_englishname_idx; Type: INDEX; Schema: public; Owner: dnetapi; Tablespace: +-- + +CREATE INDEX dsm_datasources_englishname_idx ON dsm_datasources USING btree (englishname); + + +-- +-- Name: dsm_datasources_managed_idx; Type: INDEX; Schema: public; Owner: dnetapi; Tablespace: +-- + +CREATE INDEX dsm_datasources_managed_idx ON dsm_datasources USING btree (managed); + + +-- +-- Name: dsm_datasources_officialname_idx; Type: INDEX; Schema: public; Owner: dnetapi; Tablespace: +-- + +CREATE INDEX dsm_datasources_officialname_idx ON dsm_datasources USING btree (officialname); + + +-- +-- Name: dsm_datasources_registeredby_idx; Type: INDEX; Schema: public; Owner: dnetapi; Tablespace: +-- + +CREATE INDEX dsm_datasources_registeredby_idx ON dsm_datasources USING btree (registeredby); + + +-- +-- Name: dsm_organizations_country_idx; Type: INDEX; Schema: public; Owner: dnetapi; Tablespace: +-- + +CREATE INDEX dsm_organizations_country_idx ON dsm_organizations USING btree (country); + + + + + + + + +ALTER TABLE ONLY dsm_api + ADD CONSTRAINT dsm_api_datasource_fkey FOREIGN KEY (datasource) REFERENCES dsm_datasources(id) ON DELETE CASCADE; + + +-- +-- Name: dsm_apicollections_api_fkey; Type: FK CONSTRAINT; Schema: public; Owner: dnetapi +-- + +ALTER TABLE ONLY dsm_apiparams + ADD CONSTRAINT dsm_apicollections_api_fkey FOREIGN KEY (api) REFERENCES dsm_api(id); + + +-- +-- Name: dsm_datasource_organization_datasource_fkey; Type: FK CONSTRAINT; Schema: public; Owner: dnetapi +-- + +ALTER TABLE ONLY dsm_datasource_organization + ADD CONSTRAINT dsm_datasource_organization_datasource_fkey FOREIGN KEY (datasource) REFERENCES dsm_datasources(id); + + +-- +-- Name: dsm_datasource_organization_organization_fkey; Type: FK CONSTRAINT; Schema: public; Owner: dnetapi +-- + +ALTER TABLE ONLY dsm_datasource_organization + ADD CONSTRAINT dsm_datasource_organization_organization_fkey FOREIGN KEY (organization) REFERENCES dsm_organizations(id) ON DELETE CASCADE; + + +-- +-- Name: dsm_datasourcepids_datasource_fkey; Type: FK CONSTRAINT; Schema: public; Owner: dnetapi +-- + +ALTER TABLE ONLY dsm_datasourcepids + ADD CONSTRAINT dsm_datasourcepids_datasource_fkey FOREIGN KEY (datasource) REFERENCES dsm_datasources(id); + + +-- +-- Name: dsm_datasourcepids_pid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: dnetapi +-- + +ALTER TABLE ONLY dsm_datasourcepids + ADD CONSTRAINT dsm_datasourcepids_pid_fkey FOREIGN KEY (pid) REFERENCES dsm_identities(pid); + + +-- +-- Name: dsm_datasources_collectedfrom_fkey; Type: FK CONSTRAINT; Schema: public; Owner: dnetapi +-- + +ALTER TABLE ONLY dsm_datasources + ADD CONSTRAINT dsm_datasources_collectedfrom_fkey FOREIGN KEY (collectedfrom) REFERENCES dsm_datasources(id); + + +-- +-- Name: dsm_organizationpids_organization_fkey; Type: FK CONSTRAINT; Schema: public; Owner: dnetapi +-- + +ALTER TABLE ONLY dsm_organizationpids + ADD CONSTRAINT dsm_organizationpids_organization_fkey FOREIGN KEY (organization) REFERENCES dsm_organizations(id); + + +-- +-- Name: dsm_organizationpids_pid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: dnetapi +-- + +ALTER TABLE ONLY dsm_organizationpids + ADD CONSTRAINT dsm_organizationpids_pid_fkey FOREIGN KEY (pid) REFERENCES dsm_identities(pid); + + +-- +-- Name: dsm_organizations_collectedfrom_fkey; Type: FK CONSTRAINT; Schema: public; Owner: dnetapi +-- + +ALTER TABLE ONLY dsm_organizations + ADD CONSTRAINT dsm_organizations_collectedfrom_fkey FOREIGN KEY (collectedfrom) REFERENCES dsm_datasources(id); + + + +REVOKE ALL ON SCHEMA public FROM PUBLIC; +REVOKE ALL ON SCHEMA public FROM postgres; +GRANT ALL ON SCHEMA public TO postgres; +GRANT ALL ON SCHEMA public TO PUBLIC; + + +REVOKE ALL ON TABLE browse_countries FROM PUBLIC; +REVOKE ALL ON TABLE browse_countries FROM dnet; +GRANT ALL ON TABLE browse_countries TO dnet; +GRANT ALL ON TABLE browse_countries TO dnetapi; + + +REVOKE ALL ON TABLE dsm_datasource_api FROM PUBLIC; +REVOKE ALL ON TABLE dsm_datasource_api FROM dnet; +GRANT ALL ON TABLE dsm_datasource_api TO dnet; +GRANT SELECT ON TABLE dsm_datasource_api TO dnetapi; + + diff --git a/apps/dnet-exporter-api/src/main/resources/dnet_info.sql b/apps/dnet-exporter-api/src/main/resources/dnet_info.sql new file mode 100644 index 00000000..23ddb680 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/dnet_info.sql @@ -0,0 +1,21 @@ + +CREATE TABLE info ( + key character varying(255) NOT NULL PRIMARY KEY, + value DATE +); + +GRANT ALL ON TABLE info TO dnet; +GRANT ALL ON TABLE info TO dnetapi; + +INSERT INTO info(key) VALUES ('oaf_load_date'); +INSERT INTO info(key) VALUES ('odf_load_date'); +INSERT INTO info(key) VALUES ('inference_date'); +INSERT INTO info(key) VALUES ('claim_load_date'); +INSERT INTO info(key) VALUES ('stats_update_date'); + +INSERT INTO info(key) VALUES ('crossref_update_date'); +INSERT INTO info(key) VALUES ('unpaywall_update_date'); +INSERT INTO info(key) VALUES ('orcid_update_date'); +INSERT INTO info(key) VALUES ('mag_update_date'); + + diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/projects_fundings.sql.st b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/projects_fundings.sql.st new file mode 100644 index 00000000..b75fcc9c --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/projects_fundings.sql.st @@ -0,0 +1,11 @@ +SELECT + funder, + jurisdiction, + fundingpathid, + acronym, + title, + code, + startdate, + enddate +FROM projects_api +WHERE fundingpathid like '$fundingprefix$%' diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources.sql.st b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources.sql.st new file mode 100644 index 00000000..a1d29624 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources.sql.st @@ -0,0 +1,25 @@ +select + d.id as id, + d.officialname as "officialName", + d.englishname as "englishName", + o.legalname as organization, + d.typology as typology, + d.registeredby as "registeredBy", + d.registrationdate::text as "registrationDate", + a.compatibility as compatibility, + a.last_collection_date as "lastCollectionDate", + a.last_collection_total as "lastCollectionTotal" +from + dsm_datasources d + left outer join dsm_api a on (d.id = a.datasource) + left outer join dsm_datasource_organization dso on (d.id = dso.datasource) + left outer join dsm_organizations o on (o.id = dso.organization) +where + d.registrationdate is not null + and d.registrationdate < a.last_collection_date + and d.registeredby is not null + and d.managed = true + and a.last_collection_total > 0 + and a.active = true +order by d.registrationdate desc +limit ?; diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate.st.sql b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate.st.sql new file mode 100644 index 00000000..057d5a2f --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate.st.sql @@ -0,0 +1,13 @@ +select count(d.id) as count +from + dsm_datasources d + left outer join dsm_api a on (d.id = a.datasource) + left outer join dsm_datasource_organization dso on (d.id = dso.datasource) + left outer join dsm_organizations o on (o.id = dso.organization) +where + d.registrationdate >= cast(? as date) + and d.registrationdate < a.last_collection_date + and d.registeredby is not null + and d.managed = true + and a.last_collection_total > 0 + and a.active = true; diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate_typology.st.sql b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate_typology.st.sql new file mode 100644 index 00000000..ae5de98c --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate_typology.st.sql @@ -0,0 +1,14 @@ +select count(d.id) as count +from + dsm_datasources d + left outer join dsm_api a on (d.id = a.datasource) + left outer join dsm_datasource_organization dso on (d.id = dso.datasource) + left outer join dsm_organizations o on (o.id = dso.organization) +where + d.registrationdate >= cast(? as date) + and d.typology like ? + and d.registrationdate < a.last_collection_date + and d.registeredby is not null + and d.managed = true + and a.last_collection_total > 0 + and a.active = true; diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/st/projects_dspace_header.st b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/st/projects_dspace_header.st new file mode 100644 index 00000000..3d4a236e --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/st/projects_dspace_header.st @@ -0,0 +1,4 @@ + + + + diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/st/projects_dspace_project.st b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/st/projects_dspace_project.st new file mode 100644 index 00000000..18f23df4 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/st/projects_dspace_project.st @@ -0,0 +1,4 @@ + + $p.code$ - $p.acronym$ - $p.title$ + $p.idnamespace$ + diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/st/projects_dspace_tail.st b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/st/projects_dspace_tail.st new file mode 100644 index 00000000..2a263139 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/st/projects_dspace_tail.st @@ -0,0 +1,2 @@ + + diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/st/projects_eprints.st b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/st/projects_eprints.st new file mode 100644 index 00000000..f0d21f1e --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/st/projects_eprints.st @@ -0,0 +1 @@ +$p.acronym$ - $p.title$
  • $p.code$ - $p.acronym$ - $p.title$
    • $p.idnamespace$
    • diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findCommunityContexts.xquery b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findCommunityContexts.xquery new file mode 100644 index 00000000..7f254339 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findCommunityContexts.xquery @@ -0,0 +1,3 @@ +for $x in collection('/db/DRIVER/ContextDSResources/ContextDSResourceType') +where $x[./RESOURCE_PROFILE/BODY/CONFIGURATION/context/@type = 'community' or ./RESOURCE_PROFILE/BODY/CONFIGURATION/context/@type = 'ri'] +return $x diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findContextProfiles.xquery b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findContextProfiles.xquery new file mode 100644 index 00000000..871c752a --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findContextProfiles.xquery @@ -0,0 +1,2 @@ +for $x in collection('/db/DRIVER/ContextDSResources/ContextDSResourceType') +return $x diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findContextProfilesByType.xquery b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findContextProfilesByType.xquery new file mode 100644 index 00000000..e84fafad --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findContextProfilesByType.xquery @@ -0,0 +1,3 @@ +for $x in collection('/db/DRIVER/ContextDSResources/ContextDSResourceType') +where $x[%s] +return $x diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findFunderContexts.xquery b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findFunderContexts.xquery new file mode 100644 index 00000000..d1bfe051 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findFunderContexts.xquery @@ -0,0 +1,3 @@ +for $x in collection('/db/DRIVER/ContextDSResources/ContextDSResourceType') +where $x[./RESOURCE_PROFILE/BODY/CONFIGURATION/context/@type = 'funding'] +return $x diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findIndexDsInfo.xquery b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findIndexDsInfo.xquery new file mode 100644 index 00000000..bba15af0 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findIndexDsInfo.xquery @@ -0,0 +1,11 @@ +distinct-values( + let $format := collection('/db/DRIVER/ServiceResources/SearchServiceResourceType')//SERVICE_PROPERTIES[./PROPERTY[@key = 'infrastructure' and @value = 'public']]/PROPERTY[@key = "mdformat"]/@value/string() + + for $x in collection('/db/DRIVER/IndexDSResources/IndexDSResourceType') + where + $x//METADATA_FORMAT = $format and + $x//METADATA_FORMAT_INTERPRETATION = 'openaire' and + $x//METADATA_FORMAT_LAYOUT = 'index' + return + concat($x//RESOURCE_IDENTIFIER/@value/string(), ' @@@ ', $format, ' @@@ ', $format, '-index-openaire') +) diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findObjectStore.xquery b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findObjectStore.xquery new file mode 100644 index 00000000..f5c10374 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findObjectStore.xquery @@ -0,0 +1,3 @@ +for $x in collection('/db/DRIVER/RepositoryServiceResources/RepositoryServiceResourceType') +where $x[./RESOURCE_PROFILE/BODY/CONFIGURATION/DATASOURCE_ORIGINAL_ID = "%s"] +return $x/RESOURCE_PROFILE/BODY/CONFIGURATION/INTERFACES/INTERFACE[./@compliance="files"]/INTERFACE_EXTRA_FIELD[./@name="last_download_objId"]/text() diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findSolrIndexUrl.xquery b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findSolrIndexUrl.xquery new file mode 100644 index 00000000..ba3ceab2 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findSolrIndexUrl.xquery @@ -0,0 +1 @@ +distinct-values(collection("/db/DRIVER/ServiceResources/IndexServiceResourceType")//PROTOCOL[@name = "solr" or @name = "SOLR"]/@address/string()) diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/getRepoProfile.xquery b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/getRepoProfile.xquery new file mode 100644 index 00000000..927632cf --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/getRepoProfile.xquery @@ -0,0 +1,3 @@ +for $x in collection('/db/DRIVER/RepositoryServiceResources/RepositoryServiceResourceType') +where $x/RESOURCE_PROFILE/BODY/CONFIGURATION/DATASOURCE_ORIGINAL_ID[@provenance="OPENAIRE"]/text() = "%s" +return $x diff --git a/apps/dnet-exporter-api/src/main/resources/global.properties b/apps/dnet-exporter-api/src/main/resources/global.properties new file mode 100644 index 00000000..0067964f --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/global.properties @@ -0,0 +1,105 @@ +services.is.host = localhost +services.is.port = 8280 +services.is.protocol = http +services.is.context = app +services.is.baseurl = ${services.is.protocol}://${services.is.host}:${services.is.port}/${services.is.context}/services + +openaire.exporter.isLookupUrl = ${services.is.baseurl}/isLookUp +openaire.exporter.objectStoreServiceUrl = ${services.is.baseurl}/objectStore +openaire.exporter.isRegistryServiceUrl = ${services.is.baseurl}/isRegistry + +openaire.exporter.requestWorkers = 10 +openaire.exporter.requestTimeout = 10 + +openaire.exporter.cxfClientConnectTimeout = 60000 +openaire.exporter.cxfClientReceiveTimeout = 120000 + +# JDBC +openaire.exporter.jdbc.url = jdbc:postgresql://localhost:5432/dnet_openaire +openaire.exporter.jdbc.user = dnetapi +openaire.exporter.jdbc.pwd = dnetPwd +openaire.exporter.jdbc.minIdle = 1 +openaire.exporter.jdbc.maxIdle = 20 +openaire.exporter.jdbc.maxRows = 100 + +# PROJECTS +openaire.exporter.project.dspaceHeadTemplate = classpath:/eu/dnetlib/openaire/st/projects_dspace_header.st +openaire.exporter.project.dspaceTemplate = classpath:/eu/dnetlib/openaire/st/projects_dspace_project.st +openaire.exporter.project.dspaceTailTemplate = classpath:/eu/dnetlib/openaire/st/projects_dspace_tail.st +openaire.exporter.project.eprintsTemplate = classpath:/eu/dnetlib/openaire/st/projects_eprints.st +openaire.exporter.project.tsvFields = Grant Agreement Number, Project Acronym, Project Title, Call ID, Start Date, End Date, OA Mandate on Publications, OA Mandate on Datasets, Discipline, Organization, Country, Role, Person Name, Person Second Names, Person Email +openaire.exporter.project.projectsFundingQueryTemplate=classpath:/eu/dnetlib/openaire/sql/projects_fundings.sql.st +openaire.exporter.project.flushSize = 1000 + +# DATSOURCES +openaire.exporter.datasource.title = Data Sources +openaire.exporter.datasource.mongoHost = localhost +openaire.exporter.datasource.mongoPort = 27017 +openaire.exporter.datasource.mongoConnectionsPerHost = 10 +openaire.exporter.datasource.mongoCollectionName = wf_logs +openaire.exporter.datasource.mongoDbName = dnet_logs_prod +openaire.exporter.datasource.mongoQueryLimit= 100 +openaire.exporter.findSolrIndexUrl = /eu/dnetlib/openaire/xquery/findSolrIndexUrl.xquery +openaire.exporter.findIndexDsInfo = /eu/dnetlib/openaire/xquery/findIndexDsInfo.xquery +openaire.exporter.findObjectStore = /eu/dnetlib/openaire/xquery/findObjectStore.xquery +openaire.exporter.findFunderContexts = /eu/dnetlib/openaire/xquery/findFunderContexts.xquery +openaire.exporter.findCommunityContexts = /eu/dnetlib/openaire/xquery/findCommunityContexts.xquery +openaire.exporter.findContextProfiles = /eu/dnetlib/openaire/xquery/findContextProfiles.xquery +openaire.exporter.findContextProfilesByType = /eu/dnetlib/openaire/xquery/findContextProfilesByType.xquery +openaire.exporter.getRepoProfile = /eu/dnetlib/openaire/xquery/getRepoProfile.xquery + +openaire.exporter.contentLoadQuery = { "$and" : [ { "system:profileName" : "Graph construction [PROD]" }, { "system:isCompletedSuccessfully" : "true" }, { "reuseContent" : "false" } ] } + +# REST API CONFIGURATION +openaire.exporter.swaggerDsm.apiTitle = OpenAIRE aggregator REST API +openaire.exporter.swaggerDsm.apiDescription = The OpenAIRE data provision REST API allows developers to access the metadata information space of OpenAIRE programmatically. +openaire.exporter.swaggerDsm.apiLicense = LICENSED UNDER GNU AFFERO GENERAL PUBLIC LICENSE. +openaire.exporter.swaggerDsm.apiLicenseUrl = https://www.gnu.org/licenses/agpl-3.0.txt +openaire.exporter.swaggerDsm.apiContacName = D-Net team +openaire.exporter.swaggerDsm.apiContactUrl = http://www.openaire.eu +openaire.exporter.swaggerDsm.apiContactEmail = dnet-team@isti.cnr.it + +openaire.exporter.swaggerProjects.apiTitle = OpenAIRE projects REST API +openaire.exporter.swaggerProjects.apiDescription = The OpenAIRE projects REST API allows programmatic access to funded research projects metadata. +openaire.exporter.swaggerProjects.apiLicense = ${openaire.exporter.swaggerDsm.apiLicense} +openaire.exporter.swaggerProjects.apiLicenseUrl = ${openaire.exporter.swaggerDsm.apiLicenseUrl} +openaire.exporter.swaggerProjects.apiContacName = ${openaire.exporter.swaggerDsm.apiContacName} +openaire.exporter.swaggerProjects.apiContactUrl = ${openaire.exporter.swaggerDsm.apiContactUrl} +openaire.exporter.swaggerProjects.apiContactEmail = ${openaire.exporter.swaggerDsm.apiContactEmail} + +openaire.exporter.swaggerFunders.apiTitle = OpenAIRE funders REST API +openaire.exporter.swaggerFunders.apiDescription = The OpenAIRE funders REST API allows programmatic access to the funding agencies metadata in OpenAIRE. +openaire.exporter.swaggerFunders.apiLicense = ${openaire.exporter.swaggerDsm.apiLicense} +openaire.exporter.swaggerFunders.apiLicenseUrl = ${openaire.exporter.swaggerDsm.apiLicenseUrl} +openaire.exporter.swaggerFunders.apiContacName = ${openaire.exporter.swaggerDsm.apiContacName} +openaire.exporter.swaggerFunders.apiContactUrl = ${openaire.exporter.swaggerDsm.apiContactUrl} +openaire.exporter.swaggerFunders.apiContactEmail = ${openaire.exporter.swaggerDsm.apiContactEmail} + +openaire.exporter.swaggerCommunities.apiTitle = OpenAIRE Communities REST API +openaire.exporter.swaggerCommunities.apiDescription = The OpenAIRE communities REST API allows programmatic access to the communities configurations in OpenAIRE. +openaire.exporter.swaggerCommunities.apiLicense = ${openaire.exporter.swaggerDsm.apiLicense} +openaire.exporter.swaggerCommunities.apiLicenseUrl = ${openaire.exporter.swaggerDsm.apiLicenseUrl} +openaire.exporter.swaggerCommunities.apiContacName = ${openaire.exporter.swaggerDsm.apiContacName} +openaire.exporter.swaggerCommunities.apiContactUrl = ${openaire.exporter.swaggerDsm.apiContactUrl} +openaire.exporter.swaggerCommunities.apiContactEmail = ${openaire.exporter.swaggerDsm.apiContactEmail} + +openaire.exporter.swaggerContexts.apiTitle = OpenAIRE Contexts REST API +openaire.exporter.swaggerContexts.apiDescription = The OpenAIRE contexts REST API allows programmatic access to the context profiles in OpenAIRE. +openaire.exporter.swaggerContexts.apiLicense = ${openaire.exporter.swaggerDsm.apiLicense} +openaire.exporter.swaggerContexts.apiLicenseUrl = ${openaire.exporter.swaggerDsm.apiLicenseUrl} +openaire.exporter.swaggerContexts.apiContacName = ${openaire.exporter.swaggerDsm.apiContacName} +openaire.exporter.swaggerContexts.apiContactUrl = ${openaire.exporter.swaggerDsm.apiContactUrl} +openaire.exporter.swaggerContexts.apiContactEmail = ${openaire.exporter.swaggerDsm.apiContactEmail} + +openaire.exporter.swaggerInfo.apiTitle = OpenAIRE Info REST API +openaire.exporter.swaggerInfo.apiDescription = The OpenAIRE Info REST API allows programmatic access to some interesting dates related to the content indexed in OpenAIRE. +openaire.exporter.swaggerInfo.apiLicense = ${openaire.exporter.swaggerDsm.apiLicense} +openaire.exporter.swaggerInfo.apiLicenseUrl = ${openaire.exporter.swaggerDsm.apiLicenseUrl} +openaire.exporter.swaggerInfo.apiContacName = ${openaire.exporter.swaggerDsm.apiContacName} +openaire.exporter.swaggerInfo.apiContactUrl = ${openaire.exporter.swaggerDsm.apiContactUrl} +openaire.exporter.swaggerInfo.apiContactEmail = ${openaire.exporter.swaggerDsm.apiContactEmail} + +# VOCABULARIES +openaire.exporter.vocabularies.baseUrl = http://localhost:8980/provision/mvc/vocabularies +openaire.exporter.vocabularies.countriesEndpoint = ${openaire.exporter.vocabularies.baseUrl}/dnet:countries.json +openaire.exporter.vocabularies.datasourceTypologiesEndpoint = ${openaire.exporter.vocabularies.baseUrl}/dnet:datasource_typologies.json diff --git a/apps/dnet-exporter-api/src/main/resources/logback-spring.xml b/apps/dnet-exporter-api/src/main/resources/logback-spring.xml new file mode 100644 index 00000000..756240fa --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/logback-spring.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/apps/dnet-exporter-api/src/main/resources/templates/dnetprofiles/repository.xml b/apps/dnet-exporter-api/src/main/resources/templates/dnetprofiles/repository.xml new file mode 100644 index 00000000..6e7a8b54 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/templates/dnetprofiles/repository.xml @@ -0,0 +1,79 @@ + +
    + + + + + + +
    + + + + + false + + OPENAIRE + + + 0 + 0 + 0 + + + + + + + + + + + + + + + + OpenAireDataSourceId + + + + NamespacePrefix + + + + VERIFIED + NO + + + aggregatorName + + + + dateOfValidation + + + + dateOfCollection + + + + ACTID + + + + + + + + + + + 0 + + 0.0 + + + + +     diff --git a/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/community/CommunityApiControllerTest.java b/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/community/CommunityApiControllerTest.java new file mode 100644 index 00000000..43974fd9 --- /dev/null +++ b/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/community/CommunityApiControllerTest.java @@ -0,0 +1,63 @@ +package eu.dnetlib.openaire.community; + +import static java.util.Collections.singletonList; +import static org.hamcrest.Matchers.is; +import static org.mockito.BDDMockito.given; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import java.nio.charset.Charset; +import java.util.Date; +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.http.MediaType; +import org.springframework.test.web.servlet.MockMvc; + +/** + * Created by Alessia Bardi on 2019-04-04. + * + * @author Alessia Bardi + */ +@SpringBootTest +@WebMvcTest(CommunityApiController.class) +public class CommunityApiControllerTest { + + public static final MediaType APPLICATION_JSON_UTF8 = new MediaType( + MediaType.APPLICATION_JSON.getType(), + MediaType.APPLICATION_JSON.getSubtype(), + Charset.forName("utf8")); + + @Autowired + private MockMvc mvc; + + @MockBean + private CommunityApiController communityController; + + @Test + public void testListCommunities() throws Exception { + final CommunitySummary cs = new CommunitySummary(); + cs.setDescription("the description"); + cs.setId("id1"); + cs.setLastUpdateDate(new Date()); + cs.setName("X"); + cs.setShortName("x"); + final List csList = singletonList(cs); + given(communityController.listCommunities()).willReturn(csList); + + mvc.perform(get("/community/communities").contentType(APPLICATION_JSON_UTF8)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$[0].description", is(cs.getDescription()))) + .andExpect(jsonPath("$[0].id", is(cs.getId()))) + // TODO verify the lastUpdateDate format + // .andExpect(jsonPath("$[0].lastUpdateDate", is(cs.getLastUpdateDate()))) + .andExpect(jsonPath("$[0].name", is(cs.getName()))) + .andExpect(jsonPath("$[0].shortName", is(cs.getShortName()))); + } + +} diff --git a/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/community/selectioncriteria/SelectionCriteriaTest.java b/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/community/selectioncriteria/SelectionCriteriaTest.java new file mode 100644 index 00000000..0d9420ff --- /dev/null +++ b/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/community/selectioncriteria/SelectionCriteriaTest.java @@ -0,0 +1,77 @@ +package eu.dnetlib.openaire.community.selectioncriteria; + +import com.google.gson.Gson; +import eu.dnetlib.data.bulktag.selectioncriteria.Selection; +import org.junit.jupiter.api.Test; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.http.MediaType; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.web.servlet.MockMvc; + +import java.util.ArrayList; +import java.util.List; + + +public class SelectionCriteriaTest { + + private final String json = "{\"selectioncriteria\": " + + "{ \"criteria\": [ { \"constraint\": " + + "[ { \"field\": \"contributor\", \"value\": \"Neuroinformatics\", \"verb\": \"contains\" } ] } ] }"+ + "}"; + + + @Test + public void loadSelectionCriteria(){ + SelectionCriteria sc = new Gson().fromJson(json, SelectionCriteria.class); + + + + } + + @Test + public void printSelectionCriteria(){ + SelectionCriteria sc = new SelectionCriteria(); + + List list_constraints = new ArrayList<>(); + + Constraints constraints = new Constraints(); + + List list_constraint = new ArrayList<>(); + Constraint c = new Constraint(); + c.setField("fake"); + c.setValue("fake"); + c.setVerb("fake"); + list_constraint.add(c); + c = new Constraint(); + c.setField("fake1"); + c.setValue("fake1"); + c.setVerb("fake1"); + list_constraint.add(c); + constraints.setConstraint(list_constraint); + list_constraints.add(constraints); + + constraints = new Constraints(); + list_constraint = new ArrayList<>(); + c = new Constraint(); + c.setField("fake"); + c.setValue("fake"); + c.setVerb("fake"); + list_constraint.add(c); + c = new Constraint(); + c.setField("fake1"); + c.setValue("fake1"); + c.setVerb("fake1"); + list_constraint.add(c); + constraints.setConstraint(list_constraint); + + + list_constraints.add(constraints); + sc.setCriteria(list_constraints); + + System.out.println(new Gson().toJson(sc)); + } + +} diff --git a/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/context/ContextMappingUtilsTest.java b/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/context/ContextMappingUtilsTest.java new file mode 100644 index 00000000..3a30f10f --- /dev/null +++ b/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/context/ContextMappingUtilsTest.java @@ -0,0 +1,42 @@ +package eu.dnetlib.openaire.context; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.util.concurrent.LinkedBlockingQueue; + +import org.apache.commons.io.IOUtils; +import org.junit.jupiter.api.Test; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; + +/** + * Created by Alessia Bardi on 2019-04-04. + * + * + * @author Alessia Bardi + */ +public class ContextMappingUtilsTest { + + @Test + // @Disabled + public void testParseContextProfile() throws IOException { + final String communityProfile = IOUtils.toString(getClass().getResourceAsStream("community_test.xml"), Charset.defaultCharset()); + final Context context = ContextMappingUtils.parseContext(communityProfile, new LinkedBlockingQueue<>()); + + assertNotNull(context); + assertNotNull(context.getId()); + + final Gson gson = new GsonBuilder().setPrettyPrinting().create(); + final String json = gson.toJson(context); + assertNotNull(json); + assertFalse(json.isEmpty()); + + System.out.println(gson.toJson(gson.fromJson(json, Context.class))); + + } + +} diff --git a/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/dsm/DatasourceApiControllerTest.java b/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/dsm/DatasourceApiControllerTest.java new file mode 100644 index 00000000..39a6875d --- /dev/null +++ b/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/dsm/DatasourceApiControllerTest.java @@ -0,0 +1,70 @@ +package eu.dnetlib.openaire.dsm; + +import static java.util.Collections.singletonList; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.core.Is.is; +import static org.mockito.BDDMockito.given; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import java.nio.charset.Charset; +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.http.MediaType; +import org.springframework.test.web.servlet.MockMvc; + +import eu.dnetlib.enabling.datasources.common.DsmException; +import eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils; +import eu.dnetlib.openaire.dsm.domain.DatasourceDetails; +import eu.dnetlib.openaire.dsm.domain.db.DatasourceDbEntry; +import eu.dnetlib.openaire.vocabularies.Country; + +@SpringBootTest +@WebMvcTest(DsmApiController.class) +public class DatasourceApiControllerTest { + + public static final MediaType APPLICATION_JSON_UTF8 = new MediaType( + MediaType.APPLICATION_JSON.getType(), + MediaType.APPLICATION_JSON.getSubtype(), + Charset.forName("utf8")); + + @Autowired + private MockMvc mvc; + + @MockBean + private DsmApiController dsController; + + @Test + // @Disabled + public void listCountries() throws Exception { + final Country c = new Country("it", "Italy"); + final List countries = singletonList(c); + + given(dsController.listCountries()).willReturn(countries); + + mvc.perform(get("/ds/countries") + .contentType(APPLICATION_JSON_UTF8)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$", hasSize(1))) + .andExpect(jsonPath("$[0].code", is(c.getCode()))) + .andExpect(jsonPath("$[0].name", is(c.getName()))); + } + + @Test + public void testAddDs() throws DsmException { + final DatasourceDetails dd = new DatasourceDetails().setId("openaire____::issn19718357") + .setIssn("1971-8357") + .setNamespaceprefix("19718357____"); + + final DatasourceDbEntry datasourceDbEntry = DsmMappingUtils.asDbEntry(dd); + System.out.println(datasourceDbEntry.getIssn()); + + } + +} diff --git a/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/funders/FunderContextClientTest.java b/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/funders/FunderContextClientTest.java new file mode 100644 index 00000000..3e0ca5af --- /dev/null +++ b/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/funders/FunderContextClientTest.java @@ -0,0 +1,50 @@ +package eu.dnetlib.openaire.funders; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.util.concurrent.LinkedBlockingQueue; + +import org.apache.commons.io.IOUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; + +import eu.dnetlib.openaire.context.Context; +import eu.dnetlib.openaire.context.ContextMappingUtils; +import eu.dnetlib.openaire.funders.domain.FunderDetails; + +public class FunderContextClientTest { + + private FunderDao fDao; + + @BeforeEach + public void setUp() { + fDao = new FunderDao(); + } + + @Test + public void testParseContextProfile() throws IOException { + final String contextProfile = IOUtils.toString(getClass().getResourceAsStream("ec-fp7.xml"), Charset.defaultCharset()); + final Context context = ContextMappingUtils.parseContext(contextProfile, new LinkedBlockingQueue<>()); + + assertNotNull(context); + assertNotNull(context.getId()); + + final Gson gson = new GsonBuilder().setPrettyPrinting().create(); + final String json = gson.toJson(context); + assertNotNull(json); + assertFalse(json.isEmpty()); + + // System.out.println(gson.toJson(gson.fromJson(json, Context.class))); + + final FunderDetails funderDetails = ContextMappingUtils.asFunderDetails(context); + // System.out.println(gson.toJson(funderDetails)); + + } + +} diff --git a/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/info/InfoControllerTest.java b/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/info/InfoControllerTest.java new file mode 100644 index 00000000..746fa144 --- /dev/null +++ b/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/info/InfoControllerTest.java @@ -0,0 +1,119 @@ +package eu.dnetlib.openaire.info; + +import static org.hamcrest.core.Is.is; +import static org.mockito.BDDMockito.given; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import java.nio.charset.Charset; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.util.Arrays; +import java.util.stream.Collectors; + +import org.assertj.core.util.Maps; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.http.MediaType; +import org.springframework.test.web.servlet.MockMvc; + +@SpringBootTest +@WebMvcTest(InfoController.class) +public class InfoControllerTest { + + public static final MediaType APPLICATION_JSON_UTF8 = new MediaType( + MediaType.APPLICATION_JSON.getType(), + MediaType.APPLICATION_JSON.getSubtype(), + Charset.forName("utf8")); + + @Autowired + private MockMvc mvc; + + @MockBean + private InfoController infoController; + + private LocalDate expectedDate; + private String formattedDate; + + @BeforeEach + public void setup() { + expectedDate = LocalDate.now(); + formattedDate = expectedDate.format(DateTimeFormatter.ofPattern("yyyy-MM-dd")); + given(infoController.getDate(JdbcInfoDao.DATE_INFO.claim_load_date.name())).willReturn(expectedDate); + given(infoController.getDate(JdbcInfoDao.DATE_INFO.oaf_load_date.name())).willReturn(expectedDate); + given(infoController.getDate(JdbcInfoDao.DATE_INFO.odf_load_date.name())).willReturn(expectedDate); + given(infoController.getDate(JdbcInfoDao.DATE_INFO.inference_date.name())).willReturn(expectedDate); + given(infoController.getDate(JdbcInfoDao.DATE_INFO.stats_update_date.name())).willReturn(expectedDate); + given(infoController.listInfo()).willReturn(Maps.newHashMap(JdbcInfoDao.DATE_INFO.inference_date.name(), LocalDate.now())); + given(infoController.listInfoKeys()) + .willReturn(Arrays.stream(JdbcInfoDao.DATE_INFO.values()).map(JdbcInfoDao.DATE_INFO::name).collect(Collectors.toList())); + + } + + @Test + public void testGetClaimDate() throws Exception { + mvc.perform(get("/info/claim_load_date") + .contentType(APPLICATION_JSON_UTF8)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$", is(formattedDate))); + } + + @Test + public void testGetOafLoadDate() throws Exception { + mvc.perform(get("/info/oaf_load_date") + .contentType(APPLICATION_JSON_UTF8)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$", is(formattedDate))); + } + + @Test + public void testGetOdfLoadDate() throws Exception { + mvc.perform(get("/info/odf_load_date") + .contentType(APPLICATION_JSON_UTF8)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$", is(formattedDate))); + } + + @Test + public void testGetInferenceDate() throws Exception { + mvc.perform(get("/info/inference_date") + .contentType(APPLICATION_JSON_UTF8)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$", is(formattedDate))); + } + + @Test + public void testGetStatsUpdateDateDate() throws Exception { + mvc.perform(get("/info/stats_update_date") + .contentType(APPLICATION_JSON_UTF8)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$", is(formattedDate))); + } + + @Test + public void testListAllDates() throws Exception { + System.out.println(mvc.perform(get("/info") + .contentType(APPLICATION_JSON_UTF8)) + .andExpect(status().isOk()) + .andReturn() + .getResponse() + .getContentAsString()); + + } + + @Test + public void listKeys() throws Exception { + System.out.println(mvc.perform(get("/info/keys") + .contentType(APPLICATION_JSON_UTF8)) + .andExpect(status().isOk()) + .andReturn() + .getResponse() + .getContentAsString()); + } + +} diff --git a/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/project/ProjectQueryParamsTest.java b/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/project/ProjectQueryParamsTest.java new file mode 100644 index 00000000..5524289e --- /dev/null +++ b/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/project/ProjectQueryParamsTest.java @@ -0,0 +1,41 @@ +package eu.dnetlib.openaire.project; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class ProjectQueryParamsTest { + + private ProjectQueryParams queryParams; + + @BeforeEach + public void setUp() throws Exception { + queryParams = new ProjectQueryParams(); + + } + + @Test + public void testVerifyParamWhiteSpace() { + queryParams.verifyParam("Discovery Projects"); + } + + @Test + public void testVerifyParamPercentage() { + queryParams.verifyParam("Discovery%20Projects"); + } + + @Test + public void testVerifyDateParam() { + final String correctDate = "2012-03-04"; + assertEquals(correctDate, queryParams.verifyDateParam(correctDate)); + + } + + @Test + public void testVerifyDateParamException() { + final String wrongDate = "12-12-12"; + assertThrows(IllegalArgumentException.class, () -> queryParams.verifyDateParam(wrongDate)); + } +} diff --git a/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/project/ProjectsControllerTest.java b/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/project/ProjectsControllerTest.java new file mode 100644 index 00000000..efdcea8a --- /dev/null +++ b/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/project/ProjectsControllerTest.java @@ -0,0 +1,131 @@ +package eu.dnetlib.openaire.project; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.IOException; + +import org.antlr.stringtemplate.StringTemplate; +import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.core.io.ClassPathResource; +import org.springframework.core.io.Resource; + +@Disabled +@SpringBootTest +public class ProjectsControllerTest { + + private static final Log log = LogFactory.getLog(ProjectsControllerTest.class); + private final String queryTemplate = "/eu/dnetlib/openaire/sql/projects_fundings.sql.st"; + + private final Resource expectedQueryTemplate = new ClassPathResource("/eu/dnetlib/openaire/sql/expected_projects_fundings.sql.st"); + + private ProjectsController controller; + private ProjectQueryParams params; + + @BeforeEach + public void setup() { + controller = new ProjectsController(); + final Resource template = new ClassPathResource(queryTemplate); + + // TODO reimplement bean injection for testing + // controller.setProjectsFundingQueryTemplate(template); + params = new ProjectQueryParams(); + } + + @Test + public void testObtainFP7Query() throws IllegalArgumentException, IOException { + params.setFundingProgramme("FP7"); + params.setFundingPath(null); + final String res = controller.obtainQuery(params); + final StringTemplate st = new StringTemplate(IOUtils.toString(expectedQueryTemplate.getInputStream(), ProjectsController.UTF8)); + st.setAttribute("fundingprefix", "ec__________::EC::FP7"); + log.debug(res); + log.debug(st); + assertEquals(st.toString(), res); + } + + @Test + public void testObtainFP7QuerySP1() throws IllegalArgumentException, IOException { + params.setFundingProgramme("FP7"); + params.setFundingPath("SP1"); + final String res = controller.obtainQuery(params); + final StringTemplate st = new StringTemplate(IOUtils.toString(expectedQueryTemplate.getInputStream(), ProjectsController.UTF8)); + st.setAttribute("fundingprefix", "ec__________::EC::FP7::SP1"); + log.debug(res); + assertEquals(st.toString(), res); + } + + @Test + public void testObtainFP7QueryHealth() throws IllegalArgumentException, IOException { + params.setFundingProgramme("FP7"); + params.setFundingPath("SP1::HEALTH"); + final String res = controller.obtainQuery(params); + final StringTemplate st = new StringTemplate(IOUtils.toString(expectedQueryTemplate.getInputStream(), ProjectsController.UTF8)); + st.setAttribute("fundingprefix", "ec__________::EC::FP7::SP1::HEALTH"); + log.debug(res); + assertEquals(st.toString(), res); + } + + @Test + public void testObtainFP7QueryHealth2() throws IllegalArgumentException, IOException { + params.setFundingProgramme("FP7"); + params.setFundingPath("%::HEALTH"); + final String res = controller.obtainQuery(params); + final StringTemplate st = new StringTemplate(IOUtils.toString(expectedQueryTemplate.getInputStream(), ProjectsController.UTF8)); + st.setAttribute("fundingprefix", "ec__________::EC::FP7::%::HEALTH"); + log.debug(res); + assertEquals(st.toString(), res); + } + + @Test + public void testObtainWellcomeTrustQuery() throws IllegalArgumentException, IOException { + params.setFundingProgramme("WT"); + params.setFundingPath(null); + final String res = controller.obtainQuery(params); + final StringTemplate st = new StringTemplate(IOUtils.toString(expectedQueryTemplate.getInputStream(), ProjectsController.UTF8)); + st.setAttribute("fundingprefix", "wt__________::WT"); + log.debug(res); + assertEquals(st.toString(), res); + } + + @Test + public void testObtainFCTQuery() throws IllegalArgumentException, IOException { + params.setFundingProgramme("FCT"); + params.setFundingPath(null); + final String res = controller.obtainQuery(params); + final StringTemplate st = new StringTemplate(IOUtils.toString(expectedQueryTemplate.getInputStream(), ProjectsController.UTF8)); + st.setAttribute("fundingprefix", "fct_________::FCT"); + log.debug(res); + assertEquals(st.toString(), res); + } + + @Test + public void testQueryWithDateParams() throws IllegalArgumentException, IOException { + params.setFundingProgramme("WT"); + params.setFundingPath(null); + params.setStartFrom("2015-05-04"); + final String res = controller.obtainQuery(params); + log.debug(res); + final StringTemplate st = new StringTemplate(IOUtils.toString(expectedQueryTemplate.getInputStream(), ProjectsController.UTF8)); + st.setAttribute("fundingprefix", "wt__________::WT"); + final String q = st.toString() + " AND startdate >= '2015-05-04'"; + assertEquals(q, res); + } + + @Test + public void testObtainSNSFQuery() throws IllegalArgumentException, IOException { + params.setFundingProgramme("SNSF"); + params.setFundingPath(null); + final String res = controller.obtainQuery(params); + final StringTemplate st = new StringTemplate(IOUtils.toString(expectedQueryTemplate.getInputStream(), ProjectsController.UTF8)); + st.setAttribute("fundingprefix", "snsf________::SNSF"); + log.debug(res); + assertEquals(st.toString(), res); + } + +} diff --git a/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/project/domain/ProjectDetailsTest.java b/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/project/domain/ProjectDetailsTest.java new file mode 100644 index 00000000..7d342de5 --- /dev/null +++ b/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/project/domain/ProjectDetailsTest.java @@ -0,0 +1,26 @@ +package eu.dnetlib.openaire.project.domain; + +import java.io.IOException; + +import eu.dnetlib.openaire.project.domain.db.ProjectDetails; +import org.junit.jupiter.api.Test; + +/** + * Created by claudio on 05/07/2017. + */ +public class ProjectDetailsTest { + + @Test + public void testCSV() throws IOException { + + final ProjectDetails p = ProjectDetails.fromCSV( + "arc_________::ANZCCART,,ANZCCART,{},\"[\"\"\\u003cfundingtree\\u003e\\n \\u003cfunder\\u003e\\n \\u003cid\\u003earc_________::ARC\\u003c/id\\u003e\\n \\u003cshortname\\u003eARC\\u003c/shortname\\u003e\\n \\u003cname\\u003eAustralian Research Council (ARC)\\u003c/name\\u003e\\n \\u003cjurisdiction\\u003eAU\\u003c/jurisdiction\\u003e\\n \\u003c/funder\\u003e\\n \\u003cfunding_level_0\\u003e\\n \\u003cid\\u003earc_________::ARC::Special Research initiative (Australian and New Zealand Council for the Care of Animals in Research and Teaching)\\u003c/id\\u003e\\n \\u003cname\\u003eSpecial Research initiative (Australian and New Zealand Council for the Care of Animals in Research and Teaching)\\u003c/name\\u003e\\n \\u003cdescription\\u003eSpecial Research initiative (Australian and New Zealand Council for the Care of Animals in Research and Teaching)\\u003c/description\\u003e\\n \\u003cparent/\\u003e\\n \\u003cclass\\u003earc:fundingStream\\u003c/class\\u003e\\n \\u003c/funding_level_0\\u003e\\n \\u003c/fundingtree\\u003e\"\"]\""); + + System.out.println(p.asJson()); + + System.out.println(p.asCSV()); + + + } + +} diff --git a/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/context/community_test.xml b/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/context/community_test.xml new file mode 100644 index 00000000..8508aea9 --- /dev/null +++ b/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/context/community_test.xml @@ -0,0 +1,838 @@ + +
    + + + + + +
    + + + + manager + The scope of this community is to provide access to publications, research data, projects and software that are related to agricultural and food sciences + https://pbs.twimg.com/profile_images/837614594881384448/-Jr9Fpst_400x400.jpg + Agricultural and Food Sciences + pzervas@agroknow.com + animal production and health,fisheries and aquaculture,food safety and human nutrition,information management,food technology,agri-food education and extension,natural resources and environment,food system,engineering technology and Research,agriculture,food safety risk assessment,food security,farming practices and systems,plant production and protection,agri-food economics and policy,food distribution,forestry + AGINFRA+ - Accelerating user-driven e-infrastructure innovation in Food & Agriculture has received funding from the European Union’s Horizon 2020 research and innovation programme under grant agreement No 731001. + oac_aginfra + 2018-03-01T12:00:00 + + + Strengthening European Food Chain Sustainability by Quality and Procurement Policy + 678024 + https://www.strength2food.eu/ + H2020 + EC + Strength2Food + This project has received funding from the European Union's Horizon 2020 research and innovation programme under grant agreement No 678024. + https://www.strength2food.eu/publications/ + + + Future Internet Business Collaboration Networks in Agri-Food, Transport and Logistics + 604123 + https://www.fispace.eu/ + FP7 + EC + FIspace + FIspace has received funding from the European Commission under the FP7 programme. + https://www.fispace.eu/publications.html + + + Personalised public services in support of the implementation of the Common Agricultural Policy + 693171 + https://www.recap-h2020.eu/ + H2020 + EC + RECAP + This project has received funding from the European Union's Horizon 2020 Research and Innovation Programme under Grant Agreement No. 693171. + + + Metrics, Models and Foresight for European SUStainable Food And Nutrition Security + 633692 + http://www.susfans.org/ + H2020 + EC + SUSFANS + + http://www.susfans.org/portfolio + + + Integrated and innovative key actions for mycotoxin management in the food and feed chain + 678781 + http://www.mycokey.eu/ + H2020 + EC + MycoKey + + + Smart Food and Agribusiness: Future Internet for Safe and Healthy Food from Farm to Fork + 285326 + http://smartagrifood.eu/ + FP7 + EC + SmartAgriFood + SmartAgriFood is funded by the EU Seventh Framework Programme under the FI.ICT-2011.1.8 Work Programme + + + Tools for Assessment and Planning of Aquaculture Sustainability + 678396 + http://tapas-h2020.eu/ + H2020 + EC + TAPAS + This project has received funding from the European Union’s Horizon 2020 research and innovation programme under grant agreement No 678396. + http://tapas-h2020.eu/home-publications/ + + + Next generation, Cost-effective, Compact, Multifunctional Web Enabled Ocean Sensor Systems Empowering Marine, Maritime and Fisheries Management + 614102 + http://www.nexosproject.eu/ + FP7 + EC + NeXOS + NeXOS has received funding from the European Union’s Seventh Programme for research, technological development and demonstration under grant agreement No 614102 + http://www.nexosproject.eu/dissemination/publications + + + Robot Fleets for Highly Effective Agriculture and Forestry Management + 245986 + http://www.rhea-project.eu/ + FP7 + EC + RHEA + The research leading to these results has received funding from the European Union’s Seventh Framework Programme [FP7/2007-2013] under Grant Agreement nº 245986 + http://www.rhea-project.eu/Workshops/Workshop1/RHEA-2011-_Proceedings_of_the_1st_RHEA-WorkShop.pdf + + + Optimizing the management and sustainable use of forest genetic resources in Europe + 676876 + http://www.gentree-h2020.eu/ + H2020 + EC + GENTREE + This project has received funding from the European Union’s Horizon 2020 research and innovation programme under grant agreement No 676876. + http://www.gentree-h2020.eu/resources/publications/ + + + Distributed, Integrated and Harmonised Forest Information for Bioeconomy Outlooks + 633464 + http://diabolo-project.eu/ + H2020 + EC + DIABOLO + This project has received funding from the European Union's Horizon 2020 research and innovation programme under grant agreement No 633464. + http://diabolo-project.eu/category/research-papers/ + + + PROVIding smart DElivery of public goods by EU agriculture and forestry + 633838 + http://www.provide-project.eu/ + H2020 + EC + PROVIDE + This project has received funding from the European Union's Horizon 2020 research and innovation programme under grant agreement No 633838. + + + Agricultural Low Cost Integral System of nodes with communication networks for remote water management with sensors and monitoring of the vegetative state of crops + 618123 + http://www.ict-agri.eu/node/38662 + FP7 + EC + ICT-AGRI + This project has received funding from the European Union’s Seventh Framework Programme for research, technological development and demonstration under grant agreement no 618123 [ICT-AGRI 2]. + + + LAND Management: Assessment, Research, Knowledge base + 635201 + http://landmark2020.eu/ + H2020 + EC + LANDMARK + This project has received funding from the European Union’s Horizon 2020 research and innovation programme under grant agreement No 635201. + http://landmark2020.eu/publication-trees/ + + + Soil Care for profitable and sustainable crop production in Europe + 677407 + https://www.soilcare-project.eu/ + H2020 + EC + SOILCARE + The SOILCARE project is co-funded by European Commission, Directorate General for Research under Framework Programme HORIZON 2020, [H2020-SFS-2015-2] (Contract Number [677407]) + + + Innovative tools enabling drinking WATER PROTECTion in rural and urban environments + 727450 + https://water-protect.eu/ + H2020 + EC + WATERPROTECT + + + + 654182 + http://www.envriplus.eu/ + H2020 + EC + ENVRIplus + This project has received funding from the European Union’s Horizon 2020 research and innovation programme under grant agreement No 654182. + http://www.envriplus.eu/publications/ + + + Internet of Food and Farm 2020 + 731884 + https://www.iof2020.eu/ + H2020 + EC + IOF2020 + IoF2020 has received funding from the European Union’s Horizon 2020 research and innovation programme under grant agreement no. 731884 + + + Sweet Pepper Harvesting Robot + 644313 + http://www.sweeper-robot.eu/ + H2020 + EC + SWEEPER + This project has received funding from the European Union's Horizon 2020 research and innovation programme under grant agreement No 644313. + + + Future Internet Enabled Agricultural Applications + 632874 + http://fractals-fp7.com/ + FP7 + EC + FRACTALS + + + + Low-cost, hand-held, and non-invasive optical sensor for multiparametric field analysis of grapes and leaves in vineyards + 262011 + FP7 + EC + PREMIVM + The research leading to these results has received funding from the European Community’s Seventh Framework Programme FP7/2007-2013 managed by REA-Research Executive Agency under grant agreement n° 262011 + + + Space for Agricultural Innovation + 652642 + http://agrispin.eu/ + H2020 + EC + AGRISPIN + This project has received funding from the European Union’s Horizon 2020 research and innovation programme under grant agreement No 652642 + http://agrispin.eu/publicaons/ + + + Authentication and Authorisation for Research and Collaboration + 730941 + https://aarc-project.eu/ + H2020 + EC + AARC2 + AARC2 is funded by the European Union’s Horizon 2020 research and innovation programme under Grant Agreement 730941. + + + European Open Science Cloud - pilot + 739563 + https://eoscpilot.eu/ + H2020 + EC + EOSC-pilot + EOSCPilot.eu has received funding from the European Commission’s Horizon 2020 research and innovation programme under the Grant Agreement no 739563. + + + BUSINESS INTELLIGENCE SERVICE FOR THE MANAGEMENT OF CROPS BASED ON CLOUD AND BIG DATA + 672453 + H2020 + EC + BYNSE + + + + Demonstration of a cloud-based precision farming management system for a sustainable and intensive agriculture to secure long-term food supply in Europe + 672655 + H2020 + EC + AgriCloud + + + + Demonstration of a cloud-based precision farming management system for a sustainable and intensive agriculture to secure long-term food supply in Europe - Phase II + 720176 + H2020 + EC + AgriCloud P2 + + + + Data-Driven Bioeconomy + 732064 + https://www.databio.eu/en/ + H2020 + EC + DataBio + This project has received funding from the European Union’s Horizon 2020 research and innovation programme under grant agreement No 732064 + https://www.databio.eu/en/publications/ + + + Safe Food and Feed through an Integrated ToolBox for Mycotoxin Management + 678012 + https://www.mytoolbox.eu/ + H2020 + EC + MyToolBox + This project has received funding from the European Union's Horizon 2020 research and innovation programme (link is external) under grant agreement No 678012. + https://www.mytoolbox.eu/publications/papers + + + Drone-based integrated monitoring system for early detection of crop pathology and pest control in high tech greenhouse agriculture + 697900 + H2020 + EC + GIDROM + + + + Embedding crop diversity and networking for local high quality food systems + 633571 + http://www.diversifood.eu/ + H2020 + EC + DIVERSIFOOD + This project received funding from the European Union's Horizon 2020 Research and Innovation program under Grant Agreement n° 633571 + http://www.diversifood.eu/publications/ + + + Traditional tomato varieties and cultural practices: a case for agricultural diversification with impact on food security and health of European population + 634561 + http://traditom.eu/ + H2020 + EC + TRADITOM + This project has received funding from the European Union’s Horizon 2020 research and innovation programme under grant agreement No 634561. + http://traditom.eu/publications/ + + + Linking genetic resources, genomes and phenotypes of Solanaceous crops + 677379 + http://www.g2p-sol.eu/ + H2020 + EC + G2P-SOL + The G2P-SOL project (Title: Linking genetic resources, genomes and phenotypes of Solanaceous crops) has received funding from the European Union’s Horizon 2020 research and innovation programme under grant agreement No 677379. + http://www.g2p-sol.eu/Publications.html + + + Improve performance of organic agriculture by boosting organic seed and plant breeding efforts across Europe + 727230 + https://www.liveseed.eu/ + H2020 + EC + LIVESEED + This project has received funding from the European Union’s Horizon 2020 research and innovation programme under grant agreement No 727230 and by the Swiss State Secretariat for Education, Research and Innovation under contract number 17.00090. + https://www.liveseed.eu/resources/scientific-articles/ + + + Transition paths to sustainable legume based systems in Europe + 727973 + https://www.true-project.eu/ + H2020 + EC + TRUE + TRansition paths to sUstainable legume-based systems in Europe (TRUE) has received funding from the European Union’s Horizon 2020 research and innovation programme under grant agreement No. 727973 + https://www.true-project.eu/resources-links/linked-resources/ + + + Innovine Project + 311775 + http://www.innovine.eu/home.html + FP7 + EC + INNOVINE + Innovine is a European collaborative project that has received funding from the European Union’s Seventh Framework Programme for research, technological development and demonstration under grant agreement n° 311775. + http://www.innovine.eu/publications-ressources.html + + + 633531 + https://www.feed-a-gene.eu/ + H2020 + EC + FEED-A-GENE + The Feed-a-Gene Project has received funding from the European Union’s H2020 Programme under grant agreement no 633531. + https://www.feed-a-gene.eu/media/scientific-papers + + + Multidisciplinary Approach to Practical and Acceptable Precision Livestock Farming for SMEs in Europe and world-wide + 227138 + FP7 + EC + BrightAnimal + + + + Practical implementation of precision livestock technologies and services at European pig farms using the living lab methodology + 311989 + FP7 + EC + ALL-SMART-PIGS + + + + Innovative and sustainable systems combining automatic milking and precision grazing + 314879 + https://autograssmilk.dk/ + FP7 + EC + AUTOGRASSMILK + This project has received funding from the European Union's Seventh Framework Programme managed by REA-Research Executive Agency [FP7/2007-2013] under grant agreement no. SME-2012-2-314879. + https://autograssmilk.dk/litterature/ + + + DIVERSITY OF LOCAL PIG BREEDS AND PRODUCTION SYSTEMS FOR HIGH QUALITY TRADITIONAL PRODUCTS AND SUSTAINABLE PORK CHAINS + 634476 + https://treasure.kis.si/ + H2020 + EC + TREASURE + + https://treasure.kis.si/reviews-and-publications/ + + + Innovative Management of Animal Genetic Resources + 677353 + http://www.imageh2020.eu/ + H2020 + EC + IMAGE + This project has received funding from the European Union's Horizon 2020 research and innovation programme under grant agreement No 677353 + http://www.imageh2020.eu/conteudo.php?idm=22&lang=en + + + Genomic management Tools to Optimise Resilience and Efficiency + 727213 + https://www.gentore.eu/ + H2020 + EC + GenTORE + GenTORE is a Horizon 2020 project running from 1 June 2017 to 31 May 2022. This research received funding from the European Union's H2020 Research and Innovation Program under agreement No. 727213. + + + Towards an e-infrastructure Roadmap for Open Science in Agriculture + 730988 + http://www.erosa.aginfra.eu/ + H2020 + EC + e-ROSA + e-ROSA - e-infrastructure Roadmap for Open Science in Agriculture has received funding from the European Union’s Horizon 2020 research and innovation programme under grant agreement No 730988. + http://www.erosa.aginfra.eu/publications + + + An Earth obseRvation Model based RicE information Service + 606983 + FP7 + EC + ERMES + + + + Sustainable techno-economic solutions for the agricultural value chain + 690142 + http://www.agrocycle.eu/ + H2020 + EC + AgroCycle + This project has received funding from the European Union’s Horizon 2020 research and innovation programme under grant agreement No 690142 + + + Innovative approaches to turn agricultural waste into ecological and economic assets + 688338 + http://noaw2020.eu/ + H2020 + EC + NoAW + The project leading to this application has funding from European Union’s Horizon 2020 research and innovation programme under grant agreement No 688338. + + + Bringing added value to agriculture and forest sectors by closing the research and innovation divide + 696394 + http://www.agriforvalor.eu/ + H2020 + EC + AGRIFORVALOR + This project has received funding from the European Union´s Horizon 2020 research and innovation programme under grant agreement No 696394. + http://www.agriforvalor.eu/downloads/ + + + NEFERTITI: Innovation in Demo Farms + 772705 + H2020 + EC + NEFERTITI + NEFERTITI has received funding from the European Union’s Horizon 2020 Programme for Research & Innovation under grant agreement n°772705. + + + Collective Awareness PlatformS for Environmentally-sound Land management based on data technoLogies and Agrobiodiversity + 688813 + http://www.capsella.eu/ + H2020 + EC + CAPSELLA + Capsella has received funding from the European Union’s Horizon 2020 research and innovation programme under grant agreement No 688813 + http://www.capsella.eu/conference-papers/ + + + FArming Tools for external nutrient Inputs and water Management + 633945 + http://fatima-h2020.eu/ + H2020 + EC + FATIMA + This project has received funding from the European Union’s Horizon 2020 research and innovation programme under grant agreement No 633945. + + + Advisory platform for small farms based on earth observation + 687412 + http://apollo-h2020.eu/ + H2020 + EC + APOLLO + This project has received funding from the European Union's Horizon 2020 research and innovation programme under grant agreement No 687412. + + + Data Driven Dairy Decisions 4 Farmers + 696367 + http://www.4d4f.eu/ + H2020 + EC + 4D4F + This project has received funding from the European Union’s Horizon 2020 research and innovation programme under grant agreement No 696367. + + + Aerial Data Collection and Analysis, and Automated Ground Intervention for Precision Farming + 644227 + http://flourish-project.eu/ + H2020 + EC + Flourish + The Flourish project is funded by the European Community's Horizon 2020 programme under grant agreement no 644227-Flourish and from the Swiss State Secretariat for Education, Research and Innovation (SERI) under contract number 15.0029. + http://flourish-project.eu/documents/ + + + Integration of Farm Management Information Systems to support real-time management decisions and compliance of management standards + 212117 + http://www.futurefarm.eu/ + FP7 + EC + FutureFarm + Funded by the Seventh Research Framework Programme (FP7) of the European Union under the Cooperation programme in the Food, Agriculture, Fisheries and Biotechnology theme. Grant Agreement No 212117 (Work Programme KBBE-2007-1-4-05 "The farm of tomorrow"). + http://www.futurefarm.eu/publications + + + WATER AND ENERGY ADVANCED MANAGEMENT FOR IRRIGATION + 619061 + FP7 + EC + WEAM4i + + + + A web-based system for real-time Monitoring and Decision Making for Integrated Vineyard Management + 262059 + http://www.modem-ivm.eu + FP7 + EC + MoDeM_IVM + MoDeM_IVM is funded by the European Union's Seventh Framework Programme managed by REA-Research Executive Agency ([FP7/2007-2013] [ FP7/2007-2011]) under grant agreement n° [262059] + + + Crop, Livestock and Forests Integrated System for Intelligent Automation, Processing and Control + 604659 + http://www.clafis-project.eu/ + FP7 + EC + CLAFIS + + http://www.clafis-project.eu/index.php/dissemination + + + Linked Open Earth Observation Data for Precision Farming + 611141 + http://www.linkedeodata.eu/ + FP7 + EC + LEO + + http://www.linkedeodata.eu/Publications + + + AUTONOMOUS CLOUD-COMPUTING VINEYARD ROBOT TO OPTIMISE YIELD MANAGEMENT AND WINE QUALITY + 605630 + http://vinbot.eu/ + FP7 + EC + VINBOT + + + + Online Professional Irrigation Scheduling Expert System + 613717 + http://www.opiris.eu/ + FP7 + EC + OpIRIS + + http://www.opiris.eu/index.php/publications/ + + + Development of an automatic irrigation and fertilization system + 286772 + FP7 + EC + OPTIFERT + + + + A precise irrigation sensor system to provide an accurate indication of water status in crops and deliver increased yields to farmers + 720032 + http://saturas-ag.com/ + H2020 + EC + StemSense + + + + 674786 + H2020 + EC + VitiPrecision 2020 + + + + Robot shoots herbicide only on weeds, reducing usage by more than 90% + 736354 + H2020 + EC + ASTERIX + + + + Subarea specific irrigation system for pivot- and linear fertigation techniques + 720184 + H2020 + EC + SMART Fertigation + + + + Novel sensor based soil-plant-climate control system for European smart farming + 717797 + H2020 + EC + SenSOP-II + + + + Smart Irrigation Control System with 40% Savings in Water for Universal Use + 720235 + http://yodfatengineers.com/ + H2020 + EC + IRRISAVE + + + + Real time and online monitoring of the debittering stage in the table olive processing + 697335 + http://www.global-olive.es/en/ + H2020 + EC + TELEOLIVA + + + + EARTH OBSERVATION FARMING + 650082 + H2020 + EC + EO-FARM + + + + Interactive Soil Quality Assessment in Europe and China for Agricultural Productivity and Environmental Resilience + 635750 + http://www.isqaper-project.eu/ + H2020 + EC + iSQAPER + iSQAPER is funded by: - The European Union’s Horizon 2020 Programme for research & innovation under grant agreement no 635750 - Ministry of Science and Technology, China (grant nr:2016YFE011270) - Chinese Academy of Sciences (grant nr:16146KYSB20150001) - Swiss State Secretariat for Education, Research and Innovation. Contract: 15.0170-1 + http://www.isqaper-project.eu/downloads/publications + + + Transfer of INNOvative techniques for sustainable WAter use in FERtigated crops + 689687 + http://www.fertinnowa.com/ + H2020 + EC + FERTINNOWA + This project has received funding from the European Union's Horizon 2020 research and innovation programme under grant agreement No 689687 + + + Solutions for improving Agroecosystem and Crop Efficiency for water and nutrient use + 727247 + http://www.solace-eu.net/ + H2020 + EC + SolACE + This project has received funding from the European Union's Horizon 2020 research and innovation programme under grant agreement No 727247 (SolACE) + http://www.solace-eu.net/publications.html + + + Shared Innovation Space for Sustainable Productivity of Grasslands in Europe + 727368 + http://www.inno4grass.eu/en/ + H2020 + EC + Inno4Grass + This project has received funding from the European Union’s Horizon 2020 research and innovation programme under grant agreement No 727368 + + + Peer-to-Peer Learning: Accessing Innovation through Demonstration + 727388 + https://www.plaid-h2020.eu/ + H2020 + EC + PLAID + This project has received funding from the European Union’s Horizon 2020 research and innovation programme under grant agreement No 727388 (PLAID). + + + A data infrastructure to support agricultural scientific communities + AGINFRA + 283770 + EC + corda_______::be2f6f94e5c78b3fba34cbe8454a750f + + + Accelerating user-driven e-infrastructure innovation in Food Agriculture + AGINFRA PLUS + 731001 + EC + corda__h2020::558cf59efd8258c3e65083d750182257 + + + Designing InnoVative plant teams for Ecosystem Resilience and agricultural Sustainability + DIVERSify + 727284 + EC + corda__h2020::c1c2aec17185db77fcf6c8ecb1adc68e + + + + + + opendoar____::1a551829d50f1400b0dab21fdd969c04 + + Repository of the Faculty of Food Technology and Biotechnology + true + + + opendoar____::49af6c4e558a7569d80eee2e035e2bd7 + + CemOA + true + + + opendoar____::0266e33d3f546cb5436a10798e657d97 + + Organic Eprints + true + + + opendoar____::fd4c2dc64ccb8496e6f1f94c85f30d06 + + Agritrop + true + + + opendoar____::41bfd20a38bb1b0bec75acf0845530a7 + + Epsilon Open Archive + true + + + opendoar____::87ae6fb631f7c8a627e8e28785d9992d + + Opin Visindi + true + + + + + edenis + + + + efsa-pilot + + + + egene3 + + + + efsa-kj + + + + euromixproject + + + + discardless + + + + sedinstcjfst + + + + afinet-kc + + + + 2231-4784 + + + + 2231-0606 + + + + solace + + + + pa17 + + + + smartakis + + + + sedinstcjae + + + + phenology_camera + + + + + + + + +     \ No newline at end of file diff --git a/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/exporter/FCT-project.xml b/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/exporter/FCT-project.xml new file mode 100644 index 00000000..02c03c2d --- /dev/null +++ b/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/exporter/FCT-project.xml @@ -0,0 +1,12 @@ + + + FCT + PT + 2003-10-31 + 32639 + Social Classes and «lifestyles» in the city of Oporto + fct_________::FCT::Orçamento de Funcionamento/POSC + PTDC/FIL-FIL/109889/2009 + 2000-11-01 + description of the funding path + diff --git a/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/exporter/FP7-project.xml b/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/exporter/FP7-project.xml new file mode 100644 index 00000000..33222e8d --- /dev/null +++ b/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/exporter/FP7-project.xml @@ -0,0 +1,14 @@ + + + EC + EU + 2012-07-31 + 255646 + Semiconductor lasers for generation of non-diffracting (Bessel) beams. + ec__________::EC::FP7::SP3::PEOPLE + SENDBEAMS + 2010-08-01 + description of the funding path + + + \ No newline at end of file diff --git a/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/exporter/MSES-project.xml b/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/exporter/MSES-project.xml new file mode 100644 index 00000000..695a154b --- /dev/null +++ b/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/exporter/MSES-project.xml @@ -0,0 +1,11 @@ + + + MSES + 2009-12-31 + 001-0000000-3177 + Residence time of bacteria Escherichia coli in seawater and marine organisms + + irb_hr______::MSES::fundingStream + HR + 2007-01-01 + diff --git a/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/exporter/SFI-project.xml b/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/exporter/SFI-project.xml new file mode 100644 index 00000000..8081b73a --- /dev/null +++ b/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/exporter/SFI-project.xml @@ -0,0 +1,11 @@ + + + SFI + 2016-08-31 + 14/ADV/RC3022 + Dr. Vesna Jaksic + + sfi_________::SFI::Advance Award Programme + + 2014-09-01 + diff --git a/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/exporter/WT-project.xml b/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/exporter/WT-project.xml new file mode 100644 index 00000000..a5fd058c --- /dev/null +++ b/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/exporter/WT-project.xml @@ -0,0 +1,11 @@ + + WT + 2013-07-29 + 098241 + EYEDIO DIGI - DEVELOPMENT OF AN INNOVATIVE, AFFORDABLE, EASY TO USE, HANDHELD RETINAL IMAGING PRODUCT FOR THE DIAGNOSIS OF DIABETIC RETINOPATHY. + + wt__________::WT::Immunology and Infectious Disease + 2012-01-30 + + description of the funding path + diff --git a/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/funders/ec-fp7.xml b/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/funders/ec-fp7.xml new file mode 100644 index 00000000..fa39af4a --- /dev/null +++ b/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/funders/ec-fp7.xml @@ -0,0 +1,438 @@ + +
    + + + + + +
    + + + + PROVA + + FP7 + ec__________::EC::FP7 + ec:frameworkprogram + + SP3 + ec__________::EC::FP7::SP3 + ec:specificprogram + + PEOPLE + ec__________::EC::FP7::SP3::PEOPLE + ec:program + + + + SP1 + ec__________::EC::FP7::SP1 + ec:specificprogram + + ENERGY + ec__________::EC::FP7::SP1::ENERGY + ec:program + + + ENV + ec__________::EC::FP7::SP1::ENV + ec:program + + + TPT + ec__________::EC::FP7::SP1::TPT + ec:program + + + NMP + ec__________::EC::FP7::SP1::NMP + ec:program + + + SP1-JTI + ec__________::EC::FP7::SP1::SP1-JTI + ec:program + + + HEALTH + ec__________::EC::FP7::SP1::HEALTH + ec:program + + + ICT + ec__________::EC::FP7::SP1::ICT + ec:program + + + KBBE + ec__________::EC::FP7::SP1::KBBE + ec:program + + + SEC + ec__________::EC::FP7::SP1::SEC + ec:program + + + SSH + ec__________::EC::FP7::SP1::SSH + ec:program + + + SPA + ec__________::EC::FP7::SP1::SPA + ec:program + + + GA + ec__________::EC::FP7::SP1::GA + ec:program + + + + SP2 + ec__________::EC::FP7::SP2 + ec:specificprogram + + ERC + ec__________::EC::FP7::SP2::ERC + ec:program + + + + SP4 + ec__________::EC::FP7::SP4 + ec:specificprogram + + SME + ec__________::EC::FP7::SP4::SME + ec:program + + + INFRA + ec__________::EC::FP7::SP4::INFRA + ec:program + + + SiS + ec__________::EC::FP7::SP4::SiS + ec:program + + + REGIONS + ec__________::EC::FP7::SP4::REGIONS + ec:program + + + REGPOT + ec__________::EC::FP7::SP4::REGPOT + ec:program + + + INCO + ec__________::EC::FP7::SP4::INCO + ec:program + + + COH + ec__________::EC::FP7::SP4::COH + ec:program + + + + SP5 + ec__________::EC::FP7::SP5 + ec:specificprogram + + Fission + ec__________::EC::FP7::SP5::Fission + ec:program + + + Fusion + ec__________::EC::FP7::SP5::Fusion + ec:program + + + + UNKNOWN + ec__________::EC::FP7::UNKNOWN + ec:specificprogram + + UNKNOWN + ec__________::EC::FP7::UNKNOWN::UNKNOWN + ec:program + + + + + H2020 + ec__________::EC::H2020 + ec:h2020fundings + + MSCA-IF-EF-ST + ec__________::EC::H2020::MSCA-IF-EF-ST + ec:h2020toas + + + H2020-EEN-SGA + ec__________::EC::H2020::H2020-EEN-SGA + ec:h2020toas + + + RIA + ec__________::EC::H2020::RIA + ec:h2020toas + + + ERC + ec__________::EC::H2020::ERC + ec:h2020fundings + + ERC-POC + ec__________::EC::H2020::ERC::ERC-POC + ec:h2020toas + + + ERC-STG + ec__________::EC::H2020::ERC::ERC-STG + ec:h2020toas + + + ERC-ADG + ec__________::EC::H2020::ERC::ERC-ADG + ec:h2020toas + + + ERC-COG + ec__________::EC::H2020::ERC::ERC-COG + ec:h2020toas + + + ERC-LVG + ec__________::EC::H2020::ERC::ERC-LVG + ec:h2020toas + + + + MSCA-ITN-ETN + ec__________::EC::H2020::MSCA-ITN-ETN + ec:h2020toas + + + MSCA-IF-GF + ec__________::EC::H2020::MSCA-IF-GF + ec:h2020toas + + + SME-2 + ec__________::EC::H2020::SME-2 + ec:h2020toas + + + SESAR-IA + ec__________::EC::H2020::SESAR-IA + ec:h2020toas + + + MSCA-RISE + ec__________::EC::H2020::MSCA-RISE + ec:h2020toas + + + IA + ec__________::EC::H2020::IA + ec:h2020toas + + + CSA + ec__________::EC::H2020::CSA + ec:h2020toas + + + MSCA-COFUND-DP + ec__________::EC::H2020::MSCA-COFUND-DP + ec:h2020toas + + + SME-1 + ec__________::EC::H2020::SME-1 + ec:h2020toas + + + Shift2Rail-RIA + ec__________::EC::H2020::Shift2Rail-RIA + ec:h2020toas + + + CS2-IA + ec__________::EC::H2020::CS2-IA + ec:h2020toas + + + CS2-RIA + ec__________::EC::H2020::CS2-RIA + ec:h2020toas + + + MSCA-ITN-EID + ec__________::EC::H2020::MSCA-ITN-EID + ec:h2020toas + + + MSCA-IF-EF-RI + ec__________::EC::H2020::MSCA-IF-EF-RI + ec:h2020toas + + + FCH2-RIA + ec__________::EC::H2020::FCH2-RIA + ec:h2020toas + + + ERA-NET-Cofund + ec__________::EC::H2020::ERA-NET-Cofund + ec:h2020toas + + + SESAR-CSA + ec__________::EC::H2020::SESAR-CSA + ec:h2020toas + + + SESAR-RIA + ec__________::EC::H2020::SESAR-RIA + ec:h2020toas + + + BBI-RIA + ec__________::EC::H2020::BBI-RIA + ec:h2020toas + + + MSCA-COFUND-FP + ec__________::EC::H2020::MSCA-COFUND-FP + ec:h2020toas + + + CSA-LS + ec__________::EC::H2020::CSA-LS + ec:h2020toas + + + COFUND-PCP + ec__________::EC::H2020::COFUND-PCP + ec:h2020toas + + + ECSEL-RIA + ec__________::EC::H2020::ECSEL-RIA + ec:h2020toas + + + SGA-CSA + ec__________::EC::H2020::SGA-CSA + ec:h2020toas + + + MSCA-ITN-EJD + ec__________::EC::H2020::MSCA-ITN-EJD + ec:h2020toas + + + MSCA-IF-EF-CAR + ec__________::EC::H2020::MSCA-IF-EF-CAR + ec:h2020toas + + + BBI-IA-DEMO + ec__________::EC::H2020::BBI-IA-DEMO + ec:h2020toas + + + IMI2-RIA + ec__________::EC::H2020::IMI2-RIA + ec:h2020toas + + + MSCA-IF-EF-SE + ec__________::EC::H2020::MSCA-IF-EF-SE + ec:h2020toas + + + FCH2-IA + ec__________::EC::H2020::FCH2-IA + ec:h2020toas + + + BBI-IA-FLAG + ec__________::EC::H2020::BBI-IA-FLAG + ec:h2020toas + + + ECSEL-IA + ec__________::EC::H2020::ECSEL-IA + ec:h2020toas + + + BBI-CSA + ec__________::EC::H2020::BBI-CSA + ec:h2020toas + + + IMI2-CSA + ec__________::EC::H2020::IMI2-CSA + ec:h2020toas + + + PCP + ec__________::EC::H2020::PCP + ec:h2020toas + + + CS2-CSA + ec__________::EC::H2020::CS2-CSA + ec:h2020toas + + + FCH2-CSA + ec__________::EC::H2020::FCH2-CSA + ec:h2020toas + + + COFUND-EJP + ec__________::EC::H2020::COFUND-EJP + ec:h2020toas + + + SGA-RIA + ec__________::EC::H2020::SGA-RIA + ec:h2020toas + + + Shift2Rail-IA + ec__________::EC::H2020::Shift2Rail-IA + ec:h2020toas + + + Shift2Rail-CSA + ec__________::EC::H2020::Shift2Rail-CSA + ec:h2020toas + + + PPI + ec__________::EC::H2020::PPI + ec:h2020toas + + + COFUND-PPI + ec__________::EC::H2020::COFUND-PPI + ec:h2020toas + + + + + + + +     diff --git a/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/sql/expected_projects_fundings.sql.st b/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/sql/expected_projects_fundings.sql.st new file mode 100644 index 00000000..b75fcc9c --- /dev/null +++ b/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/sql/expected_projects_fundings.sql.st @@ -0,0 +1,11 @@ +SELECT + funder, + jurisdiction, + fundingpathid, + acronym, + title, + code, + startdate, + enddate +FROM projects_api +WHERE fundingpathid like '$fundingprefix$%' diff --git a/apps/dnet-exporter-api/src/test/resources/logback-test.xml b/apps/dnet-exporter-api/src/test/resources/logback-test.xml new file mode 100644 index 00000000..e369115c --- /dev/null +++ b/apps/dnet-exporter-api/src/test/resources/logback-test.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/apps/pom.xml b/apps/pom.xml index bd79869f..00414a3f 100644 --- a/apps/pom.xml +++ b/apps/pom.xml @@ -16,6 +16,7 @@ dhp-broker-public-application dhp-mdstore-manager dnet-orgs-database-application + dnet-exporter-api -- 2.17.1 From cf612424c9b38c7ae30e3961c3a41d391602d141 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Mon, 7 Feb 2022 09:03:36 +0100 Subject: [PATCH 02/19] remove of deprecated methods and repo profiles --- .../openaire/common/ExporterConstants.java | 21 +- .../eu/dnetlib/openaire/common/ISClient.java | 24 +- .../dnetlib/openaire/common/ISClientImpl.java | 289 +++------- .../openaire/dsm/DsmApiController.java | 513 +++++++----------- .../java/eu/dnetlib/openaire/dsm/DsmCore.java | 263 +++------ .../dsm/dao/utils/DsmMappingUtils.java | 79 +-- .../src/main/resources/dnet_dsm.data.sql | 176 ------ .../src/main/resources/dnet_dsm.sql | 510 ----------------- .../src/main/resources/dnet_info.sql | 21 - .../templates/dnetprofiles/repository.xml | 79 --- 10 files changed, 384 insertions(+), 1591 deletions(-) delete mode 100644 apps/dnet-exporter-api/src/main/resources/dnet_dsm.data.sql delete mode 100644 apps/dnet-exporter-api/src/main/resources/dnet_dsm.sql delete mode 100644 apps/dnet-exporter-api/src/main/resources/dnet_info.sql delete mode 100644 apps/dnet-exporter-api/src/main/resources/templates/dnetprofiles/repository.xml diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ExporterConstants.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ExporterConstants.java index 5fd8b6f7..29e4e38a 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ExporterConstants.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ExporterConstants.java @@ -3,8 +3,8 @@ package eu.dnetlib.openaire.common; public class ExporterConstants { /* - Tags used to group the operations on the swagger UI - */ + * Tags used to group the operations on the swagger UI + */ public final static String C = "Community"; public final static String C_CP = "Community content providers"; public final static String C_PJ = "Community projects"; @@ -27,21 +27,4 @@ public class ExporterConstants { public static final String OAI = "oai"; public static final String SET = "set"; - // Paths used in the repository profile mapping: Datasource - public static final String ADMIN_INFO = "//CONFIGURATION/ADMIN_INFO"; - public static final String ENGLISH_NAME = "//CONFIGURATION/ENGLISH_NAME"; - public static final String OFFICIAL_NAME = "//CONFIGURATION/OFFICIAL_NAME"; - public static final String TIMEZONE = "//CONFIGURATION/LOCATION/TIMEZONE"; - public static final String LATITUDE = "//CONFIGURATION/LOCATION/LATITUDE"; - public static final String LONGITUDE = "//CONFIGURATION/LOCATION/LONGITUDE"; - public static final String PLATFORM = "//CONFIGURATION/TYPOLOGY"; - public static final String TYPOLOGY = "//CONFIGURATION/DATASOURCE_TYPE"; - public static final String WEBSITE_URL = "//CONFIGURATION/REPOSITORY_WEBPAGE"; - - // Paths used in the repository profile mapping: Interface - public static final String OAI_SET = "/ACCESS_PROTOCOL/@set"; - public static final String BASE_URL = "/BASE_URL"; - public static final String COMPLIANCE = "/@compliance"; - public static final String REMOVABLE = "/@removable"; - } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClient.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClient.java index 848654cc..01752de5 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClient.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClient.java @@ -5,11 +5,8 @@ import java.util.List; import java.util.Map; import eu.dnetlib.enabling.datasources.common.DsmException; -import eu.dnetlib.enabling.datasources.common.DsmForbiddenException; import eu.dnetlib.openaire.context.Context; import eu.dnetlib.openaire.dsm.dao.utils.IndexDsInfo; -import eu.dnetlib.openaire.dsm.domain.ApiDetails; -import eu.dnetlib.openaire.dsm.domain.DatasourceDetails; public interface ISClient { @@ -31,25 +28,16 @@ public interface ISClient { void removeConcept(String id, String categoryId, String conceptId); - void updateDatasourceFields(String dsId, Map changes); - - void addAPIAttribute(String dsId, String apiId, Map changes); - - void updateAPIField(String dsId, String apiId, Map changes); - - void registerDS(DatasourceDetails d); - - void registerAPI(ApiDetails api); - - void removeAPI(String apiId) throws DsmForbiddenException; - void dropCache(); /** * - * @param id id of the concept to be updated (i.e. ni::projects::2) - * @param name name of the attribute to be updated - * @param value new value for the attribute + * @param id + * id of the concept to be updated (i.e. ni::projects::2) + * @param name + * name of the attribute to be updated + * @param value + * new value for the attribute */ void updateConceptAttribute(String id, String name, String value); diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClientImpl.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClientImpl.java index 07e28df9..3e316a07 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClientImpl.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClientImpl.java @@ -1,5 +1,7 @@ package eu.dnetlib.openaire.common; +import static eu.dnetlib.openaire.common.Utils.escape; + import java.io.IOException; import java.nio.charset.Charset; import java.util.List; @@ -10,27 +12,10 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.function.Function; import java.util.stream.Collectors; -import com.google.common.collect.Lists; -import com.google.common.escape.Escaper; -import com.google.common.xml.XmlEscapers; -import eu.dnetlib.OpenaireExporterConfig; -import eu.dnetlib.enabling.datasources.common.DsmException; -import eu.dnetlib.enabling.datasources.common.DsmForbiddenException; -import eu.dnetlib.enabling.datasources.common.DsmRuntimeException; -import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException; -import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; -import eu.dnetlib.enabling.is.registry.rmi.ISRegistryException; -import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService; -import eu.dnetlib.openaire.dsm.dao.utils.IndexDsInfo; -import eu.dnetlib.openaire.context.Context; -import eu.dnetlib.openaire.context.ContextMappingUtils; -import eu.dnetlib.openaire.dsm.domain.ApiDetails; -import eu.dnetlib.openaire.dsm.domain.DatasourceDetails; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.http.HttpStatus; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.cache.annotation.CacheEvict; import org.springframework.cache.annotation.Cacheable; @@ -38,9 +23,19 @@ import org.springframework.core.io.ClassPathResource; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; -import static eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils.asRepositoryInterfce; -import static eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils.asRepositoryProfile; -import static eu.dnetlib.openaire.common.Utils.escape; +import com.google.common.collect.Lists; +import com.google.common.escape.Escaper; +import com.google.common.xml.XmlEscapers; + +import eu.dnetlib.OpenaireExporterConfig; +import eu.dnetlib.enabling.datasources.common.DsmException; +import eu.dnetlib.enabling.datasources.common.DsmRuntimeException; +import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException; +import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; +import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService; +import eu.dnetlib.openaire.context.Context; +import eu.dnetlib.openaire.context.ContextMappingUtils; +import eu.dnetlib.openaire.dsm.dao.utils.IndexDsInfo; /** * Created by claudio on 20/10/2016. @@ -48,7 +43,7 @@ import static eu.dnetlib.openaire.common.Utils.escape; @Component public class ISClientImpl implements ISClient { - private static final Log log = LogFactory.getLog(ISClientImpl.class); + private static final Log log = LogFactory.getLog(ISClientImpl.class); @Autowired private OpenaireExporterConfig config; @@ -110,240 +105,138 @@ public class ISClientImpl implements ISClient { final String xqueryTemplate = _getQuery(config.getFindContextProfilesByType()); final String xquery = String.format(xqueryTemplate, type.stream() - .map(t -> String.format("./RESOURCE_PROFILE/BODY/CONFIGURATION/context/@type = '%s'", t)) - .collect(Collectors.joining(" or "))); + .map(t -> String.format("./RESOURCE_PROFILE/BODY/CONFIGURATION/context/@type = '%s'", t)) + .collect(Collectors.joining(" or "))); return _processContext(xquery); } } @Override - @CacheEvict(value = { "context-cache", "context-cache-funder"}, allEntries = true) + @CacheEvict(value = { + "context-cache", "context-cache-funder" + }, allEntries = true) public void updateContextParam(final String id, final String name, final String value) { try { _quickSeachProfile(getXQuery(id, name, value)); - } catch (ISLookUpException e) { + } catch (final ISLookUpException e) { throw new DsmRuntimeException(String.format("unable update context param [id: %s, name: %s, value: %s]", id, name, value), e); } } @Override - @CacheEvict(value = { "context-cache", "context-cache-funder"}, allEntries = true) + @CacheEvict(value = { + "context-cache", "context-cache-funder" + }, allEntries = true) public void updateContextAttribute(final String id, final String name, final String value) { final Escaper esc = XmlEscapers.xmlAttributeEscaper(); try { - _quickSeachProfile(String.format( - "update value collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" + - "/RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s']/@%s with '%s'", id, name, escape(esc, value))); - } catch (ISLookUpException e) { + _quickSeachProfile(String.format("update value collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" + + "/RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s']/@%s with '%s'", id, name, escape(esc, value))); + } catch (final ISLookUpException e) { throw new DsmRuntimeException(String.format("unable update context attribute [id: %s, name: %s, data: %s]", id, name, value), e); } } @Override - @CacheEvict(value = { "context-cache", "context-cache-funder"}, allEntries = true) + @CacheEvict(value = { + "context-cache", "context-cache-funder" + }, allEntries = true) public void addConcept(final String id, final String categoryId, final String data) { try { - _quickSeachProfile(String.format( - "update insert %s into collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" + - "/RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s']/category[./@id = '%s']", data, id, categoryId)); - } catch (ISLookUpException e) { + _quickSeachProfile(String.format("update insert %s into collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" + + "/RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s']/category[./@id = '%s']", data, id, categoryId)); + } catch (final ISLookUpException e) { throw new DsmRuntimeException(String.format("unable add concept [id: %s, categoryId: %s, data: %s]", id, categoryId, data), e); } } @Override - @CacheEvict(value = { "context-cache", "context-cache-funder"}, allEntries = true) + @CacheEvict(value = { + "context-cache", "context-cache-funder" + }, allEntries = true) public void removeConcept(final String id, final String categoryId, final String conceptId) { try { - _quickSeachProfile(String.format( - "for $concept in collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" + - "/RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s']" + - "/category[./@id = '%s']/concept[./@id = '%s'] " + - "return update delete $concept", id, categoryId, conceptId)); - } catch (ISLookUpException e) { + _quickSeachProfile(String.format("for $concept in collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" + + "/RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s']" + + "/category[./@id = '%s']/concept[./@id = '%s'] " + + "return update delete $concept", id, categoryId, conceptId)); + } catch (final ISLookUpException e) { throw new DsmRuntimeException(String.format("unable remove concept [id: %s, categoryId: %s, conceptId: %s]", id, categoryId, conceptId), e); } } @Override - @CacheEvict(value = { "context-cache", "context-cache-community", "context-cache-funder"}, allEntries = true) - public void updateConceptAttribute(String id, String name, String value) { + @CacheEvict(value = { + "context-cache", "context-cache-community", "context-cache-funder" + }, allEntries = true) + public void updateConceptAttribute(final String id, final String name, final String value) { final Escaper esc = XmlEscapers.xmlAttributeEscaper(); try { - _quickSeachProfile(String.format( - "update value collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" + - "/RESOURCE_PROFILE/BODY/CONFIGURATION/context/category/concept[./@id = '%s']/@%s with '%s'", id, name, escape(esc, value))); - } catch (ISLookUpException e) { + _quickSeachProfile(String.format("update value collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" + + "/RESOURCE_PROFILE/BODY/CONFIGURATION/context/category/concept[./@id = '%s']/@%s with '%s'", id, name, escape(esc, value))); + } catch (final ISLookUpException e) { throw new DsmRuntimeException(String.format("unable update concept attribute [id: %s, name: %s, value: %s]", id, name, value), e); } } @Override - @CacheEvict(value = { "context-cache", "context-cache-funder"}, allEntries = true) - public void updateConceptParam(String id, String name, String value) { + @CacheEvict(value = { + "context-cache", "context-cache-funder" + }, allEntries = true) + public void updateConceptParam(final String id, final String name, final String value) { try { _quickSeachProfile(getConceptXQuery(id, name, value)); - } catch (ISLookUpException e) { + } catch (final ISLookUpException e) { throw new DsmRuntimeException(String.format("unable update concept param [id: %s, name: %s, value: %s]", id, name, value), e); } } @Override - @CacheEvict(value = { "context-cache", "context-cache-funder"}, allEntries = true) - public void updateConceptParamNoEscape(String id, String name, String value) { + @CacheEvict(value = { + "context-cache", "context-cache-funder" + }, allEntries = true) + public void updateConceptParamNoEscape(final String id, final String name, final String value) { try { _quickSeachProfile(getConceptXQueryNoEscape(id, name, value)); - } catch (ISLookUpException e) { + } catch (final ISLookUpException e) { throw new DsmRuntimeException(String.format("unable update concept param [id: %s, name: %s, value: %s]", id, name, value), e); } } - @Override - public void updateDatasourceFields(final String dsId, final Map changes) { - operationManager.addOperation(() -> { - Thread.currentThread().setName("update-ds:" + dsId); - changes.forEach((xpath, value) -> { - try { - _isLookUp(String.format( - "for $x in collection('/db/DRIVER/RepositoryServiceResources/RepositoryServiceResourceType')\n" + - "where $x/RESOURCE_PROFILE/BODY/CONFIGURATION/DATASOURCE_ORIGINAL_ID[@provenance='OPENAIRE']/text() = '%s' \n" + - "return update value $x%s with '%s'", dsId, xpath, value)); - } catch (ISLookUpException e) { - throw new DsmRuntimeException(String.format("unable update datasource fields [id: %s, changes: %s]", dsId, changes), e); - } - }); - }); - } - - @Override - public void addAPIAttribute(final String dsId, final String apiId, final Map changes) { - operationManager.addOperation(() -> { - Thread.currentThread().setName("update-api:" + dsId); - changes.forEach((xpath, value) -> { - try { - final String attribute = StringUtils.substringAfter(xpath, "@"); - final String parentElement = StringUtils.substringBeforeLast(xpath, "/"); - _isLookUp(String.format( - "let $x:=/RESOURCE_PROFILE/BODY/CONFIGURATION/DATASOURCE_ORIGINAL_ID[@provenance='OPENAIRE' and ./text() = '%s']\n" + - "return update insert attribute %s {'%s'} into $x/..//INTERFACE[./@id = '%s']%s", - dsId, attribute, value, apiId, parentElement)); - } catch (ISLookUpException e) { - throw new DsmRuntimeException(String.format("unable add API attribute [dsId: %s, apiId: %s, changes: %s]", dsId, apiId, changes), e); - } - }); - }); - } - - @Override - public void updateAPIField(final String dsId, final String apiId, final Map changes) { - operationManager.addOperation(() -> { - Thread.currentThread().setName("update-api:" + dsId); - changes.forEach((xpath, value) -> { - try { - _isLookUp(String.format( - "let $x:=/RESOURCE_PROFILE/BODY/CONFIGURATION/DATASOURCE_ORIGINAL_ID[@provenance='OPENAIRE' and ./text() = '%s']\n" + - "return update value $x/..//INTERFACE[./@id = '%s']%s with '%s'", - dsId, apiId, xpath, value)); - } catch (ISLookUpException e) { - throw new DsmRuntimeException(String.format("unable update API fields [dsId: %s, apiId: %s, changes: %s]", dsId, apiId, changes), e); - } - }); - }); - } - - @Override - public void registerDS(final DatasourceDetails d) { - operationManager.addOperation(() -> { - Thread.currentThread().setName("save-ds:" + d.getId()); - try { - final String id = isRegistryService.registerProfile(asRepositoryProfile(d)); - log.debug(String.format("registered DS profile %s", id)); - } catch (ISRegistryException e) { - throw new DsmRuntimeException("unable to register DS profile: " + d.getId(), e); - } - }); - } - - @Override - public void registerAPI(final ApiDetails api) { - operationManager.addOperation(() -> { - Thread.currentThread().setName("save-api:" + api.getId()); - try { - final String dsId = api.getDatasource(); - final String iface = asRepositoryInterfce(api); - _isLookUp(String.format( - "let $x:=/RESOURCE_PROFILE/BODY/CONFIGURATION/DATASOURCE_ORIGINAL_ID[@provenance='OPENAIRE' and ./text() = '%s']\n" + - "return update insert %s into $x/../INTERFACES", dsId, iface)); - - log.debug(String.format("registered API %s", api.getId())); - } catch (ISLookUpException e) { - throw new DsmRuntimeException("unable to register API: " + api.getId(), e); - } - }); - } - - @Override - public void removeAPI(final String apiId) throws DsmForbiddenException { - try { - final List metaWorkflows = _quickSeachProfile(String.format( - "distinct-values(for $x in collection('/db/DRIVER/MetaWorkflowDSResources/MetaWorkflowDSResourceType')\n" + - "where $x/RESOURCE_PROFILE/BODY/DATAPROVIDER[./@interface = '%s']\n" + - "return $x/RESOURCE_PROFILE/BODY/DATAPROVIDER/@id/string())", apiId)); - if (!metaWorkflows.isEmpty()) { - throw new DsmForbiddenException( - HttpStatus.SC_FORBIDDEN, - String.format("cannot remove api '%s', it has workflows associated", apiId)); - } - isLookUpService.quickSearchProfile(String.format( - " update delete /RESOURCE_PROFILE/BODY/CONFIGURATION/INTERFACES/INTERFACE[./@id = '%s']", apiId)); - - log.info(String.format("deleted API %s", apiId)); - } catch (ISLookUpException e) { - throw new DsmRuntimeException(String.format("unable to remove API %s", apiId), e); - } - } - /// HELPERS private String getXQuery(final String id, final String name, final String value) { final Escaper esc = XmlEscapers.xmlContentEscaper(); if (StringUtils.isNotBlank(value)) { - return String.format( - "update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" + - "/RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s']/param[./@name = '%s'] with %s", id, name, name, - escape(esc, value)); + return String.format("update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" + + "/RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s']/param[./@name = '%s'] with %s", id, name, name, escape(esc, value)); } else { - return String.format( - "update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" + - "/RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s']/param[./@name = '%s'] with ", id, name, name); + return String.format("update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" + + "/RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s']/param[./@name = '%s'] with ", id, name, name); } } private String getConceptXQuery(final String id, final String name, final String value) { final Escaper esc = XmlEscapers.xmlContentEscaper(); if (StringUtils.isNotBlank(value)) { - return String.format( - "update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')//" + - "concept[./@id = '%s']/param[./@name = '%s'] with %s", id, name, name, - escape(esc, value)); + return String.format("update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')//" + + "concept[./@id = '%s']/param[./@name = '%s'] with %s", id, name, name, escape(esc, value)); } else { - return String.format( - "update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')//concept[./@id = '%s']/param[./@name = '%s'] with ", id, name, name); + return String + .format("update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')//concept[./@id = '%s']/param[./@name = '%s'] with ", id, name, name); } } private String getConceptXQueryNoEscape(final String id, final String name, final String value) { if (StringUtils.isNotBlank(value)) { - return String.format( - "update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')//" + - "concept[./@id = '%s']/param[./@name = '%s'] with %s", id, name, name, - value); + return String.format("update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')//" + + "concept[./@id = '%s']/param[./@name = '%s'] with %s", id, name, name, value); } else { - return String.format( - "update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')//concept[./@id = '%s']/param[./@name = '%s'] with ", id, name, name); + return String + .format("update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')//concept[./@id = '%s']/param[./@name = '%s'] with ", id, name, name); } } @@ -354,15 +247,12 @@ public class ISClientImpl implements ISClient { private Map _processContext(final Queue errors, final String xquery) throws IOException { try { return getContextProfiles(errors, xquery).stream() - .filter(StringUtils::isNotBlank) - .map(s -> ContextMappingUtils.parseContext(s, errors)) - .collect(Collectors.toMap( - Context::getId, - Function.identity(), - (c1, c2) -> { - log.warn(String.format("found duplicate context profile '%s'", c1.getId())); - return c1; - })); + .filter(StringUtils::isNotBlank) + .map(s -> ContextMappingUtils.parseContext(s, errors)) + .collect(Collectors.toMap(Context::getId, Function.identity(), (c1, c2) -> { + log.warn(String.format("found duplicate context profile '%s'", c1.getId())); + return c1; + })); } finally { if (!errors.isEmpty()) { log.error(errors); @@ -375,7 +265,7 @@ public class ISClientImpl implements ISClient { log.warn("getContextProfiles(): not using cache"); try { return _quickSeachProfile(xquery); - } catch (ISLookUpException e) { + } catch (final ISLookUpException e) { throw new DsmRuntimeException("unable to get context profiles", e); } } @@ -386,28 +276,29 @@ public class ISClientImpl implements ISClient { private String _isLookUp(final String xquery) throws ISLookUpException { log.debug(String.format("running xquery:\n%s", xquery)); - //log.debug(String.format("query result: %s", res)); + // log.debug(String.format("query result: %s", res)); return isLookUpService.getResourceProfileByQuery(xquery); } private List _quickSeachProfile(final String xquery) throws ISLookUpException { final List res = Lists.newArrayList(); - log.debug(String.format("running xquery:\n%s", xquery)); - final List list = isLookUpService.quickSearchProfile(xquery); - if (list != null) { - res.addAll(list); - } - log.debug(String.format("query result size: %s", res.size())); - return res; + log.debug(String.format("running xquery:\n%s", xquery)); + final List list = isLookUpService.quickSearchProfile(xquery); + if (list != null) { + res.addAll(list); + } + log.debug(String.format("query result size: %s", res.size())); + return res; } - @CacheEvict(cacheNames = { "context-cache", "indexdsinfo-cache", "objectstoreid-cache" }, allEntries = true) + @Override + @CacheEvict(cacheNames = { + "context-cache", "indexdsinfo-cache", "objectstoreid-cache" + }, allEntries = true) @Scheduled(fixedDelayString = "${openaire.exporter.cache.ttl}") public void dropCache() { log.debug("dropped dsManager IS cache"); } - - } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmApiController.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmApiController.java index c380a6cf..1a5d7598 100755 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmApiController.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmApiController.java @@ -1,7 +1,6 @@ package eu.dnetlib.openaire.dsm; import static eu.dnetlib.openaire.common.ExporterConstants.API; -import static eu.dnetlib.openaire.common.ExporterConstants.D; import static eu.dnetlib.openaire.common.ExporterConstants.DS; import static eu.dnetlib.openaire.common.ExporterConstants.M; import static eu.dnetlib.openaire.common.ExporterConstants.R; @@ -11,8 +10,6 @@ import java.util.List; import javax.validation.Valid; -import eu.dnetlib.enabling.datasources.common.AggregationInfo; -import eu.dnetlib.openaire.dsm.domain.*; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.time.StopWatch; import org.apache.http.HttpStatus; @@ -31,13 +28,28 @@ import eu.dnetlib.enabling.datasources.common.DsmForbiddenException; import eu.dnetlib.enabling.datasources.common.DsmNotFoundException; import eu.dnetlib.openaire.common.AbstractExporterController; import eu.dnetlib.openaire.common.OperationManager; +import eu.dnetlib.openaire.dsm.domain.AggregationHistoryResponse; +import eu.dnetlib.openaire.dsm.domain.ApiDetails; +import eu.dnetlib.openaire.dsm.domain.ApiDetailsResponse; +import eu.dnetlib.openaire.dsm.domain.DatasourceDetailResponse; +import eu.dnetlib.openaire.dsm.domain.DatasourceDetails; +import eu.dnetlib.openaire.dsm.domain.DatasourceDetailsUpdate; +import eu.dnetlib.openaire.dsm.domain.DatasourceResponse; +import eu.dnetlib.openaire.dsm.domain.DatasourceSnippetResponse; +import eu.dnetlib.openaire.dsm.domain.RequestFilter; +import eu.dnetlib.openaire.dsm.domain.RequestSort; +import eu.dnetlib.openaire.dsm.domain.RequestSortOrder; +import eu.dnetlib.openaire.dsm.domain.Response; +import eu.dnetlib.openaire.dsm.domain.SimpleResponse; import eu.dnetlib.openaire.vocabularies.Country; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiResponse; import io.swagger.annotations.ApiResponses; @RestController -@CrossOrigin(origins = { "*" }) +@CrossOrigin(origins = { + "*" +}) @ConditionalOnProperty(value = "openaire.exporter.enable.dsm", havingValue = "true") @io.swagger.annotations.Api(tags = "OpenAIRE DSM API", description = "the OpenAIRE Datasource Manager API") public class DsmApiController extends AbstractExporterController { @@ -45,181 +57,220 @@ public class DsmApiController extends AbstractExporterController { @Autowired private DsmCore dsmCore; - @RequestMapping(value = "/ds/countries", produces = { "application/json" }, method = RequestMethod.GET) - @ApiOperation(value = "list the datasource countries", notes = "list the datasource countries", tags = { DS, R }, response = Country[].class) + @RequestMapping(value = "/ds/countries", produces = { + "application/json" + }, method = RequestMethod.GET) + @ApiOperation(value = "list the datasource countries", notes = "list the datasource countries", tags = { + DS, R + }, response = Country[].class) @ApiResponses(value = { - @ApiResponse(code = 200, message = "OK", response = Country[].class), - @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + @ApiResponse(code = 200, message = "OK", response = Country[].class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) + }) public List listCountries() throws DsmException { return dsmCore.listCountries(); } - @Deprecated - @RequestMapping(value = "/ds/search/{page}/{size}", produces = { "application/json" }, method = RequestMethod.POST) - @ApiOperation(value = "search datasources", notes = "Returns list of Datasource details.", tags = { DS, R }, response = DatasourceSearchResponse.class) + @RequestMapping(value = "/ds/searchdetails/{page}/{size}", produces = { + "application/json" + }, method = RequestMethod.POST) + @ApiOperation(value = "search datasources", notes = "Returns list of Datasource details.", tags = { + DS, R + }, response = DatasourceDetailResponse.class) @ApiResponses(value = { - @ApiResponse(code = 200, message = "OK", response = DatasourceSearchResponse.class), - @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) - public DatasourceSearchResponse search( - @RequestParam final RequestSort requestSortBy, - @RequestParam final RequestSortOrder order, - @RequestBody final RequestFilter requestFilter, - @PathVariable final int page, - @PathVariable final int size) throws DsmException { - final StopWatch stop = StopWatch.createStarted(); - final DatasourceSearchResponse rsp = dsmCore.search(requestSortBy, order, requestFilter, page, size); - return prepareResponse(page, size, stop, rsp); - } - - @RequestMapping(value = "/ds/searchdetails/{page}/{size}", produces = { "application/json" }, method = RequestMethod.POST) - @ApiOperation(value = "search datasources", notes = "Returns list of Datasource details.", tags = { DS, R }, response = DatasourceDetailResponse.class) - @ApiResponses(value = { - @ApiResponse(code = 200, message = "OK", response = DatasourceDetailResponse.class), - @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + @ApiResponse(code = 200, message = "OK", response = DatasourceDetailResponse.class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) + }) public DatasourceDetailResponse searchDsDetails( - @RequestParam final RequestSort requestSortBy, - @RequestParam final RequestSortOrder order, - @RequestBody final RequestFilter requestFilter, - @PathVariable final int page, - @PathVariable final int size) throws DsmException { + @RequestParam final RequestSort requestSortBy, + @RequestParam final RequestSortOrder order, + @RequestBody final RequestFilter requestFilter, + @PathVariable final int page, + @PathVariable final int size) throws DsmException { final StopWatch stop = StopWatch.createStarted(); final DatasourceDetailResponse rsp = dsmCore.searchDsDetails(requestSortBy, order, requestFilter, page, size); return prepareResponse(page, size, stop, rsp); } - @RequestMapping(value = "/ds/aggregationhistory/{dsId}", produces = { "application/json" }, method = RequestMethod.GET) - @ApiOperation(value = "search datasources", notes = "Returns list of Datasource details.", tags = { DS, R }, response = AggregationHistoryResponse.class) + @RequestMapping(value = "/ds/aggregationhistory/{dsId}", produces = { + "application/json" + }, method = RequestMethod.GET) + @ApiOperation(value = "search datasources", notes = "Returns list of Datasource details.", tags = { + DS, R + }, response = AggregationHistoryResponse.class) @ApiResponses(value = { - @ApiResponse(code = 200, message = "OK", response = AggregationHistoryResponse.class), - @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + @ApiResponse(code = 200, message = "OK", response = AggregationHistoryResponse.class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) + }) public AggregationHistoryResponse aggregationHistory(@PathVariable final String dsId) throws DsmException { final StopWatch stop = StopWatch.createStarted(); final AggregationHistoryResponse rsp = dsmCore.aggregationhistory(dsId); return prepareResponse(0, rsp.getAggregationInfo().size(), stop, rsp); } - @RequestMapping(value = "/ds/searchsnippet/{page}/{size}", produces = { "application/json" }, method = RequestMethod.POST) - @ApiOperation(value = "search datasources", notes = "Returns list of Datasource basic info.", tags = { DS, R }, response = DatasourceSnippetResponse.class) + @RequestMapping(value = "/ds/searchsnippet/{page}/{size}", produces = { + "application/json" + }, method = RequestMethod.POST) + @ApiOperation(value = "search datasources", notes = "Returns list of Datasource basic info.", tags = { + DS, R + }, response = DatasourceSnippetResponse.class) @ApiResponses(value = { - @ApiResponse(code = 200, message = "OK", response = DatasourceSnippetResponse.class), - @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + @ApiResponse(code = 200, message = "OK", response = DatasourceSnippetResponse.class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) + }) public DatasourceSnippetResponse searchSnippet( - @RequestParam final RequestSort requestSortBy, - @RequestParam final RequestSortOrder order, - @RequestBody final RequestFilter requestFilter, - @PathVariable final int page, - @PathVariable final int size) throws DsmException { + @RequestParam final RequestSort requestSortBy, + @RequestParam final RequestSortOrder order, + @RequestBody final RequestFilter requestFilter, + @PathVariable final int page, + @PathVariable final int size) throws DsmException { final StopWatch stop = StopWatch.createStarted(); final DatasourceSnippetResponse rsp = dsmCore.searchSnippet(requestSortBy, order, requestFilter, page, size); return prepareResponse(page, size, stop, rsp); } - @RequestMapping(value = "/ds/searchregistered/{page}/{size}", produces = { "application/json" }, method = RequestMethod.POST) - @ApiOperation(value = "search among registered datasources", notes = "Returns list of Datasource basic info.", tags = { DS, - R }, response = DatasourceSnippetResponse.class) + @RequestMapping(value = "/ds/searchregistered/{page}/{size}", produces = { + "application/json" + }, method = RequestMethod.POST) + @ApiOperation(value = "search among registered datasources", notes = "Returns list of Datasource basic info.", tags = { + DS, + R + }, response = DatasourceSnippetResponse.class) @ApiResponses(value = { - @ApiResponse(code = 200, message = "OK", response = DatasourceSnippetResponse.class), - @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + @ApiResponse(code = 200, message = "OK", response = DatasourceSnippetResponse.class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) + }) public DatasourceSnippetResponse searchRegistered( - @RequestParam final RequestSort requestSortBy, - @RequestParam final RequestSortOrder order, - @RequestBody final RequestFilter requestFilter, - @PathVariable final int page, - @PathVariable final int size) throws DsmException { + @RequestParam final RequestSort requestSortBy, + @RequestParam final RequestSortOrder order, + @RequestBody final RequestFilter requestFilter, + @PathVariable final int page, + @PathVariable final int size) throws DsmException { final StopWatch stop = StopWatch.createStarted(); final DatasourceSnippetResponse rsp = dsmCore.searchRegistered(requestSortBy, order, requestFilter, page, size); return prepareResponse(page, size, stop, rsp); } - @RequestMapping(value = "/ds/recentregistered/{size}", produces = { "application/json" }, method = RequestMethod.GET) - @ApiOperation(value = "return the latest datasources that were registered through Provide", notes = "Returns list of Datasource basic info.", tags = { DS, - R }, response = SimpleResponse.class) + @RequestMapping(value = "/ds/recentregistered/{size}", produces = { + "application/json" + }, method = RequestMethod.GET) + @ApiOperation(value = "return the latest datasources that were registered through Provide", notes = "Returns list of Datasource basic info.", tags = { + DS, + R + }, response = SimpleResponse.class) @ApiResponses(value = { - @ApiResponse(code = 200, message = "OK", response = DatasourceResponse.class), - @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + @ApiResponse(code = 200, message = "OK", response = DatasourceResponse.class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) + }) public SimpleResponse recentRegistered(@PathVariable final int size) throws Throwable { final StopWatch stop = StopWatch.createStarted(); final SimpleResponse rsp = dsmCore.searchRecentRegistered(size); return prepareResponse(1, size, stop, rsp); } - @RequestMapping(value = "/ds/countregistered", produces = { "application/json" }, method = RequestMethod.GET) - @ApiOperation(value = "return the number of datasources registered after the given date", notes = "Returns a number.", tags = { DS, - R }, response = Long.class) + @RequestMapping(value = "/ds/countregistered", produces = { + "application/json" + }, method = RequestMethod.GET) + @ApiOperation(value = "return the number of datasources registered after the given date", notes = "Returns a number.", tags = { + DS, + R + }, response = Long.class) @ApiResponses(value = { - @ApiResponse(code = 200, message = "OK", response = Long.class), - @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + @ApiResponse(code = 200, message = "OK", response = Long.class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) + }) public Long countRegistered(@RequestParam final String fromDate, - @RequestParam(required = false) final String typologyFilter) throws Throwable { + @RequestParam(required = false) final String typologyFilter) throws Throwable { return dsmCore.countRegisteredAfter(fromDate, typologyFilter); } - @RequestMapping(value = "/ds/api/{dsId}", produces = { "application/json" }, method = RequestMethod.GET) - @ApiOperation(value = "get the list of API for a given datasource", notes = "Returns the list of API for a given datasource.", tags = { API, - R }, response = ApiDetailsResponse.class) + @RequestMapping(value = "/ds/api/{dsId}", produces = { + "application/json" + }, method = RequestMethod.GET) + @ApiOperation(value = "get the list of API for a given datasource", notes = "Returns the list of API for a given datasource.", tags = { + API, + R + }, response = ApiDetailsResponse.class) @ApiResponses(value = { - @ApiResponse(code = 200, message = "OK", response = ApiDetailsResponse.class), - @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + @ApiResponse(code = 200, message = "OK", response = ApiDetailsResponse.class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) + }) public ApiDetailsResponse getApi( - @PathVariable final String dsId) throws DsmException { + @PathVariable final String dsId) throws DsmException { final StopWatch stop = StopWatch.createStarted(); final ApiDetailsResponse rsp = dsmCore.getApis(dsId); return prepareResponse(0, rsp.getApi().size(), stop, rsp); } - @RequestMapping(value = "/api/baseurl/{page}/{size}", produces = { "application/json" }, method = RequestMethod.POST) + @RequestMapping(value = "/api/baseurl/{page}/{size}", produces = { + "application/json" + }, method = RequestMethod.POST) @ApiOperation(value = "search for the list of base URLs of Datasource APIs managed by a user", notes = "Returns the list of base URLs of Datasource APIs managed by a user", tags = { - DS, API, R }, response = String[].class) + DS, API, R + }, response = String[].class) @ApiResponses(value = { - @ApiResponse(code = 200, message = "OK", response = String[].class), - @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + @ApiResponse(code = 200, message = "OK", response = String[].class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) + }) public List searchBaseUrls( - @RequestBody final RequestFilter requestFilter, - @PathVariable final int page, - @PathVariable final int size) throws DsmException { + @RequestBody final RequestFilter requestFilter, + @PathVariable final int page, + @PathVariable final int size) throws DsmException { return dsmCore.findBaseURLs(requestFilter, page, size); } @RequestMapping(value = "/ds/api/{apiId}", method = RequestMethod.DELETE) - @ApiOperation(value = "delete an API", notes = "delete an API, if removable", tags = { API, W }) + @ApiOperation(value = "delete an API", notes = "delete an API, if removable", tags = { + API, W + }) @ApiResponses(value = { - @ApiResponse(code = 200, message = "OK"), - @ApiResponse(code = 400, message = "Api not found", response = ErrorMessage.class), - @ApiResponse(code = 403, message = "Api not removable", response = ErrorMessage.class), - @ApiResponse(code = 500, message = "DSM Server error", response = ErrorMessage.class)}) + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 400, message = "Api not found", response = ErrorMessage.class), + @ApiResponse(code = 403, message = "Api not removable", response = ErrorMessage.class), + @ApiResponse(code = 500, message = "DSM Server error", response = ErrorMessage.class) + }) public void deleteApi(@PathVariable final String apiId) throws DsmForbiddenException, DsmNotFoundException { dsmCore.deleteApi(apiId); } @RequestMapping(value = "/ds/manage", method = RequestMethod.POST) - @ApiOperation(value = "set the managed status for a given datasource", notes = "set the managed status for a given datasource", tags = { DS, W }) + @ApiOperation(value = "set the managed status for a given datasource", notes = "set the managed status for a given datasource", tags = { + DS, W + }) @ApiResponses(value = { - @ApiResponse(code = 200, message = "OK"), - @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) + }) public void setManaged( - @RequestParam final String id, - @RequestParam final boolean managed) throws DsmException { + @RequestParam final String id, + @RequestParam final boolean managed) throws DsmException { dsmCore.setManaged(id, managed); } @RequestMapping(value = "/ds/managed/{id}", method = RequestMethod.GET) - @ApiOperation(value = "get the datasource managed status", notes = "get the datasource managed status", tags = { DS, R }) + @ApiOperation(value = "get the datasource managed status", notes = "get the datasource managed status", tags = { + DS, R + }) @ApiResponses(value = { - @ApiResponse(code = 200, message = "OK"), - @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) + }) public boolean isManaged(@PathVariable final String id) throws DsmException { return dsmCore.isManaged(id); } @RequestMapping(value = "/ds/add", method = RequestMethod.POST) - @ApiOperation(value = "add a new Datasource", notes = "add a new Datasource", tags = { DS, W }) + @ApiOperation(value = "add a new Datasource", notes = "add a new Datasource", tags = { + DS, W + }) @ApiResponses(value = { - @ApiResponse(code = 200, message = "OK"), - @ApiResponse(code = 400, message = "Malformed request", response = ErrorMessage[].class), - @ApiResponse(code = 500, message = "Unexpected error", response = ErrorMessage.class) }) + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 400, message = "Malformed request", response = ErrorMessage[].class), + @ApiResponse(code = 500, message = "Unexpected error", response = ErrorMessage.class) + }) public void saveDs(@Valid @RequestBody final DatasourceDetails datasource) throws DsmException { if (dsmCore.exist(datasource)) { // TODO further check that the DS doesn't have any API @@ -229,155 +280,76 @@ public class DsmApiController extends AbstractExporterController { } @RequestMapping(value = "/ds/update", method = RequestMethod.POST) - @ApiOperation(value = "update Datasource details", notes = "update Datasource details", tags = { DS, W }) + @ApiOperation(value = "update Datasource details", notes = "update Datasource details", tags = { + DS, W + }) @ApiResponses(value = { - @ApiResponse(code = 200, message = "OK"), - @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) + }) public void updateDatasource( - @RequestBody final DatasourceDetailsUpdate ds) throws DsmException, DsmNotFoundException { + @RequestBody final DatasourceDetailsUpdate ds) throws DsmException, DsmNotFoundException { dsmCore.updateDatasource(ds); } - @Deprecated - @RequestMapping(value = "/ds/logourl", method = RequestMethod.POST) - @ApiOperation(value = "update a datasource name", notes = "update a datasource name", tags = { DS, W, D }) - @ApiResponses(value = { - @ApiResponse(code = 200, message = "OK"), - @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) - public void updateDatasourceLogoURL( - @RequestParam final String dsId, - @RequestParam final String logourl) throws DsmException { - - dsmCore.updateDatasourceLogoUrl(dsId, logourl); - } - - @Deprecated - @RequestMapping(value = "/ds/name", method = RequestMethod.POST) - @ApiOperation(value = "update a datasource name", notes = "update a datasource name", tags = { DS, W, D }) - @ApiResponses(value = { - @ApiResponse(code = 200, message = "OK"), - @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) - public void updateDatasourceName( - @RequestParam final String dsId, - @RequestParam final String officialname, - @RequestParam final String englishname) throws DsmException { - - dsmCore.updateDatasourcename(dsId, officialname, englishname); - } - - @Deprecated - @RequestMapping(value = "/ds/coordinates", method = RequestMethod.POST) - @ApiOperation(value = "update the datasource coordinates (latitude, longitude)", notes = "update the datasource coordinates (latitude, longitude)", tags = { - DS, W, D }) - @ApiResponses(value = { - @ApiResponse(code = 200, message = "OK"), - @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) - public void updateCoordinates( - @RequestParam final String dsId, - @RequestParam final Double latitude, - @RequestParam final Double longitude) throws DsmException { - - dsmCore.updateCoordinates(dsId, latitude, longitude); - } - - @Deprecated - @RequestMapping(value = "/ds/timezone", method = RequestMethod.POST) - @ApiOperation(value = "update a datasource timezone", notes = "update a datasource timezone", tags = { DS, W, D }) - @ApiResponses(value = { - @ApiResponse(code = 200, message = "OK"), - @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) - public void updateTimezone( - @RequestParam final String dsId, - @RequestParam final String timezone) throws DsmException { - - dsmCore.updateTimezone(dsId, timezone); - } - - @Deprecated - @RequestMapping(value = "/ds/typology", method = RequestMethod.POST) - @ApiOperation(value = "update a datasource typology code", notes = "update a datasource typology code", tags = { DS, W, D }) - @ApiResponses(value = { - @ApiResponse(code = 200, message = "OK"), - @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) - public void updateTypology( - @RequestParam final String dsId, - @RequestParam final String typology) throws DsmException { - - dsmCore.updateDsTypology(dsId, typology); - } - - @Deprecated - @RequestMapping(value = "/ds/registeredby", method = RequestMethod.POST) - @ApiOperation(value = "update a datasource registeredBy", notes = "update a datasource registeredBy", tags = { DS, W, D }) - @ApiResponses(value = { - @ApiResponse(code = 200, message = "OK"), - @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) - public void updateRegisteringUser( - @RequestParam final String dsId, - @RequestParam final String registeredBy) throws DsmException { - - dsmCore.updateDsRegisteringUser(dsId, registeredBy); - } - - @Deprecated - @RequestMapping(value = "/ds/platform", method = RequestMethod.POST) - @ApiOperation(value = "update a datasource platform", notes = "update a datasource platform", tags = { DS, W, D }) - @ApiResponses(value = { - @ApiResponse(code = 200, message = "OK"), - @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) - public void updatePlatform( - @RequestParam final String dsId, - @RequestParam final String platform) throws DsmException { - - dsmCore.updateDsPlatform(dsId, platform); - } - @RequestMapping(value = "/ds/api/baseurl", method = RequestMethod.POST) - @ApiOperation(value = "update the base URL of a datasource interface", notes = "update the base URL of a datasource interface", tags = { API, W }) + @ApiOperation(value = "update the base URL of a datasource interface", notes = "update the base URL of a datasource interface", tags = { + API, W + }) @ApiResponses(value = { - @ApiResponse(code = 200, message = "OK"), - @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) + }) public void updateBaseUrl( - @RequestParam final String dsId, - @RequestParam final String apiId, - @RequestParam final String baseUrl) throws DsmException { + @RequestParam final String dsId, + @RequestParam final String apiId, + @RequestParam final String baseUrl) throws DsmException { dsmCore.updateApiBaseurl(dsId, apiId, baseUrl); } @RequestMapping(value = "/ds/api/compliance", method = RequestMethod.POST) - @ApiOperation(value = "update the compatibility of a datasource interface", notes = "update the compatibility of a datasource interface", tags = { API, W }) + @ApiOperation(value = "update the compatibility of a datasource interface", notes = "update the compatibility of a datasource interface", tags = { + API, W + }) @ApiResponses(value = { - @ApiResponse(code = 200, message = "OK"), - @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) + }) public void updateCompliance( - @RequestParam final String dsId, - @RequestParam final String apiId, - @RequestParam final String compliance, - @RequestParam(required = false, defaultValue = "false") final boolean override) throws DsmException { + @RequestParam final String dsId, + @RequestParam final String apiId, + @RequestParam final String compliance, + @RequestParam(required = false, defaultValue = "false") final boolean override) throws DsmException { dsmCore.updateApiCompatibility(dsId, apiId, compliance, override); } @RequestMapping(value = "/ds/api/oaiset", method = RequestMethod.POST) - @ApiOperation(value = "update the OAI set of a datasource interface", notes = "update the OAI set of a datasource interface", tags = { API, W }) + @ApiOperation(value = "update the OAI set of a datasource interface", notes = "update the OAI set of a datasource interface", tags = { + API, W + }) @ApiResponses(value = { - @ApiResponse(code = 200, message = "OK"), - @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) + }) public void updateOaiSetl( - @RequestParam final String dsId, - @RequestParam final String apiId, - @RequestParam final String oaiSet) throws DsmException { + @RequestParam final String dsId, + @RequestParam final String apiId, + @RequestParam final String oaiSet) throws DsmException { dsmCore.updateApiOaiSet(dsId, apiId, oaiSet); } @RequestMapping(value = "/ds/api/add", method = RequestMethod.POST) - @ApiOperation(value = "adds a new Interface to one Datasource", notes = "adds an Interface to one Datasource", tags = { API, W }) + @ApiOperation(value = "adds a new Interface to one Datasource", notes = "adds an Interface to one Datasource", tags = { + API, W + }) @ApiResponses(value = { - @ApiResponse(code = 200, message = "OK"), - @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) + }) public void addApi(@RequestBody final ApiDetails api) throws DsmException { if (StringUtils.isBlank(api.getDatasource())) { throw new DsmException(HttpStatus.SC_BAD_REQUEST, "missing datasource id"); } dsmCore.addApi(api); @@ -389,110 +361,47 @@ public class DsmApiController extends AbstractExporterController { private OperationManager operationManager; @RequestMapping(value = "/dsm/ops", method = RequestMethod.GET) - @ApiOperation(value = "get the number of pending operations", notes = "get the number of pending operations", tags = { R, M }) + @ApiOperation(value = "get the number of pending operations", notes = "get the number of pending operations", tags = { + R, M + }) @ApiResponses(value = { - @ApiResponse(code = 200, message = "OK"), - @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) + }) public int getOps() throws DsmException { return operationManager.getOpSize(); } @RequestMapping(value = "/dsm/killops", method = RequestMethod.POST) - @ApiOperation(value = "interrupts the pending operations", notes = "return the number of interrupted operations", tags = { W, M }) + @ApiOperation(value = "interrupts the pending operations", notes = "return the number of interrupted operations", tags = { + W, M + }) @ApiResponses(value = { - @ApiResponse(code = 200, message = "OK"), - @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) + }) public int killOps() throws DsmException { return operationManager.dropAll(); } @RequestMapping(value = "/dsm/dropcache", method = RequestMethod.POST) - @ApiOperation(value = "drop the caches", notes = "drop the internal caches", tags = { W, M }) + @ApiOperation(value = "drop the caches", notes = "drop the internal caches", tags = { + W, M + }) @ApiResponses(value = { - @ApiResponse(code = 200, message = "OK"), - @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) + }) public void dropCache() throws DsmException { dsmCore.dropCaches(); } - // OLD DEPRECATED METHODS - - @Deprecated - @ApiOperation(value = "search datasources by name", notes = "Returns list of Datasource details.", tags = { D }) - @ApiResponses(value = { - @ApiResponse(code = 200, message = "OK", response = DatasourceSearchResponse.class), - @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) - @RequestMapping(value = "/ds/search/name/{page}/{size}", produces = { "application/json" }, method = RequestMethod.GET) - DatasourceSearchResponse searchByName(final String name, final int page, final int size) throws DsmException { - final RequestSort sort = RequestSort.id; - final RequestSortOrder order = RequestSortOrder.ASCENDING; - final RequestFilter filter = new RequestFilter(); - filter.put(FilterName.englishname, name); - - final StopWatch stop = StopWatch.createStarted(); - final DatasourceSearchResponse rsp = dsmCore.search(sort, order, filter, page, size); - return prepareResponse(page, size, stop, rsp); - } - - @Deprecated - @ApiOperation(value = "search datasources by contact email", notes = "Returns list of Datasource details.", tags = { D }) - @ApiResponses(value = { - @ApiResponse(code = 200, message = "OK", response = DatasourceSearchResponse.class), - @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) - @RequestMapping(value = "/ds/search/email/{page}/{size}", produces = { "application/json" }, method = RequestMethod.GET) - DatasourceSearchResponse searchByContactemail(final String contactemail, final int page, final int size) throws DsmException { - final RequestSort sort = RequestSort.id; - final RequestSortOrder order = RequestSortOrder.ASCENDING; - final RequestFilter filter = new RequestFilter(); - filter.put(FilterName.contactemail, contactemail); - - final StopWatch stop = StopWatch.createStarted(); - final DatasourceSearchResponse rsp = dsmCore.search(sort, order, filter, page, size); - return prepareResponse(page, size, stop, rsp); - } - - @Deprecated - @ApiOperation(value = "search datasources by country", notes = "Returns list of Datasource details.", tags = { D }) - @ApiResponses(value = { - @ApiResponse(code = 200, message = "OK", response = DatasourceSearchResponse.class), - @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) - @RequestMapping(value = "/ds/search/country/{page}/{size}", produces = { "application/json" }, method = RequestMethod.GET) - DatasourceSearchResponse searchByCountry(final String country, final Boolean managed, final int page, final int size) throws DsmException { - final RequestSort sort = RequestSort.id; - final RequestSortOrder order = RequestSortOrder.ASCENDING; - final RequestFilter filter = new RequestFilter(); - filter.put(FilterName.country, country); - filter.put(FilterName.managed, managed); - - final StopWatch stop = StopWatch.createStarted(); - final DatasourceSearchResponse rsp = dsmCore.search(sort, order, filter, page, size); - return prepareResponse(page, size, stop, rsp); - } - - @Deprecated - @ApiOperation(value = "search datasources by registering user", notes = "Returns list of Datasource details.", tags = { D }) - @ApiResponses(value = { - @ApiResponse(code = 200, message = "OK", response = DatasourceSearchResponse.class), - @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) - @RequestMapping(value = "/ds/search/registeredby/{page}/{size}", produces = { "application/json" }, method = RequestMethod.GET) - DatasourceSearchResponse searchByRegisteringUser(final String registeredBy, final int page, final int size) throws DsmException { - final RequestSort sort = RequestSort.id; - final RequestSortOrder order = RequestSortOrder.ASCENDING; - final RequestFilter filter = new RequestFilter(); - filter.put(FilterName.registeredby, registeredBy); - - final StopWatch stop = StopWatch.createStarted(); - final DatasourceSearchResponse rsp = dsmCore.search(sort, order, filter, page, size); - return prepareResponse(page, size, stop, rsp); - } - // HELPERS - private T prepareResponse(final int page, final int size, final StopWatch stopWatch, final T rsp) { rsp.getHeader() - .setTime(stopWatch.getTime()) - .setPage(page) - .setSize(size); + .setTime(stopWatch.getTime()) + .setPage(page) + .setSize(size); return rsp; } } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmCore.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmCore.java index 8e9139fd..437a58cb 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmCore.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmCore.java @@ -1,19 +1,7 @@ package eu.dnetlib.openaire.dsm; -import static eu.dnetlib.openaire.common.ExporterConstants.BASE_URL; -import static eu.dnetlib.openaire.common.ExporterConstants.COMPLIANCE; -import static eu.dnetlib.openaire.common.ExporterConstants.ENGLISH_NAME; -import static eu.dnetlib.openaire.common.ExporterConstants.LATITUDE; -import static eu.dnetlib.openaire.common.ExporterConstants.LONGITUDE; -import static eu.dnetlib.openaire.common.ExporterConstants.OAI_SET; -import static eu.dnetlib.openaire.common.ExporterConstants.OFFICIAL_NAME; -import static eu.dnetlib.openaire.common.ExporterConstants.PLATFORM; -import static eu.dnetlib.openaire.common.ExporterConstants.REMOVABLE; -import static eu.dnetlib.openaire.common.ExporterConstants.TIMEZONE; -import static eu.dnetlib.openaire.common.ExporterConstants.TYPOLOGY; import static eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils.asDbEntry; import static eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils.asDetails; -import static eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils.asMapOfChanges; import static eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils.copyNonNullProperties; import static eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils.createId; @@ -21,7 +9,6 @@ import java.nio.charset.Charset; import java.util.Collection; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Queue; import java.util.Set; import java.util.concurrent.CountDownLatch; @@ -34,9 +21,6 @@ import java.util.stream.Stream; import javax.annotation.PostConstruct; -import eu.dnetlib.openaire.dsm.domain.*; -import eu.dnetlib.openaire.dsm.domain.ApiDetailsResponse; -import eu.dnetlib.openaire.dsm.domain.SimpleResponse; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; @@ -49,15 +33,11 @@ import org.springframework.jdbc.core.BeanPropertyRowMapper; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Component; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.collect.Queues; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import com.google.common.xml.XmlEscapers; import eu.dnetlib.OpenaireExporterConfig; import eu.dnetlib.enabling.datasources.common.AggregationInfo; @@ -77,6 +57,19 @@ import eu.dnetlib.openaire.dsm.dao.VocabularyClient; import eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils; import eu.dnetlib.openaire.dsm.dao.utils.IndexDsInfo; import eu.dnetlib.openaire.dsm.dao.utils.IndexRecordsInfo; +import eu.dnetlib.openaire.dsm.domain.AggregationHistoryResponse; +import eu.dnetlib.openaire.dsm.domain.ApiDetails; +import eu.dnetlib.openaire.dsm.domain.ApiDetailsResponse; +import eu.dnetlib.openaire.dsm.domain.DatasourceDetailResponse; +import eu.dnetlib.openaire.dsm.domain.DatasourceDetails; +import eu.dnetlib.openaire.dsm.domain.DatasourceDetailsUpdate; +import eu.dnetlib.openaire.dsm.domain.DatasourceInfo; +import eu.dnetlib.openaire.dsm.domain.DatasourceSnippetResponse; +import eu.dnetlib.openaire.dsm.domain.RegisteredDatasourceInfo; +import eu.dnetlib.openaire.dsm.domain.RequestFilter; +import eu.dnetlib.openaire.dsm.domain.RequestSort; +import eu.dnetlib.openaire.dsm.domain.RequestSortOrder; +import eu.dnetlib.openaire.dsm.domain.SimpleResponse; import eu.dnetlib.openaire.dsm.domain.db.ApiDbEntry; import eu.dnetlib.openaire.dsm.domain.db.DatasourceDbEntry; import eu.dnetlib.openaire.dsm.domain.db.IdentityDbEntry; @@ -120,7 +113,7 @@ public class DsmCore { @PostConstruct public void init() { executor = MoreExecutors.listeningDecorator(new ScheduledThreadPoolExecutor(config.getRequestWorkers(), - new ThreadFactoryBuilder().setNameFormat("dsm-client-%d").build())); + new ThreadFactoryBuilder().setNameFormat("dsm-client-%d").build())); } public List listCountries() throws DsmException { @@ -132,48 +125,16 @@ public class DsmCore { } } - @Deprecated - public DatasourceSearchResponse search(final RequestSort requestSortBy, - final RequestSortOrder order, - final RequestFilter requestFilter, - final int page, - final int size) - throws DsmException { - - try { - final List datasourceInfo = Lists.newArrayList(); - final Queue errors = Queues.newLinkedBlockingQueue(); - final CountDownLatch outerLatch = new CountDownLatch(2); - - final Page dsPage = dsDao.search(requestSortBy, order, requestFilter, page, size); - if (dsPage.getTotalElements() > 0 && dsPage.getNumberOfElements() > 0) { - dsPage.forEach(d -> datasourceInfo.add(enrichDatasourceInfo(asDetails(d), outerLatch, errors))); - waitLatch(outerLatch, errors, config.getRequestTimeout()); - } - - if (!errors.isEmpty()) { - // TODO report on error metrics - errors.forEach(log::error); - } - return ResponseUtils.searchResponse(datasourceInfo, dsPage.getTotalElements()); - } catch (final Throwable e) { - log.error("error searching datasources", e); - throw e; - } - } - public DatasourceDetailResponse searchDsDetails(final RequestSort requestSortBy, - final RequestSortOrder order, - final RequestFilter requestFilter, - final int page, - final int size) - throws DsmException { + final RequestSortOrder order, + final RequestFilter requestFilter, + final int page, + final int size) + throws DsmException { try { final Page dsPage = dsDao.search(requestSortBy, order, requestFilter, page, size); - return ResponseUtils.detailsResponse( - dsPage.map(d -> asDetails(d)).getContent(), - dsPage.getTotalElements()); + return ResponseUtils.detailsResponse(dsPage.map(d -> asDetails(d)).getContent(), dsPage.getTotalElements()); } catch (final Throwable e) { log.error("error searching datasources", e); throw e; @@ -181,16 +142,14 @@ public class DsmCore { } public DatasourceSnippetResponse searchSnippet(final RequestSort requestSortBy, - final RequestSortOrder order, - final RequestFilter requestFilter, - final int page, - final int size) - throws DsmException { + final RequestSortOrder order, + final RequestFilter requestFilter, + final int page, + final int size) + throws DsmException { try { final Page dsPage = dsDao.search(requestSortBy, order, requestFilter, page, size); - return ResponseUtils.snippetResponse( - dsPage.map(DsmMappingUtils::asSnippetExtended).getContent(), - dsPage.getTotalElements()); + return ResponseUtils.snippetResponse(dsPage.map(DsmMappingUtils::asSnippetExtended).getContent(), dsPage.getTotalElements()); } catch (final Throwable e) { log.error("error searching datasources", e); throw e; @@ -198,16 +157,14 @@ public class DsmCore { } public DatasourceSnippetResponse searchRegistered(final RequestSort requestSortBy, - final RequestSortOrder order, - final RequestFilter requestFilter, - final int page, - final int size) - throws DsmException { + final RequestSortOrder order, + final RequestFilter requestFilter, + final int page, + final int size) + throws DsmException { try { final Page dsPage = dsDao.searchRegistered(requestSortBy, order, requestFilter, page, size); - return ResponseUtils.snippetResponse( - dsPage.map(DsmMappingUtils::asSnippetExtended).getContent(), - dsPage.getTotalElements()); + return ResponseUtils.snippetResponse(dsPage.map(DsmMappingUtils::asSnippetExtended).getContent(), dsPage.getTotalElements()); } catch (final Throwable e) { log.error("error searching datasources", e); throw e; @@ -227,8 +184,8 @@ public class DsmCore { try { final List apis = dsDao.getApis(dsId); final List api = apis.stream() - .map(DsmMappingUtils::asDetails) - .collect(Collectors.toList()); + .map(DsmMappingUtils::asDetails) + .collect(Collectors.toList()); return ResponseUtils.apiResponse(api, api.size()); } catch (final Throwable e) { log.error(String.format("error searching datasource api %s", dsId), e); @@ -239,17 +196,6 @@ public class DsmCore { public void setManaged(final String dsId, final boolean managed) throws DsmException { log.info(String.format("updated ds '%s' managed with '%s'", dsId, managed)); dsDao.setManaged(dsId, managed); - final List apis = dsDao.getApis(dsId); - for (final ApiDbEntry a : apis) { - setApiRemovable(dsId, a.getId(), true); - } - } - - protected void setApiRemovable(final String dsId, final String apiId, final boolean removable) { - log.info(String.format("updated api '%s' removable with '%s'", apiId, removable)); - final Map changes = Maps.newHashMap(); - changes.put(REMOVABLE, String.valueOf(removable)); - isClient.updateAPIField(dsId, apiId, changes); } public boolean isManaged(final String dsId) throws DsmException { @@ -263,7 +209,6 @@ public class DsmCore { public void save(final DatasourceDetails d) throws DsmException { try { dsDao.saveDs(asDbEntry(d)); - isClient.registerDS(d); } catch (final Throwable e) { log.error(ExceptionUtils.getStackTrace(e)); throw e; @@ -281,10 +226,10 @@ public class DsmCore { final DatasourceDbEntry update = asDbEntry(d); if (d.getIdentities() != null) { final Set identities = new HashSet<>( - Stream.of(update.getIdentities(), dbEntry.getIdentities()) - .flatMap(Collection::stream) - .collect(Collectors.toMap(i -> i.getIssuertype() + i.getPid(), Function.identity(), (i1, i2) -> i1)) - .values()); + Stream.of(update.getIdentities(), dbEntry.getIdentities()) + .flatMap(Collection::stream) + .collect(Collectors.toMap(i -> i.getIssuertype() + i.getPid(), Function.identity(), (i1, i2) -> i1)) + .values()); copyNonNullProperties(update, dbEntry); dbEntry.setIdentities(identities); } else { @@ -292,79 +237,12 @@ public class DsmCore { } dsDao.saveDs(dbEntry); - isClient.updateDatasourceFields(d.getId(), asMapOfChanges(d)); } catch (final Throwable e) { log.error(ExceptionUtils.getStackTrace(e)); throw e; } } - @Deprecated - public void updateDatasourcename(final String dsId, final String officialname, final String englishname) throws DsmException { - log.info(String.format("updated datasource '%s' with officialname '%s' and englishname '%s'", dsId, officialname, englishname)); - dsDao.updateName(dsId, officialname, englishname); - - final Map changes = Maps.newHashMap(); - changes.put(OFFICIAL_NAME, XmlEscapers.xmlContentEscaper().escape(officialname)); - changes.put(ENGLISH_NAME, XmlEscapers.xmlContentEscaper().escape(englishname)); - isClient.updateDatasourceFields(dsId, changes); - } - - @Deprecated - public void updateDatasourceLogoUrl(final String dsId, final String logourl) throws DsmException { - log.info(String.format("updated datasource '%s' with logo URL '%s'", dsId, logourl)); - - dsDao.updateLogoUrl(dsId, logourl); - } - - @Deprecated - public void updateCoordinates(final String dsId, final Double latitude, final Double longitude) throws DsmException { - log.info(String.format("updated datasource '%s' with coordinates Lat:'%s', Lon:'%s'", dsId, latitude, longitude)); - dsDao.updateCoordinates(dsId, latitude, longitude); - - final Map changes = Maps.newHashMap(); - changes.put(LATITUDE, XmlEscapers.xmlContentEscaper().escape(String.valueOf(latitude))); - changes.put(LONGITUDE, XmlEscapers.xmlContentEscaper().escape(String.valueOf(longitude))); - isClient.updateDatasourceFields(dsId, changes); - } - - @Deprecated - public void updateTimezone(final String dsId, final String timezone) throws DsmException { - log.info(String.format("updated datasource '%s' timezone with '%s'", dsId, timezone)); - dsDao.updateTimezone(dsId, timezone); - - final Map changes = Maps.newHashMap(); - changes.put(TIMEZONE, XmlEscapers.xmlContentEscaper().escape(timezone)); - isClient.updateDatasourceFields(dsId, changes); - } - - @Deprecated - public void updateDsTypology(final String dsId, final String typology) throws DsmException { - log.info(String.format("updated datasource '%s' typology with '%s'", dsId, typology)); - dsDao.updateTypology(dsId, typology); - - final Map changes = Maps.newHashMap(); - changes.put(TYPOLOGY, XmlEscapers.xmlContentEscaper().escape(typology)); - isClient.updateDatasourceFields(dsId, changes); - } - - @Deprecated - public void updateDsRegisteringUser(final String dsId, final String registeredBy) throws DsmException { - log.info(String.format("setting datasource '%s' registering user with '%s'", dsId, registeredBy)); - dsDao.updateRegisteringUser(dsId, registeredBy); - } - - @Deprecated - public void updateDsPlatform(final String dsId, final String platform) throws DsmException { - log.info(String.format("updated datasource '%s' platform with '%s'", dsId, platform)); - dsDao.updatePlatform(dsId, platform); - - final Map changes = Maps.newHashMap(); - changes.put(PLATFORM, XmlEscapers.xmlContentEscaper().escape(platform)); // this is not a typo, Repository profiles map the platform - // in the DATASOURCE_TYPE field. - isClient.updateDatasourceFields(dsId, changes); - } - // TODO remove if unused public void deleteDs(final String dsId) throws DsmException { log.info(String.format("deleted datasource '%s'", dsId)); @@ -374,35 +252,17 @@ public class DsmCore { // API public void updateApiOaiSet(final String dsId, final String apiId, final String oaiSet) throws DsmException { - final boolean insert = dsDao.upsertApiOaiSet(apiId, oaiSet); - final Map changes = Maps.newHashMap(); - changes.put(OAI_SET, XmlEscapers.xmlContentEscaper().escape(oaiSet)); - - if (!insert) { - isClient.updateAPIField(dsId, apiId, changes); - } else { - isClient.addAPIAttribute(dsId, apiId, changes); - } + dsDao.upsertApiOaiSet(apiId, oaiSet); } public void updateApiBaseurl(final String dsId, final String apiId, final String baseUrl) throws DsmException { log.info(String.format("updated api '%s' baseurl with '%s'", apiId, baseUrl)); dsDao.updateApiBaseUrl(apiId, baseUrl); - - final Map changes = Maps.newHashMap(); - changes.put(BASE_URL, XmlEscapers.xmlContentEscaper().escape(baseUrl)); - - isClient.updateAPIField(dsId, apiId, changes); } public void updateApiCompatibility(final String dsId, final String apiId, final String compliance, final boolean override) throws DsmException { log.info(String.format("updated api '%s' compliance with '%s'", apiId, compliance)); dsDao.updateCompliance(null, apiId, compliance, override); - - final Map changes = Maps.newHashMap(); - changes.put(COMPLIANCE, XmlEscapers.xmlAttributeEscaper().escape(compliance)); - - isClient.updateAPIField(dsId, apiId, changes); } public void addApi(final ApiDetails api) throws DsmException { @@ -410,14 +270,11 @@ public class DsmCore { api.setId(createId(api)); log.info(String.format("missing api id, created '%s'", api.getId())); } - dsDao.addApi(asDbEntry(api)); - isClient.registerAPI(api); } public void deleteApi(final String apiId) throws DsmForbiddenException, DsmNotFoundException { // TODO handle the api removal in case of associated workflows. - isClient.removeAPI(apiId); dsDao.deleteApi(null, apiId); } @@ -438,9 +295,9 @@ public class DsmCore { } private void getAggregationHistory(final String dsId, - final CountDownLatch outerLatch, - final Queue errors, - final DatasourceInfo datasourceInfo) { + final CountDownLatch outerLatch, + final Queue errors, + final DatasourceInfo datasourceInfo) { Futures.addCallback(executor.submit(() -> mongoLoggerClient.getAggregationHistory(dsId)), new FutureCallback>() { @Override @@ -462,15 +319,15 @@ public class DsmCore { datasourceInfo.setAggregationHistory(info); if (!info.isEmpty()) { datasourceInfo - .setLastCollection(info.stream().filter(a -> AggregationStage.COLLECT.equals(a.getAggregationStage())).findFirst().get()) - .setLastTransformation(info.stream().filter(a -> AggregationStage.TRANSFORM.equals(a.getAggregationStage())).findFirst().get()); + .setLastCollection(info.stream().filter(a -> AggregationStage.COLLECT.equals(a.getAggregationStage())).findFirst().get()) + .setLastTransformation(info.stream().filter(a -> AggregationStage.TRANSFORM.equals(a.getAggregationStage())).findFirst().get()); } } private void getIndexDsInfo(final String dsId, - final CountDownLatch outerLatch, - final Queue errors, - final DatasourceInfo datasourceInfo) { + final CountDownLatch outerLatch, + final Queue errors, + final DatasourceInfo datasourceInfo) { Futures.addCallback(executor.submit(() -> isClient.calculateCurrentIndexDsInfo()), new FutureCallback() { @Override @@ -483,9 +340,9 @@ public class DsmCore { @Override public void onSuccess(final IndexRecordsInfo info) { datasourceInfo - .setIndexRecords(info.getTotal()) - .setFundedContent(info.getFunded()) - .setLastIndexingDate(info.getDate()); + .setIndexRecords(info.getTotal()) + .setFundedContent(info.getFunded()) + .setLastIndexingDate(info.getDate()); innerLatch.countDown(); } @@ -538,7 +395,7 @@ public class DsmCore { public SimpleResponse searchRecentRegistered(final int size) throws Throwable { try { final String sql = - IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/recent_registered_datasources.sql.st"), Charset.defaultCharset()); + IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/recent_registered_datasources.sql.st"), Charset.defaultCharset()); final List list = jdbcTemplate.query(sql, BeanPropertyRowMapper.newInstance(RegisteredDatasourceInfo.class), size); @@ -548,28 +405,34 @@ public class DsmCore { throw e; } } + public Long countRegisteredAfter(final String fromDate, final String typologyFilter) throws Throwable { try { if (StringUtils.isNotBlank(typologyFilter)) { final String sql = - IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate_typology.st.sql"), Charset.defaultCharset()); + IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate_typology.st.sql"), Charset + .defaultCharset()); - return jdbcTemplate.queryForObject(sql, new Object[] { fromDate, typologyFilter + "%" }, Long.class); + return jdbcTemplate.queryForObject(sql, new Object[] { + fromDate, typologyFilter + "%" + }, Long.class); } else { final String sql = - IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate.st.sql"), Charset.defaultCharset()); + IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate.st.sql"), Charset + .defaultCharset()); - return jdbcTemplate.queryForObject(sql, new Object[] { fromDate }, Long.class); + return jdbcTemplate.queryForObject(sql, new Object[] { + fromDate + }, Long.class); } - } catch (final Throwable e) { log.error("error searching recent datasources", e); throw e; } } - public AggregationHistoryResponse aggregationhistory(String dsId) throws DsmException { + public AggregationHistoryResponse aggregationhistory(final String dsId) throws DsmException { final List history = mongoLoggerClient.getAggregationHistory(dsId); final AggregationHistoryResponse rsp = new AggregationHistoryResponse(history); rsp.setHeader(ResponseUtils.header(history.size())); diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/utils/DsmMappingUtils.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/utils/DsmMappingUtils.java index bc65a0a2..6d7a253d 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/utils/DsmMappingUtils.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/utils/DsmMappingUtils.java @@ -1,18 +1,7 @@ package eu.dnetlib.openaire.dsm.dao.utils; -import static eu.dnetlib.openaire.common.ExporterConstants.ADMIN_INFO; -import static eu.dnetlib.openaire.common.ExporterConstants.ENGLISH_NAME; -import static eu.dnetlib.openaire.common.ExporterConstants.LATITUDE; -import static eu.dnetlib.openaire.common.ExporterConstants.LONGITUDE; -import static eu.dnetlib.openaire.common.ExporterConstants.OFFICIAL_NAME; -import static eu.dnetlib.openaire.common.ExporterConstants.PLATFORM; -import static eu.dnetlib.openaire.common.ExporterConstants.TIMEZONE; -import static eu.dnetlib.openaire.common.ExporterConstants.TYPOLOGY; -import static eu.dnetlib.openaire.common.ExporterConstants.WEBSITE_URL; - import java.sql.Date; import java.util.HashSet; -import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -25,15 +14,12 @@ import org.springframework.beans.BeanWrapper; import org.springframework.beans.BeanWrapperImpl; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.Maps; -import com.google.common.xml.XmlEscapers; import eu.dnetlib.data.transform.xml.AbstractDNetXsltFunctions; import eu.dnetlib.miscutils.datetime.DateUtils; import eu.dnetlib.openaire.dsm.domain.ApiDetails; import eu.dnetlib.openaire.dsm.domain.DatasourceDetails; import eu.dnetlib.openaire.dsm.domain.DatasourceDetailsUpdate; -import eu.dnetlib.openaire.dsm.domain.DatasourceSnippet; import eu.dnetlib.openaire.dsm.domain.DatasourceSnippetExtended; import eu.dnetlib.openaire.dsm.domain.OrganizationDetails; import eu.dnetlib.openaire.dsm.domain.db.ApiDbEntry; @@ -57,15 +43,6 @@ public class DsmMappingUtils { return details.setOpenaireId(asOpenaireId(details.getId())); } - @Deprecated - public static DatasourceSnippet asSnippet(final DatasourceDbEntry d) { - final DatasourceSnippet ds = new DatasourceSnippet(); - ds.setId(d.getId()); - ds.setOfficialname(d.getOfficialname()); - ds.setEnglishname(d.getEnglishname()); - return ds; - } - public static DatasourceSnippetExtended asSnippetExtended(final DatasourceDbEntry d) { final DatasourceSnippetExtended ds = new DatasourceSnippetExtended(); ds.setId(d.getId()); @@ -88,11 +65,11 @@ public class DsmMappingUtils { private static OrganizationDetails asOrganizationDetail(final OrganizationDbEntry o) { return new OrganizationDetails() - .setCountry(o.getCountry()) - .setLegalname(o.getLegalname()) - .setLegalshortname(o.getLegalshortname()) - .setWebsiteurl(o.getWebsiteurl()) - .setLogourl(o.getLogourl()); + .setCountry(o.getCountry()) + .setLegalname(o.getLegalname()) + .setLegalshortname(o.getLegalshortname()) + .setWebsiteurl(o.getWebsiteurl()) + .setLogourl(o.getLogourl()); } public static ApiDetails asDetails(final ApiDbEntry d) { @@ -112,7 +89,7 @@ public class DsmMappingUtils { final DatasourceDbEntry dbe = _convert(d, DatasourceDbEntry.class); if (dbe.getOrganizations() != null) { dbe.getOrganizations().forEach(o -> { - String prefix = StringUtils.isNotBlank(dbe.getNamespaceprefix()) ? dbe.getNamespaceprefix() : dbe.getId(); + final String prefix = StringUtils.isNotBlank(dbe.getNamespaceprefix()) ? dbe.getNamespaceprefix() : dbe.getId(); o.setId(prefix + ID_SEPARATOR + o.getLegalname()); if (o.getDateofcollection() == null) { o.setDateofcollection(new Date(System.currentTimeMillis())); @@ -203,12 +180,12 @@ public class DsmMappingUtils { final Element iface = DocumentHelper.createElement("INTERFACE"); iface.addAttribute("active", String.valueOf(api.getActive())) - .addAttribute("compliance", api.getCompatibility()) - .addAttribute("contentDescription", api.getContentdescription()) - .addAttribute("id", api.getId()) - .addAttribute("label", String.format("%s (%s)", api.getTypology(), api.getCompatibility())) - .addAttribute("removable", String.valueOf(api.getRemovable())) - .addAttribute("typology", api.getTypology()); + .addAttribute("compliance", api.getCompatibility()) + .addAttribute("contentDescription", api.getContentdescription()) + .addAttribute("id", api.getId()) + .addAttribute("label", String.format("%s (%s)", api.getTypology(), api.getCompatibility())) + .addAttribute("removable", String.valueOf(api.getRemovable())) + .addAttribute("typology", api.getTypology()); iface.addElement("ACCESS_PROTOCOL").setText(api.getProtocol()); if (api.getApiParams() != null) { final Element accessProtocol = (Element) iface.selectSingleNode("./ACCESS_PROTOCOL"); @@ -277,36 +254,4 @@ public class DsmMappingUtils { return emptyNames.toArray(result); } - public static Map asMapOfChanges(final DatasourceDetailsUpdate d) { - final Map changes = Maps.newHashMap(); - - if (d.getContactemail() != null) { - changes.put(ADMIN_INFO, XmlEscapers.xmlContentEscaper().escape(d.getContactemail())); - } - if (d.getEnglishname() != null) { - changes.put(ENGLISH_NAME, XmlEscapers.xmlContentEscaper().escape(d.getEnglishname())); - } - if (d.getOfficialname() != null) { - changes.put(OFFICIAL_NAME, XmlEscapers.xmlContentEscaper().escape(d.getOfficialname())); - } - if (d.getTimezone() != null) { - changes.put(TIMEZONE, XmlEscapers.xmlContentEscaper().escape(d.getTimezone())); - } - if (d.getLatitude() != null) { - changes.put(LATITUDE, XmlEscapers.xmlContentEscaper().escape(String.valueOf(d.getLatitude()))); - } - if (d.getLongitude() != null) { - changes.put(LONGITUDE, XmlEscapers.xmlContentEscaper().escape(String.valueOf(d.getLongitude()))); - } - if (d.getPlatform() != null) { - changes.put(PLATFORM, XmlEscapers.xmlContentEscaper().escape(d.getPlatform())); - } - if (d.getTypology() != null) { - changes.put(TYPOLOGY, XmlEscapers.xmlContentEscaper().escape(d.getTypology())); - } - if (d.getWebsiteurl() != null) { - changes.put(WEBSITE_URL, XmlEscapers.xmlContentEscaper().escape(d.getWebsiteurl())); - } - return changes; - } } diff --git a/apps/dnet-exporter-api/src/main/resources/dnet_dsm.data.sql b/apps/dnet-exporter-api/src/main/resources/dnet_dsm.data.sql deleted file mode 100644 index ee3ec218..00000000 --- a/apps/dnet-exporter-api/src/main/resources/dnet_dsm.data.sql +++ /dev/null @@ -1,176 +0,0 @@ -INSERT INTO dsm_identities(pid, issuertype) VALUES ('77', 'roar'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('680', 'roar'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('260', 'roar'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('637', 'roar'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('528', 'roar'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('577', 'roar'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('566', 'roar'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('241', 'roar'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('491', 'roar'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('324', 'roar'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('375', 'roar'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('152', 'roar'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('1025', 'roar'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('10.13039/100004440', 'doi'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('10.13039/100000001', 'doi'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('10.13039/100000002', 'doi'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('10.13039/501100007601', 'doi'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('10.13039/501100001871', 'doi'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('10.13039/501100008982', 'doi'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('10.13039/501100000925', 'doi'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('10.13039/501100000923', 'doi'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('10.13039/501100001602', 'doi'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('10.13039/501100002428', 'doi'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('10.13039/501100000690', 'doi'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('10.13039/501100002341', 'doi'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 2242 8989', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0004 5900 900X', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 2169 9189', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 2181 2823', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0004 0427 7672', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 1958 7073', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 2169 1945', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 2154 0709', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 2176 1982', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 0789 9694', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 2228 3249', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 2176 7727', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0004 0611 9213', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0004 0609 4140', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0004 4663 8325', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 1092 7772', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0004 0452 5752', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 2297 5165', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 0672 3101', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 1957 0992', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 1091 8438', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 0507 0997', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 0665 7300', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 0685 2712', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0004 0647 6886', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 0943 9683', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 2096 9829', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 1957 9997', 'isni'); - -INSERT INTO funder_identity(funder, pid) VALUES ('H2020', '680'); -INSERT INTO funder_identity(funder, pid) VALUES ('H2020', '10.13039/501100007601'); -INSERT INTO funder_identity(funder, pid) VALUES ('H2020', '0000 0001 2242 8989'); -INSERT INTO funder_identity(funder, pid) VALUES ('FP7', '0000 0004 5900 900X'); -INSERT INTO funder_identity(funder, pid) VALUES ('FCT', '260'); -INSERT INTO funder_identity(funder, pid) VALUES ('FCT', '10.13039/501100001871'); -INSERT INTO funder_identity(funder, pid) VALUES ('FCT', '0000 0001 2169 9189'); -INSERT INTO funder_identity(funder, pid) VALUES ('FCT', '0000 0001 2181 2823'); -INSERT INTO funder_identity(funder, pid) VALUES ('WT', '637'); -INSERT INTO funder_identity(funder, pid) VALUES ('WT', '10.13039/100004440'); -INSERT INTO funder_identity(funder, pid) VALUES ('WT', '0000 0004 0427 7672'); -INSERT INTO funder_identity(funder, pid) VALUES ('NSF', '528'); -INSERT INTO funder_identity(funder, pid) VALUES ('NSF', '10.13039/501100008982'); -INSERT INTO funder_identity(funder, pid) VALUES ('NSF', '10.13039/100000001'); -INSERT INTO funder_identity(funder, pid) VALUES ('NSF', '0000 0001 1958 7073'); -INSERT INTO funder_identity(funder, pid) VALUES ('NSF', '0000 0001 2169 1945'); -INSERT INTO funder_identity(funder, pid) VALUES ('NSF', '0000 0001 2154 0709'); -INSERT INTO funder_identity(funder, pid) VALUES ('NHMRC', '577'); -INSERT INTO funder_identity(funder, pid) VALUES ('NHMRC', '10.13039/501100000925'); -INSERT INTO funder_identity(funder, pid) VALUES ('NHMRC', '0000 0001 2176 1982'); -INSERT INTO funder_identity(funder, pid) VALUES ('NHMRC', '0000 0001 0789 9694'); -INSERT INTO funder_identity(funder, pid) VALUES ('NHMRC', '0000 0001 2228 3249'); -INSERT INTO funder_identity(funder, pid) VALUES ('NHMRC', '0000 0001 2176 7727'); -INSERT INTO funder_identity(funder, pid) VALUES ('NHMRC', '0000 0001 0789 9694'); -INSERT INTO funder_identity(funder, pid) VALUES ('NHMRC', '0000 0001 2228 3249'); -INSERT INTO funder_identity(funder, pid) VALUES ('ARC', '566'); -INSERT INTO funder_identity(funder, pid) VALUES ('ARC', '10.13039/501100000923'); -INSERT INTO funder_identity(funder, pid) VALUES ('ARC', '0000 0004 0611 9213'); -INSERT INTO funder_identity(funder, pid) VALUES ('MSES', '0000 0004 0609 4140'); -INSERT INTO funder_identity(funder, pid) VALUES ('CSF', '0000 0004 4663 8325'); -INSERT INTO funder_identity(funder, pid) VALUES ('NWO', '241'); -INSERT INTO funder_identity(funder, pid) VALUES ('NWO', '0000 0001 1092 7772'); -INSERT INTO funder_identity(funder, pid) VALUES ('SFI', '10.13039/501100001602'); -INSERT INTO funder_identity(funder, pid) VALUES ('SFI', '0000 0004 0452 5752'); -INSERT INTO funder_identity(funder, pid) VALUES ('NIH', '10.13039/100000002'); -INSERT INTO funder_identity(funder, pid) VALUES ('NIH', '0000 0001 2297 5165'); -INSERT INTO funder_identity(funder, pid) VALUES ('NIH', '491'); -INSERT INTO funder_identity(funder, pid) VALUES ('SNSF', '324'); -INSERT INTO funder_identity(funder, pid) VALUES ('SNSF', '0000 0001 0672 3101'); -INSERT INTO funder_identity(funder, pid) VALUES ('SNSF', '0000 0001 1957 0992'); -INSERT INTO funder_identity(funder, pid) VALUES ('FWF', '77'); -INSERT INTO funder_identity(funder, pid) VALUES ('FWF', '10.13039/501100002428'); -INSERT INTO funder_identity(funder, pid) VALUES ('FWF', '0000 0001 1091 8438'); -INSERT INTO funder_identity(funder, pid) VALUES ('RCUK', '375'); -INSERT INTO funder_identity(funder, pid) VALUES ('RCUK', '10.13039/501100000690'); -INSERT INTO funder_identity(funder, pid) VALUES ('RCUK', '0000 0001 0507 0997'); -INSERT INTO funder_identity(funder, pid) VALUES ('TBT', '0000 0001 0665 7300'); -INSERT INTO funder_identity(funder, pid) VALUES ('TBT', '0000 0001 0685 2712'); -INSERT INTO funder_identity(funder, pid) VALUES ('AFF', '1025'); -INSERT INTO funder_identity(funder, pid) VALUES ('AFF', '10.13039/501100002341'); -INSERT INTO funder_identity(funder, pid) VALUES ('AFF', '0000 0004 0647 6886'); -INSERT INTO funder_identity(funder, pid) VALUES ('CONICYT', '0000 0001 0943 9683'); -INSERT INTO funder_identity(funder, pid) VALUES ('DFG', '152'); -INSERT INTO funder_identity(funder, pid) VALUES ('DFG', '0000 0001 2096 9829'); -INSERT INTO funder_identity(funder, pid) VALUES ('DFG', '0000 0001 1957 9997'); - - -INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) -VALUES ('H2020', 'European Commission - Horizon 2020', 'H2020', 'EU', 'http://ec.europa.eu/research/participants/data/ref/h2020/grants_manual/hi/oa_pilot/h2020-hi-oa-pilot-guide_en.pdf', 'OA mandate for publications; OA to research data by default', '2015-08-24'); - -INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) -VALUES ('FP7', 'European Commission - 7th Framework program', 'FP7', 'EU', 'http://ec.europa.eu/research/fp7', 'OA advised for publications; 2012 – 2013 (FP7)Special Clause 39 ERC on Open Access applies', '2013-05-07'); - -INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) -VALUES ('FCT', 'Portuguese Foundation for Science and Technology', 'FCT', 'PT', 'https://www.fct.pt/acessoaberto/index.phtml.en', 'OA mandate for publications; OA to research data advised', '2015-02-27'); - -INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) -VALUES ('WT', 'Wellcome Trust', 'WT', '', 'https://wellcome.ac.uk/funding/managing-grant/open-access-policy', 'OA mandate for articles, monographs and book chapters; CC-BY licence required', '2013-05-07'); - -INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) -VALUES ('NSF', 'National Science Foundation', 'NSF', 'US', 'https://www.nsf.gov/news/special_reports/public_access', 'Version of record or AAM of articles, conferences and data deposited into the repository; persistent identifier to full text on publisher’s website', '2016-03-04'); - -INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) -VALUES ('NHMRC', 'National Health and Medical Research Council', 'NHMRC', 'AU', 'https://www.nhmrc.gov.au/grants-funding/policy/nhmrc-open-access-policy', 'OA mandate for publications', '2015-08-24'); - -INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) -VALUES ('ARC', 'Australian Research Council', 'ARC', 'AU', 'http://www.arc.gov.au/arc-open-access-policy', 'OA mandate for research outputs, except for research data', '2015-08-24'); - -INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) -VALUES ('MESTD', 'Ministry of Education, Science and Technological Development', 'MESTD', 'RS', 'NOAD', 'NOAD', '2017-01-23'); - -INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) -VALUES ('MSES', 'Ministry of Science Education and Sport', 'MSES/MZOS', 'HR', 'NOAD', 'NOAD', '2015-09-14'); - -INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) -VALUES ('CSF', 'Croatian Science Foundation', 'CSF/HRZZ', 'HR', 'NOAD', 'NOAD', '2015-09-14'); - -INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) -VALUES ('NWO', 'Netherlands Organisation for Scientific Research', 'NWO', 'NL', 'NOAD', 'NOAD', '2016-06-23'); - -INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) -VALUES ('SFI', 'Science Foundation Ireland', 'SFI', 'IE', 'http://www.sfi.ie/resources/open-access-dec-10.pdf', 'OA mandate to publications, conference proceedings and technical reports', '2015-07-21'); - -INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) -VALUES ('NIH', 'National Institute of Health', 'NIH', 'US', 'https://publicaccess.nih.gov/policy.htm', 'OA to articles', '2017-09-21'); - -INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) -VALUES ('SNSF', 'Swiss National Science Foundation', 'SNSF', 'CH', 'NOAD', 'NOAD', '2016-11-16'); - -INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) -VALUES ('FWF', 'Austrian Science Fund', 'FWF', 'AT', 'NOAD', 'NOAD', '2017-09-21'); - -INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) -VALUES ('RCUK', 'Research Council UK', 'RCUK', 'UK', 'NOAD', 'NOAD', '2017-11-02'); - -INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) -VALUES ('TBT', 'Scientific and Technological Research Council of Turkey', 'Tubitak', 'TR', 'NOAD', 'NOAD', '2017-11-02'); - -INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) -VALUES ('DFG', 'German Research Foundation', 'DFG', 'DE', 'NOAD', 'NOAD', '2017-09-29'); - -INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) -VALUES ('TARA', 'Tara Expedition Foundation', 'TARA', 'FR', 'NOAD', 'NOAD', '2016-12-02'); - -INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) -VALUES ('AFF', 'Academy of Finland', 'AFF', 'FI', 'NOAD', 'NOAD', '2017-09-21'); - -INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) -VALUES ('CONICYT', 'Comisión Nacional de Investigación Científica y Tecnológica', 'CONICYT', 'CL', 'http://www.lareferencia.info', '', '2017-09-08'); - -INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) -VALUES ('SGOV', 'Ministry of Economy, Industry and Competitiveness', 'SGOV', 'ES', 'NOAD', 'NOAD', '2017-09-14'); diff --git a/apps/dnet-exporter-api/src/main/resources/dnet_dsm.sql b/apps/dnet-exporter-api/src/main/resources/dnet_dsm.sql deleted file mode 100644 index b40ee542..00000000 --- a/apps/dnet-exporter-api/src/main/resources/dnet_dsm.sql +++ /dev/null @@ -1,510 +0,0 @@ - -CREATE TABLE dsm_api ( - id character varying(255) NOT NULL, - protocol character varying(255), - datasource character varying(255), - contentdescription character varying(255) DEFAULT 'metadata'::character varying, - active boolean DEFAULT false, - removable boolean DEFAULT false, - typology character varying(255) DEFAULT 'UNKNOWN'::character varying, - compatibility character varying(255) DEFAULT 'UNKNOWN'::character varying, - metadata_identifier_path character varying(512) DEFAULT NULL::character varying, - last_collection_total integer, - last_collection_date timestamp without time zone, - last_collection_mdid character varying(255) DEFAULT NULL::character varying, - last_aggregation_total integer, - last_aggregation_date timestamp without time zone, - last_aggregation_mdid character varying(255) DEFAULT NULL::character varying, - last_download_total integer, - last_download_date timestamp without time zone, - last_download_objid character varying(255) DEFAULT NULL::character varying, - last_validation_job character varying(255) DEFAULT NULL::character varying, - baseurl text, - _dnet_resource_identifier_ character varying(2048) DEFAULT ((('temp_'::text || md5((clock_timestamp())::text)) || '_'::text) || md5((random())::text)), - compatibility_override character varying(255) DEFAULT NULL::character varying -); - - -ALTER TABLE public.dsm_api OWNER TO dnetapi; - - -CREATE TABLE dsm_datasource_organization ( - datasource character varying(255) NOT NULL, - organization character varying(255) NOT NULL, - _dnet_resource_identifier_ character varying(2048) DEFAULT ((('temp_'::text || md5((clock_timestamp())::text)) || '_'::text) || md5((random())::text)) -); - - -ALTER TABLE public.dsm_datasource_organization OWNER TO dnetapi; - --- --- Name: dsm_datasources; Type: TABLE; Schema: public; Owner: dnetapi; Tablespace: --- - -CREATE TABLE dsm_datasources ( - id character varying(255) NOT NULL, - officialname character varying(512) NOT NULL, - englishname character varying(512), - websiteurl character varying(255), - logourl character varying(255), - contactemail character varying(255), - latitude double precision DEFAULT 0.0, - longitude double precision DEFAULT 0.0, - timezone character varying(10) DEFAULT '0.0'::character varying, - namespaceprefix character(12) NOT NULL, - languages text, - od_contenttypes text, - collectedfrom character varying(255), - dateofvalidation date, - optional1 character varying(255), - optional2 character varying(255), - typology character varying(255) NOT NULL, - provenanceaction character varying(255) DEFAULT 'UNKNOWN'::character varying, - dateofcollection date DEFAULT ('now'::text)::date NOT NULL, - platform character varying(255), - activationid character varying(255), - description text, - releasestartdate date, - releaseenddate date, - missionstatementurl character varying(512), - dataprovider boolean, - serviceprovider boolean, - databaseaccesstype character varying(64), - datauploadtype character varying(64), - databaseaccessrestriction character varying(64), - datauploadrestriction character varying(64), - versioning boolean, - citationguidelineurl character varying(512), - qualitymanagementkind character varying(64), - pidsystems text, - certificates text, - aggregator character varying(64) DEFAULT 'OPENAIRE'::character varying NOT NULL, - issn character varying(20), - eissn character varying(20), - lissn character varying(20), - registeredby character varying(255), - subjects text, - managed boolean DEFAULT false, - registrationdate date, - consentTermsOfUse boolean default null, - fullTextDownload boolean default null, - _dnet_resource_identifier_ character varying(2048) DEFAULT ((('temp_'::text || md5((clock_timestamp())::text)) || '_'::text) || md5((random())::text)) -); - - -ALTER TABLE public.dsm_datasources OWNER TO dnetapi; - --- --- Name: dsm_organizations; Type: TABLE; Schema: public; Owner: dnetapi; Tablespace: --- - -CREATE TABLE dsm_organizations ( - id character varying(255) NOT NULL, - legalshortname character varying(255), - legalname character varying(255), - websiteurl text, - logourl character varying(255), - ec_legalbody boolean DEFAULT false, - ec_legalperson boolean DEFAULT false, - ec_nonprofit boolean DEFAULT false, - ec_researchorganization boolean DEFAULT false, - ec_highereducation boolean DEFAULT false, - ec_internationalorganizationeurinterests boolean DEFAULT false, - ec_internationalorganization boolean DEFAULT false, - ec_enterprise boolean DEFAULT false, - ec_smevalidated boolean DEFAULT false, - ec_nutscode boolean DEFAULT false, - country character varying(255), - collectedfrom character varying(255), - optional1 character varying(255), - optional2 character varying(255), - dateofcollection date DEFAULT ('now'::text)::date NOT NULL, - provenanceaction character varying(255) DEFAULT 'UNKNOWN'::character varying, - _dnet_resource_identifier_ character varying(2048) DEFAULT ((('temp_'::text || md5((clock_timestamp())::text)) || '_'::text) || md5((random())::text)), - lastupdate date DEFAULT ('now'::text)::date NOT NULL, - trust double precision DEFAULT 0.9 -); - - -ALTER TABLE public.dsm_organizations OWNER TO dnetapi; - --- --- Name: browse_countries; Type: VIEW; Schema: public; Owner: dnet --- - -CREATE VIEW browse_countries AS - SELECT o.country AS term, - count(*) AS total - FROM (((dsm_api a - LEFT JOIN dsm_datasources d ON (((a.datasource)::text = (d.id)::text))) - LEFT JOIN dsm_datasource_organization dao ON (((d.id)::text = (dao.datasource)::text))) - LEFT JOIN dsm_organizations o ON (((dao.organization)::text = (o.id)::text))) - GROUP BY o.country - ORDER BY count(*) DESC; - - -ALTER TABLE public.browse_countries OWNER TO dnet; - - - -CREATE TABLE dsm_apiparams ( - param character varying(255) NOT NULL, - value text DEFAULT ''::character varying NOT NULL, - api character varying(255) NOT NULL, - _dnet_resource_identifier_ character varying(2048) DEFAULT ((('temp_'::text || md5((clock_timestamp())::text)) || '_'::text) || md5((random())::text)) -); - - -ALTER TABLE public.dsm_apiparams OWNER TO dnetapi; - --- --- Name: dsm_datasource_api; Type: VIEW; Schema: public; Owner: dnet --- - -CREATE OR REPLACE VIEW dsm_datasource_api AS - SELECT - row_number() OVER (ORDER BY a.id) AS rowid, - d.id, - d.officialname, - d.englishname, - d.websiteurl, - d.contactemail, - d.collectedfrom, - d.typology, - d.platform, - d.registeredby, - d.managed, - a.protocol, - a.contentdescription, - a.active, - a.removable, - a.typology AS apitypology, - a.compatibility, - a.baseurl - FROM (dsm_datasources d - LEFT JOIN dsm_api a ON (((d.id)::text = (a.datasource)::text))); - --- --- Name: dsm_datasourcepids; Type: TABLE; Schema: public; Owner: dnetapi; Tablespace: --- - -CREATE TABLE dsm_datasourcepids ( - datasource character varying(255) NOT NULL, - pid character varying(255) NOT NULL, - _dnet_resource_identifier_ character varying(2048) DEFAULT ((('temp_'::text || md5((clock_timestamp())::text)) || '_'::text) || md5((random())::text)) -); - - -ALTER TABLE public.dsm_datasourcepids OWNER TO dnetapi; - --- --- Name: dsm_identities; Type: TABLE; Schema: public; Owner: dnetapi; Tablespace: --- - -CREATE TABLE dsm_identities ( - pid character varying(255) NOT NULL, - issuertype character varying(255), - _dnet_resource_identifier_ character varying(2048) DEFAULT ((('temp_'::text || md5((clock_timestamp())::text)) || '_'::text) || md5((random())::text)) -); - - -ALTER TABLE public.dsm_identities OWNER TO dnetapi; - --- --- Name: dsm_organizationpids; Type: TABLE; Schema: public; Owner: dnetapi; Tablespace: --- - -CREATE TABLE dsm_organizationpids ( - organization character varying(255) NOT NULL, - pid character varying(255) NOT NULL, - _dnet_resource_identifier_ character varying(2048) DEFAULT ((('temp_'::text || md5((clock_timestamp())::text)) || '_'::text) || md5((random())::text)) -); - - -ALTER TABLE public.dsm_organizationpids OWNER TO dnetapi; - --- --- Name: funder_identity; Type: TABLE; Schema: public; Owner: dnet; Tablespace: --- - -CREATE TABLE funder_identity ( - funder character varying(255), - pid character varying(255) -); - - -ALTER TABLE public.funder_identity OWNER TO dnet; - --- --- Name: funders; Type: TABLE; Schema: public; Owner: dnet; Tablespace: --- - -CREATE TABLE funders ( - id character varying(255) NOT NULL, - name character varying(255), - shortname character varying(255), - jurisdiction character varying(255), - websiteurl text, - policy character varying(255), - registrationdate date DEFAULT ('now'::text)::date NOT NULL, - lastupdatedate date -); - - -ALTER TABLE public.funders OWNER TO dnet; - --- --- Name: fundingpaths; Type: TABLE; Schema: public; Owner: dnet; Tablespace: --- - - - - - - - - - - - - - - --- --- Name: project_organization; Type: TABLE; Schema: public; Owner: dnet; Tablespace: --- - ---CREATE TABLE project_organization ( --- participantnumber integer, --- project character varying(255) NOT NULL, --- resporganization character varying(255) NOT NULL, --- semanticclass character varying(255) DEFAULT 'UNKNOWN'::character varying, --- trust double precision DEFAULT 0.9, --- _dnet_resource_identifier_ character varying(2048) DEFAULT ((('temp_'::text || md5((clock_timestamp())::text)) || '_'::text) || md5((random())::text)) ---); - - ---ALTER TABLE public.project_organization OWNER TO dnet; - - - - --- --- Name: dsm_api_pkey; Type: CONSTRAINT; Schema: public; Owner: dnetapi; Tablespace: --- - -ALTER TABLE ONLY dsm_api - ADD CONSTRAINT dsm_api_pkey PRIMARY KEY (id); - - --- --- Name: dsm_apicollection_pkey; Type: CONSTRAINT; Schema: public; Owner: dnetapi; Tablespace: --- - -ALTER TABLE ONLY dsm_apiparams - ADD CONSTRAINT dsm_apicollection_pkey PRIMARY KEY (api, param); - - --- --- Name: dsm_datasourcepids_pkey; Type: CONSTRAINT; Schema: public; Owner: dnetapi; Tablespace: --- - -ALTER TABLE ONLY dsm_datasourcepids - ADD CONSTRAINT dsm_datasourcepids_pkey PRIMARY KEY (datasource, pid); - - --- --- Name: dsm_datasources_namespaceprefix_key; Type: CONSTRAINT; Schema: public; Owner: dnetapi; Tablespace: --- - -ALTER TABLE ONLY dsm_datasources - ADD CONSTRAINT dsm_datasources_namespaceprefix_key UNIQUE (namespaceprefix); - - --- --- Name: dsm_datasources_pkey; Type: CONSTRAINT; Schema: public; Owner: dnetapi; Tablespace: --- - -ALTER TABLE ONLY dsm_datasources - ADD CONSTRAINT dsm_datasources_pkey PRIMARY KEY (id); - - --- --- Name: dsm_identities_pkey; Type: CONSTRAINT; Schema: public; Owner: dnetapi; Tablespace: --- - -ALTER TABLE ONLY dsm_identities - ADD CONSTRAINT dsm_identities_pkey PRIMARY KEY (pid); - - --- --- Name: dsm_organization_datasource_pkey; Type: CONSTRAINT; Schema: public; Owner: dnetapi; Tablespace: --- - -ALTER TABLE ONLY dsm_datasource_organization - ADD CONSTRAINT dsm_organization_datasource_pkey PRIMARY KEY (datasource, organization); - - --- --- Name: dsm_organizationpids_pkey; Type: CONSTRAINT; Schema: public; Owner: dnetapi; Tablespace: --- - -ALTER TABLE ONLY dsm_organizationpids - ADD CONSTRAINT dsm_organizationpids_pkey PRIMARY KEY (organization, pid); - - --- --- Name: dsm_organizations_pkey; Type: CONSTRAINT; Schema: public; Owner: dnetapi; Tablespace: --- - -ALTER TABLE ONLY dsm_organizations - ADD CONSTRAINT dsm_organizations_pkey PRIMARY KEY (id); - - - - - - --- --- Name: dsm_datasources_contactemail_idx; Type: INDEX; Schema: public; Owner: dnetapi; Tablespace: --- - -CREATE INDEX dsm_datasources_contactemail_idx ON dsm_datasources USING btree (contactemail); - - --- --- Name: dsm_datasources_englishname_idx; Type: INDEX; Schema: public; Owner: dnetapi; Tablespace: --- - -CREATE INDEX dsm_datasources_englishname_idx ON dsm_datasources USING btree (englishname); - - --- --- Name: dsm_datasources_managed_idx; Type: INDEX; Schema: public; Owner: dnetapi; Tablespace: --- - -CREATE INDEX dsm_datasources_managed_idx ON dsm_datasources USING btree (managed); - - --- --- Name: dsm_datasources_officialname_idx; Type: INDEX; Schema: public; Owner: dnetapi; Tablespace: --- - -CREATE INDEX dsm_datasources_officialname_idx ON dsm_datasources USING btree (officialname); - - --- --- Name: dsm_datasources_registeredby_idx; Type: INDEX; Schema: public; Owner: dnetapi; Tablespace: --- - -CREATE INDEX dsm_datasources_registeredby_idx ON dsm_datasources USING btree (registeredby); - - --- --- Name: dsm_organizations_country_idx; Type: INDEX; Schema: public; Owner: dnetapi; Tablespace: --- - -CREATE INDEX dsm_organizations_country_idx ON dsm_organizations USING btree (country); - - - - - - - - -ALTER TABLE ONLY dsm_api - ADD CONSTRAINT dsm_api_datasource_fkey FOREIGN KEY (datasource) REFERENCES dsm_datasources(id) ON DELETE CASCADE; - - --- --- Name: dsm_apicollections_api_fkey; Type: FK CONSTRAINT; Schema: public; Owner: dnetapi --- - -ALTER TABLE ONLY dsm_apiparams - ADD CONSTRAINT dsm_apicollections_api_fkey FOREIGN KEY (api) REFERENCES dsm_api(id); - - --- --- Name: dsm_datasource_organization_datasource_fkey; Type: FK CONSTRAINT; Schema: public; Owner: dnetapi --- - -ALTER TABLE ONLY dsm_datasource_organization - ADD CONSTRAINT dsm_datasource_organization_datasource_fkey FOREIGN KEY (datasource) REFERENCES dsm_datasources(id); - - --- --- Name: dsm_datasource_organization_organization_fkey; Type: FK CONSTRAINT; Schema: public; Owner: dnetapi --- - -ALTER TABLE ONLY dsm_datasource_organization - ADD CONSTRAINT dsm_datasource_organization_organization_fkey FOREIGN KEY (organization) REFERENCES dsm_organizations(id) ON DELETE CASCADE; - - --- --- Name: dsm_datasourcepids_datasource_fkey; Type: FK CONSTRAINT; Schema: public; Owner: dnetapi --- - -ALTER TABLE ONLY dsm_datasourcepids - ADD CONSTRAINT dsm_datasourcepids_datasource_fkey FOREIGN KEY (datasource) REFERENCES dsm_datasources(id); - - --- --- Name: dsm_datasourcepids_pid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: dnetapi --- - -ALTER TABLE ONLY dsm_datasourcepids - ADD CONSTRAINT dsm_datasourcepids_pid_fkey FOREIGN KEY (pid) REFERENCES dsm_identities(pid); - - --- --- Name: dsm_datasources_collectedfrom_fkey; Type: FK CONSTRAINT; Schema: public; Owner: dnetapi --- - -ALTER TABLE ONLY dsm_datasources - ADD CONSTRAINT dsm_datasources_collectedfrom_fkey FOREIGN KEY (collectedfrom) REFERENCES dsm_datasources(id); - - --- --- Name: dsm_organizationpids_organization_fkey; Type: FK CONSTRAINT; Schema: public; Owner: dnetapi --- - -ALTER TABLE ONLY dsm_organizationpids - ADD CONSTRAINT dsm_organizationpids_organization_fkey FOREIGN KEY (organization) REFERENCES dsm_organizations(id); - - --- --- Name: dsm_organizationpids_pid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: dnetapi --- - -ALTER TABLE ONLY dsm_organizationpids - ADD CONSTRAINT dsm_organizationpids_pid_fkey FOREIGN KEY (pid) REFERENCES dsm_identities(pid); - - --- --- Name: dsm_organizations_collectedfrom_fkey; Type: FK CONSTRAINT; Schema: public; Owner: dnetapi --- - -ALTER TABLE ONLY dsm_organizations - ADD CONSTRAINT dsm_organizations_collectedfrom_fkey FOREIGN KEY (collectedfrom) REFERENCES dsm_datasources(id); - - - -REVOKE ALL ON SCHEMA public FROM PUBLIC; -REVOKE ALL ON SCHEMA public FROM postgres; -GRANT ALL ON SCHEMA public TO postgres; -GRANT ALL ON SCHEMA public TO PUBLIC; - - -REVOKE ALL ON TABLE browse_countries FROM PUBLIC; -REVOKE ALL ON TABLE browse_countries FROM dnet; -GRANT ALL ON TABLE browse_countries TO dnet; -GRANT ALL ON TABLE browse_countries TO dnetapi; - - -REVOKE ALL ON TABLE dsm_datasource_api FROM PUBLIC; -REVOKE ALL ON TABLE dsm_datasource_api FROM dnet; -GRANT ALL ON TABLE dsm_datasource_api TO dnet; -GRANT SELECT ON TABLE dsm_datasource_api TO dnetapi; - - diff --git a/apps/dnet-exporter-api/src/main/resources/dnet_info.sql b/apps/dnet-exporter-api/src/main/resources/dnet_info.sql deleted file mode 100644 index 23ddb680..00000000 --- a/apps/dnet-exporter-api/src/main/resources/dnet_info.sql +++ /dev/null @@ -1,21 +0,0 @@ - -CREATE TABLE info ( - key character varying(255) NOT NULL PRIMARY KEY, - value DATE -); - -GRANT ALL ON TABLE info TO dnet; -GRANT ALL ON TABLE info TO dnetapi; - -INSERT INTO info(key) VALUES ('oaf_load_date'); -INSERT INTO info(key) VALUES ('odf_load_date'); -INSERT INTO info(key) VALUES ('inference_date'); -INSERT INTO info(key) VALUES ('claim_load_date'); -INSERT INTO info(key) VALUES ('stats_update_date'); - -INSERT INTO info(key) VALUES ('crossref_update_date'); -INSERT INTO info(key) VALUES ('unpaywall_update_date'); -INSERT INTO info(key) VALUES ('orcid_update_date'); -INSERT INTO info(key) VALUES ('mag_update_date'); - - diff --git a/apps/dnet-exporter-api/src/main/resources/templates/dnetprofiles/repository.xml b/apps/dnet-exporter-api/src/main/resources/templates/dnetprofiles/repository.xml deleted file mode 100644 index 6e7a8b54..00000000 --- a/apps/dnet-exporter-api/src/main/resources/templates/dnetprofiles/repository.xml +++ /dev/null @@ -1,79 +0,0 @@ - -
    - - - - - - -
    - - - - - false - - OPENAIRE - - - 0 - 0 - 0 - - - - - - - - - - - - - - - - OpenAireDataSourceId - - - - NamespacePrefix - - - - VERIFIED - NO - - - aggregatorName - - - - dateOfValidation - - - - dateOfCollection - - - - ACTID - - - - - - - - - - - 0 - - 0.0 - - - - -     -- 2.17.1 From 9f2d5c6c24bef3ec97f3cf9d20f8f55ea886bad3 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Mon, 7 Feb 2022 10:09:18 +0100 Subject: [PATCH 03/19] cleaning --- .../DNetOpenaireExporterApplication.java | 107 ++- .../DNetOpenaireExporterBeanFactory.java | 141 ---- .../DNetOpenaireExporterConfiguration.java | 106 +++ ...va => DnetOpenaireExporterProperties.java} | 2 +- .../main/java/eu/dnetlib/SwaggerConfig.java | 115 --- .../java/eu/dnetlib/SwaggerController.java | 19 + .../dnetlib/openaire/common/ISClientImpl.java | 11 +- .../openaire/common/RFC3339DateFormat.java | 51 +- .../openaire/community/CommunityApiCore.java | 680 +++++++++--------- .../community/CommunityContentprovider.java | 25 +- .../community/CommunityException.java | 5 + .../community/CommunityNotFoundException.java | 5 + .../openaire/community/CommunitySummary.java | 65 +- .../selectioncriteria/Constraint.java | 53 +- .../selectioncriteria/Constraints.java | 26 +- .../selectioncriteria/SelectionCriteria.java | 33 +- .../openaire/context/ContextApiCore.java | 101 ++- .../openaire/context/ContextException.java | 5 + .../context/ContextNotFoundException.java | 5 + .../openaire/dsm/DsmApiController.java | 4 +- .../java/eu/dnetlib/openaire/dsm/DsmCore.java | 172 +---- .../openaire/dsm/dao/DatasourceDaoImpl.java | 122 ++-- .../dsm/dao/DatasourceDbEntryRepository.java | 7 +- .../dsm/dao/DatasourceIndexClientImpl.java | 4 +- .../dsm/dao/MongoLoggerClientImpl.java | 6 +- .../openaire/dsm/dao/ResponseUtils.java | 18 +- .../dsm/dao/VocabularyClientImpl.java | 4 +- .../openaire/dsm/domain/RequestFilter.java | 6 + .../openaire/dsm/domain/SimpleResponse.java | 2 +- .../openaire/dsm/domain/db/ApiDbEntry.java | 2 +- .../dsm/domain/db/ApiParamKeyDbEntry.java | 6 + .../dsm/domain/db/DatasourceApiDbEntry.java | 8 + .../dnetlib/openaire/funders/FunderDao.java | 30 +- .../openaire/funders/FundersApiException.java | 8 +- .../dnetlib/openaire/info/InfoController.java | 105 +-- .../eu/dnetlib/openaire/info/JdbcInfoDao.java | 9 - .../openaire/info/JdbcInfoDaoImpl.java | 58 +- .../openaire/project/ProjectQueryParams.java | 12 +- .../openaire/project/ProjectsController.java | 6 +- .../openaire/project/dao/JdbcApiDaoImpl.java | 4 +- .../src/main/resources/dnet_dsm.data.sql | 176 +++++ .../src/main/resources/dnet_dsm.sql | 510 +++++++++++++ .../src/main/resources/dnet_info.sql | 21 + 43 files changed, 1695 insertions(+), 1160 deletions(-) delete mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/DNetOpenaireExporterBeanFactory.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/DNetOpenaireExporterConfiguration.java rename apps/dnet-exporter-api/src/main/java/eu/dnetlib/{OpenaireExporterConfig.java => DnetOpenaireExporterProperties.java} (99%) delete mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/SwaggerConfig.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/SwaggerController.java create mode 100644 apps/dnet-exporter-api/src/main/resources/dnet_dsm.data.sql create mode 100644 apps/dnet-exporter-api/src/main/resources/dnet_dsm.sql create mode 100644 apps/dnet-exporter-api/src/main/resources/dnet_info.sql diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/DNetOpenaireExporterApplication.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/DNetOpenaireExporterApplication.java index 1152802e..5f61230e 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/DNetOpenaireExporterApplication.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/DNetOpenaireExporterApplication.java @@ -1,31 +1,114 @@ package eu.dnetlib; -import java.io.IOException; -import javax.servlet.http.HttpServletResponse; +import static springfox.documentation.builders.RequestHandlerSelectors.basePackage; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.solr.SolrAutoConfiguration; import org.springframework.cache.annotation.EnableCaching; +import org.springframework.context.annotation.Bean; import org.springframework.scheduling.annotation.EnableScheduling; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; + +import eu.dnetlib.DnetOpenaireExporterProperties.Swagger; +import eu.dnetlib.common.app.AbstractDnetApp; +import eu.dnetlib.openaire.community.CommunityApiController; +import eu.dnetlib.openaire.context.ContextApiController; +import eu.dnetlib.openaire.dsm.DsmApiController; +import eu.dnetlib.openaire.funders.FundersApiController; +import eu.dnetlib.openaire.info.InfoController; +import eu.dnetlib.openaire.project.ProjectsController; +import springfox.documentation.builders.ApiInfoBuilder; +import springfox.documentation.service.ApiInfo; +import springfox.documentation.service.Contact; +import springfox.documentation.spi.DocumentationType; +import springfox.documentation.spring.web.plugins.Docket; +import springfox.documentation.swagger2.annotations.EnableSwagger2; @EnableCaching -@RestController @EnableScheduling +@EnableSwagger2 @SpringBootApplication -@EnableAutoConfiguration(exclude = { SolrAutoConfiguration.class }) -public class DNetOpenaireExporterApplication { +@EnableAutoConfiguration(exclude = { + SolrAutoConfiguration.class +}) +public class DNetOpenaireExporterApplication extends AbstractDnetApp { - @RequestMapping(value = { "/", "/docs" }) - public void index(final HttpServletResponse response) throws IOException { - response.sendRedirect("swagger-ui.html"); - } + public static final String V1 = "1.0.0"; - public static void main(String[] args) throws Exception { + public static void main(final String[] args) throws Exception { SpringApplication.run(DNetOpenaireExporterApplication.class, args); } + @Autowired + private DnetOpenaireExporterProperties config; + + @Bean + public Docket dsm() { + return _docket("Datasource Manager", DsmApiController.class.getPackage().getName(), config.getSwaggerDsm(), V1); + } + + @Bean + public Docket projects() { + return _docket("OpenAIRE Projects", ProjectsController.class.getPackage().getName(), config.getSwaggerProjects(), V1); + } + + @Bean + public Docket funders() { + return _docket("OpenAIRE Funders", FundersApiController.class.getPackage().getName(), config.getSwaggerFunders(), V1); + } + + @Bean + public Docket communities() { + return _docket("OpenAIRE Communities", CommunityApiController.class.getPackage().getName(), config.getSwaggerCommunities(), V1); + } + + @Bean + public Docket contexts() { + return _docket("OpenAIRE Contexts", ContextApiController.class.getPackage().getName(), config.getSwaggerCommunities(), V1); + } + + @Bean + public Docket info() { + return _docket("OpenAIRE Info", InfoController.class.getPackage().getName(), config.getSwaggerInfo(), V1); + } + + private Docket _docket(final String groupName, final String controllerPackage, final Swagger swag, final String version) { + final Docket d = new Docket(DocumentationType.SWAGGER_2); + configSwagger(d, groupName, controllerPackage, swag, version); + return d; + } + + @Override + protected void configSwagger(final Docket docket) { + configSwagger(docket, "OpenAIRE Info", InfoController.class.getPackage().getName(), config.getSwaggerInfo(), V1); + } + + private void configSwagger(final Docket docket, final String groupName, final String controllerPackage, final Swagger swag, final String version) { + docket + .groupName(groupName) + .select() + .apis(basePackage(controllerPackage)) + .build() + .directModelSubstitute(org.joda.time.LocalDate.class, java.sql.Date.class) + .directModelSubstitute(org.joda.time.DateTime.class, java.util.Date.class) + .apiInfo(apiInfo(swag, version)); + } + + private ApiInfo apiInfo(final Swagger swag, final String version) { + return new ApiInfoBuilder() + .title(swag.getApiTitle()) + .description(swag.getApiDescription()) + .license(swag.getApiLicense()) + .licenseUrl(swag.getApiLicenseUrl()) + .termsOfServiceUrl("") + .version(version) + .contact(new Contact( + swag.getApiContactName(), + swag.getApiContactUrl(), + swag.getApiContactEmail())) + .build(); + } + } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/DNetOpenaireExporterBeanFactory.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/DNetOpenaireExporterBeanFactory.java deleted file mode 100644 index 32e1940b..00000000 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/DNetOpenaireExporterBeanFactory.java +++ /dev/null @@ -1,141 +0,0 @@ -package eu.dnetlib; - -import com.google.common.collect.Lists; -import com.mongodb.MongoClient; -import com.mongodb.MongoClientOptions; -import com.mongodb.ServerAddress; -import eu.dnetlib.OpenaireExporterConfig.Jdbc; -import eu.dnetlib.data.objectstore.rmi.ObjectStoreService; -import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; -import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService; -import io.micrometer.core.instrument.ImmutableTag; -import io.micrometer.core.instrument.Metrics; -import org.apache.commons.dbcp2.BasicDataSource; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.cxf.endpoint.Client; -import org.apache.cxf.frontend.ClientProxy; -import org.apache.cxf.jaxws.JaxWsProxyFactoryBean; -import org.apache.cxf.transport.http.HTTPConduit; -import org.apache.cxf.transports.http.configuration.HTTPClientPolicy; -import org.apache.maven.model.Model; -import org.apache.maven.model.io.xpp3.MavenXpp3Reader; -import org.codehaus.plexus.util.xml.pull.XmlPullParserException; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.core.io.ClassPathResource; -import org.springframework.core.io.ResourceLoader; - -import javax.annotation.PostConstruct; -import javax.sql.DataSource; -import java.io.IOException; -import java.io.InputStreamReader; - -/** - * Created by claudio on 07/07/2017. - */ -@Configuration -public class DNetOpenaireExporterBeanFactory { - - private static final Log log = LogFactory.getLog(DNetOpenaireExporterBeanFactory.class); - - @Autowired - private ResourceLoader resourceLoader; - - @Value("pom.xml") - private ClassPathResource pom; - - @PostConstruct - public void init() { - final MavenXpp3Reader reader = new MavenXpp3Reader(); - try { - final Model model = reader.read(new InputStreamReader(pom.getInputStream())); - - log.info(String.format("registering metric for %s", model.getArtifactId())); - Metrics.gauge("micrometer_info", Lists.newArrayList( - new ImmutableTag("component", model.getGroupId()+":"+model.getArtifactId()), - new ImmutableTag("version", model.getVersion()), - new ImmutableTag("scmtag", model.getScm().getTag())), 1); - } catch (IOException | XmlPullParserException e) { - log.error(e); - } - } - - @Autowired - private OpenaireExporterConfig config; - - @Bean - public ISLookUpService getLookUpService() { - return getServiceStub(ISLookUpService.class, config.getIsLookupUrl()); - } - - @Bean - public ObjectStoreService getObjectStoreService() { - return getServiceStub(ObjectStoreService.class, config.getObjectStoreServiceUrl()); - } - - @Bean - public ISRegistryService getRegistryService() { - return getServiceStub(ISRegistryService.class, config.getIsRegistryServiceUrl()); - } - - @SuppressWarnings("unchecked") - private T getServiceStub(final Class clazz, final String endpoint) { - log.info(String.format("Initializing service stub %s, endpoint %s", clazz.toString(),endpoint)); - final JaxWsProxyFactoryBean jaxWsProxyFactory = new JaxWsProxyFactoryBean(); - jaxWsProxyFactory.setServiceClass(clazz); - jaxWsProxyFactory.setAddress(endpoint); - - final T service = (T) jaxWsProxyFactory.create(); - - Client client = ClientProxy.getClient(service); - if (client != null) { - HTTPConduit conduit = (HTTPConduit) client.getConduit(); - HTTPClientPolicy policy = new HTTPClientPolicy(); - - log.info(String.format("setting connectTimeout to %s, receiveTimeout to %s for service %s", - config.getCxfClientConnectTimeout(), - config.getCxfClientReceiveTimeout(), - clazz.getCanonicalName())); - - policy.setConnectionTimeout(config.getCxfClientConnectTimeout()); - policy.setReceiveTimeout(config.getCxfClientReceiveTimeout()); - conduit.setClient(policy); - } - - return service; - } - - @Bean - public DataSource getSqlDataSource() { - final Jdbc jdbc = config.getJdbc(); - return getDatasource( - jdbc.getDriverClassName(), - jdbc.getUrl(), - jdbc.getUser(), - jdbc.getPwd(), - jdbc.getMinIdle(), - jdbc.getMaxRows()); - } - - private BasicDataSource getDatasource(String driverClassName, String jdbcUrl, String jdbcUser, String jdbcPwd, int jdbcMinIdle, int jdbcMaxIdle) { - final BasicDataSource d = new BasicDataSource(); - d.setDriverClassName(driverClassName); - d.setUrl(jdbcUrl); - d.setUsername(jdbcUser); - d.setPassword(jdbcPwd); - d.setMinIdle(jdbcMinIdle); - d.setMaxIdle(jdbcMaxIdle); - return d; - } - - @Bean - public MongoClient getMongoClient() { - return new MongoClient( - new ServerAddress(config.getDatasource().getMongoHost(), config.getDatasource().getMongoPort()), - MongoClientOptions.builder().connectionsPerHost(config.getDatasource().getMongoConnectionsPerHost()).build()); - } - -} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/DNetOpenaireExporterConfiguration.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/DNetOpenaireExporterConfiguration.java new file mode 100644 index 00000000..24674d3a --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/DNetOpenaireExporterConfiguration.java @@ -0,0 +1,106 @@ +package eu.dnetlib; + +import javax.sql.DataSource; + +import org.apache.commons.dbcp2.BasicDataSource; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.cxf.endpoint.Client; +import org.apache.cxf.frontend.ClientProxy; +import org.apache.cxf.jaxws.JaxWsProxyFactoryBean; +import org.apache.cxf.transport.http.HTTPConduit; +import org.apache.cxf.transports.http.configuration.HTTPClientPolicy; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import com.mongodb.MongoClient; +import com.mongodb.MongoClientOptions; +import com.mongodb.ServerAddress; + +import eu.dnetlib.DnetOpenaireExporterProperties.Jdbc; +import eu.dnetlib.data.objectstore.rmi.ObjectStoreService; +import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; +import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService; + +/** + * Created by claudio on 07/07/2017. + */ +@Configuration +public class DNetOpenaireExporterConfiguration { + + private static final Log log = LogFactory.getLog(DNetOpenaireExporterConfiguration.class); + + @Autowired + private DnetOpenaireExporterProperties props; + + @Bean + public ISLookUpService getLookUpService() { + return getServiceStub(ISLookUpService.class, props.getIsLookupUrl()); + } + + @Bean + public ObjectStoreService getObjectStoreService() { + return getServiceStub(ObjectStoreService.class, props.getObjectStoreServiceUrl()); + } + + @Bean + public ISRegistryService getRegistryService() { + return getServiceStub(ISRegistryService.class, props.getIsRegistryServiceUrl()); + } + + @SuppressWarnings("unchecked") + private T getServiceStub(final Class clazz, final String endpoint) { + log.info(String.format("Initializing service stub %s, endpoint %s", clazz.toString(), endpoint)); + final JaxWsProxyFactoryBean jaxWsProxyFactory = new JaxWsProxyFactoryBean(); + jaxWsProxyFactory.setServiceClass(clazz); + jaxWsProxyFactory.setAddress(endpoint); + + final T service = (T) jaxWsProxyFactory.create(); + + final Client client = ClientProxy.getClient(service); + if (client != null) { + final HTTPConduit conduit = (HTTPConduit) client.getConduit(); + final HTTPClientPolicy policy = new HTTPClientPolicy(); + + log.info(String.format("setting connectTimeout to %s, receiveTimeout to %s for service %s", props.getCxfClientConnectTimeout(), props + .getCxfClientReceiveTimeout(), clazz.getCanonicalName())); + + policy.setConnectionTimeout(props.getCxfClientConnectTimeout()); + policy.setReceiveTimeout(props.getCxfClientReceiveTimeout()); + conduit.setClient(policy); + } + + return service; + } + + @Bean + public DataSource getSqlDataSource() { + final Jdbc jdbc = props.getJdbc(); + return getDatasource(jdbc.getDriverClassName(), jdbc.getUrl(), jdbc.getUser(), jdbc.getPwd(), jdbc.getMinIdle(), jdbc.getMaxRows()); + } + + private BasicDataSource getDatasource(final String driverClassName, + final String jdbcUrl, + final String jdbcUser, + final String jdbcPwd, + final int jdbcMinIdle, + final int jdbcMaxIdle) { + final BasicDataSource d = new BasicDataSource(); + d.setDriverClassName(driverClassName); + d.setUrl(jdbcUrl); + d.setUsername(jdbcUser); + d.setPassword(jdbcPwd); + d.setMinIdle(jdbcMinIdle); + d.setMaxIdle(jdbcMaxIdle); + return d; + } + + @Bean + public MongoClient getMongoClient() { + return new MongoClient( + new ServerAddress(props.getDatasource().getMongoHost(), props.getDatasource().getMongoPort()), + MongoClientOptions.builder().connectionsPerHost(props.getDatasource().getMongoConnectionsPerHost()).build()); + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/OpenaireExporterConfig.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/DnetOpenaireExporterProperties.java similarity index 99% rename from apps/dnet-exporter-api/src/main/java/eu/dnetlib/OpenaireExporterConfig.java rename to apps/dnet-exporter-api/src/main/java/eu/dnetlib/DnetOpenaireExporterProperties.java index 87a54dd4..fbdaf701 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/OpenaireExporterConfig.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/DnetOpenaireExporterProperties.java @@ -15,7 +15,7 @@ import org.springframework.core.io.Resource; @Configuration @PropertySource("classpath:global.properties") @ConfigurationProperties(prefix = "openaire.exporter") -public class OpenaireExporterConfig { +public class DnetOpenaireExporterProperties { // ISLOOKUP private ClassPathResource findSolrIndexUrl; diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/SwaggerConfig.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/SwaggerConfig.java deleted file mode 100644 index 3276bf1c..00000000 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/SwaggerConfig.java +++ /dev/null @@ -1,115 +0,0 @@ -package eu.dnetlib; - -import eu.dnetlib.OpenaireExporterConfig.Swagger; -import eu.dnetlib.openaire.community.CommunityApiController; -import eu.dnetlib.openaire.context.ContextApiController; -import eu.dnetlib.openaire.dsm.DsmApiController; -import eu.dnetlib.openaire.funders.FundersApiController; -import eu.dnetlib.openaire.info.InfoController; -import eu.dnetlib.openaire.project.ProjectsController; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import springfox.documentation.builders.ApiInfoBuilder; -import springfox.documentation.service.ApiInfo; -import springfox.documentation.service.Contact; -import springfox.documentation.spi.DocumentationType; -import springfox.documentation.spring.web.plugins.Docket; -import springfox.documentation.swagger2.annotations.EnableSwagger2; - -import static springfox.documentation.builders.RequestHandlerSelectors.basePackage; - -@Configuration -@EnableSwagger2 -public class SwaggerConfig { - - private static final Log log = LogFactory.getLog(SwaggerConfig.class); - - public static final String V1 = "1.0.0"; - - @Autowired - private OpenaireExporterConfig config; - - @Bean - public Docket dsm() { - return _docket( - "Datasource Manager", - DsmApiController.class.getPackage().getName(), - config.getSwaggerDsm(), - V1); - } - - @Bean - public Docket projects() { - return _docket( - "OpenAIRE Projects", - ProjectsController.class.getPackage().getName(), - config.getSwaggerProjects(), - V1); - } - - @Bean - public Docket funders() { - return _docket( - "OpenAIRE Funders", - FundersApiController.class.getPackage().getName(), - config.getSwaggerFunders(), - V1); - } - - @Bean - public Docket communities() { - return _docket( - "OpenAIRE Communities", - CommunityApiController.class.getPackage().getName(), - config.getSwaggerCommunities(), - V1); - } - - @Bean - public Docket contexts() { - return _docket( - "OpenAIRE Contexts", - ContextApiController.class.getPackage().getName(), - config.getSwaggerCommunities(), - V1); - } - - @Bean - public Docket info() { - return _docket( - "OpenAIRE Info", - InfoController.class.getPackage().getName(), - config.getSwaggerInfo(), - V1); - } - - private Docket _docket(final String groupName, final String controllerPackage, final Swagger swag, final String version) { - return new Docket(DocumentationType.SWAGGER_2) - .groupName(groupName) - .select() - .apis(basePackage(controllerPackage)) - .build() - .directModelSubstitute(org.joda.time.LocalDate.class, java.sql.Date.class) - .directModelSubstitute(org.joda.time.DateTime.class, java.util.Date.class) - .apiInfo(apiInfo(swag, version)); - } - - private ApiInfo apiInfo(final Swagger swag, final String version) { - return new ApiInfoBuilder() - .title(swag.getApiTitle()) - .description(swag.getApiDescription()) - .license(swag.getApiLicense()) - .licenseUrl(swag.getApiLicenseUrl()) - .termsOfServiceUrl("") - .version(version) - .contact(new Contact( - swag.getApiContactName(), - swag.getApiContactUrl(), - swag.getApiContactEmail())) - .build(); - } - -} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/SwaggerController.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/SwaggerController.java new file mode 100644 index 00000000..c3cadff1 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/SwaggerController.java @@ -0,0 +1,19 @@ +package eu.dnetlib; +import java.io.IOException; + +import javax.servlet.http.HttpServletResponse; + +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.RequestMapping; + +@Controller +public class SwaggerController { + + @RequestMapping(value = { + "/", "/docs" + }) + public void index(final HttpServletResponse response) throws IOException { + response.sendRedirect("swagger-ui.html"); + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClientImpl.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClientImpl.java index 3e316a07..3bb72c60 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClientImpl.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClientImpl.java @@ -27,12 +27,11 @@ import com.google.common.collect.Lists; import com.google.common.escape.Escaper; import com.google.common.xml.XmlEscapers; -import eu.dnetlib.OpenaireExporterConfig; +import eu.dnetlib.DnetOpenaireExporterProperties; import eu.dnetlib.enabling.datasources.common.DsmException; import eu.dnetlib.enabling.datasources.common.DsmRuntimeException; import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException; import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; -import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService; import eu.dnetlib.openaire.context.Context; import eu.dnetlib.openaire.context.ContextMappingUtils; import eu.dnetlib.openaire.dsm.dao.utils.IndexDsInfo; @@ -46,17 +45,11 @@ public class ISClientImpl implements ISClient { private static final Log log = LogFactory.getLog(ISClientImpl.class); @Autowired - private OpenaireExporterConfig config; + private DnetOpenaireExporterProperties config; @Autowired private ISLookUpService isLookUpService; - @Autowired - private ISRegistryService isRegistryService; - - @Autowired - private OperationManager operationManager; - @Override @Cacheable("indexdsinfo-cache") public IndexDsInfo calculateCurrentIndexDsInfo() throws DsmException { diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/RFC3339DateFormat.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/RFC3339DateFormat.java index 28fe4e3f..51c35b25 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/RFC3339DateFormat.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/RFC3339DateFormat.java @@ -1,18 +1,27 @@ package eu.dnetlib.openaire.common; import java.text.FieldPosition; -import java.util.*; +import java.util.Calendar; +import java.util.Date; +import java.util.GregorianCalendar; +import java.util.Locale; +import java.util.TimeZone; import com.fasterxml.jackson.databind.util.StdDateFormat; public class RFC3339DateFormat extends StdDateFormat { + /** + * + */ + private static final long serialVersionUID = 8174507696046505992L; + private static final TimeZone TIMEZONE_Z = TimeZone.getTimeZone("UTC"); // Same as ISO8601DateFormat but serializing milliseconds. @Override - public StringBuffer format(Date date, StringBuffer toAppendTo, FieldPosition fieldPosition) { - String value = format(date, true, TIMEZONE_Z, Locale.US); + public StringBuffer format(final Date date, final StringBuffer toAppendTo, final FieldPosition fieldPosition) { + final String value = format(date, true, TIMEZONE_Z, Locale.US); toAppendTo.append(value); return toAppendTo; } @@ -20,37 +29,31 @@ public class RFC3339DateFormat extends StdDateFormat { /** * Format date into yyyy-MM-ddThh:mm:ss[.sss][Z|[+-]hh:mm] * - * @param date the date to format - * @param millis true to include millis precision otherwise false - * @param tz timezone to use for the formatting (UTC will produce 'Z') + * @param date + * the date to format + * @param millis + * true to include millis precision otherwise false + * @param tz + * timezone to use for the formatting (UTC will produce 'Z') * @return the date formatted as yyyy-MM-ddThh:mm:ss[.sss][Z|[+-]hh:mm] */ - private static String format(Date date, boolean millis, TimeZone tz, Locale loc) { - Calendar calendar = new GregorianCalendar(tz, loc); + private static String format(final Date date, final boolean millis, final TimeZone tz, final Locale loc) { + final Calendar calendar = new GregorianCalendar(tz, loc); calendar.setTime(date); // estimate capacity of buffer as close as we can (yeah, that's pedantic ;) - StringBuilder sb = new StringBuilder(30); - sb.append(String.format( - "%04d-%02d-%02dT%02d:%02d:%02d", - calendar.get(Calendar.YEAR), - calendar.get(Calendar.MONTH) + 1, - calendar.get(Calendar.DAY_OF_MONTH), - calendar.get(Calendar.HOUR_OF_DAY), - calendar.get(Calendar.MINUTE), - calendar.get(Calendar.SECOND) - )); + final StringBuilder sb = new StringBuilder(30); + sb.append(String.format("%04d-%02d-%02dT%02d:%02d:%02d", calendar.get(Calendar.YEAR), calendar.get(Calendar.MONTH) + 1, calendar + .get(Calendar.DAY_OF_MONTH), calendar.get(Calendar.HOUR_OF_DAY), calendar.get(Calendar.MINUTE), calendar.get(Calendar.SECOND))); if (millis) { sb.append(String.format(".%03d", calendar.get(Calendar.MILLISECOND))); } - int offset = tz.getOffset(calendar.getTimeInMillis()); + final int offset = tz.getOffset(calendar.getTimeInMillis()); if (offset != 0) { - int hours = Math.abs((offset / (60 * 1000)) / 60); - int minutes = Math.abs((offset / (60 * 1000)) % 60); - sb.append(String.format("%c%02d:%02d", - (offset < 0 ? '-' : '+'), - hours, minutes)); + final int hours = Math.abs(offset / (60 * 1000) / 60); + final int minutes = Math.abs(offset / (60 * 1000) % 60); + sb.append(String.format("%c%02d:%02d", offset < 0 ? '-' : '+', hours, minutes)); } else { sb.append('Z'); } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityApiCore.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityApiCore.java index c655a91a..bc8c2c18 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityApiCore.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityApiCore.java @@ -1,11 +1,37 @@ package eu.dnetlib.openaire.community; -import com.google.common.base.Functions; -import com.google.common.base.Joiner; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; -import eu.dnetlib.openaire.common.ISClient; -import eu.dnetlib.openaire.context.Context; +import static eu.dnetlib.openaire.community.CommunityConstants.CCONTENTPROVIDER_NAME; +import static eu.dnetlib.openaire.community.CommunityConstants.CCONTENTPROVIDER_OFFICIALNAME; +import static eu.dnetlib.openaire.community.CommunityConstants.CCONTENTPROVIDER_SELCRITERIA; +import static eu.dnetlib.openaire.community.CommunityConstants.CLABEL; +import static eu.dnetlib.openaire.community.CommunityConstants.CONTENTPROVIDERS_ID_SUFFIX; +import static eu.dnetlib.openaire.community.CommunityConstants.CORGANIZATION_LOGOURL; +import static eu.dnetlib.openaire.community.CommunityConstants.CORGANIZATION_NAME; +import static eu.dnetlib.openaire.community.CommunityConstants.CORGANIZATION_WEBSITEURL; +import static eu.dnetlib.openaire.community.CommunityConstants.CPROFILE_SUBJECT; +import static eu.dnetlib.openaire.community.CommunityConstants.CPROJECT_ACRONYM; +import static eu.dnetlib.openaire.community.CommunityConstants.CPROJECT_FULLNAME; +import static eu.dnetlib.openaire.community.CommunityConstants.CPROJECT_FUNDER; +import static eu.dnetlib.openaire.community.CommunityConstants.CPROJECT_NUMBER; +import static eu.dnetlib.openaire.community.CommunityConstants.CSUMMARY_DESCRIPTION; +import static eu.dnetlib.openaire.community.CommunityConstants.CSUMMARY_LOGOURL; +import static eu.dnetlib.openaire.community.CommunityConstants.CSUMMARY_NAME; +import static eu.dnetlib.openaire.community.CommunityConstants.CSUMMARY_STATUS; +import static eu.dnetlib.openaire.community.CommunityConstants.CSUMMARY_ZENODOC; +import static eu.dnetlib.openaire.community.CommunityConstants.CSV_DELIMITER; +import static eu.dnetlib.openaire.community.CommunityConstants.ID_SEPARATOR; +import static eu.dnetlib.openaire.community.CommunityConstants.OPENAIRE_ID; +import static eu.dnetlib.openaire.community.CommunityConstants.ORGANIZATION_ID_SUFFIX; +import static eu.dnetlib.openaire.community.CommunityConstants.PROJECTS_ID_SUFFIX; +import static eu.dnetlib.openaire.community.CommunityConstants.ZENODOCOMMUNITY_ID_SUFFIX; + +import java.util.Base64; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.stream.Collectors; + import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -14,373 +40,329 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.cache.annotation.CacheEvict; import org.springframework.stereotype.Component; -import java.io.IOException; -import java.util.*; -import java.util.stream.Collectors; +import com.google.common.base.Functions; +import com.google.common.base.Joiner; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; -import static eu.dnetlib.openaire.community.CommunityConstants.*; +import eu.dnetlib.openaire.common.ISClient; @Component @ConditionalOnProperty(value = "openaire.exporter.enable.community", havingValue = "true") -public class CommunityApiCore {//implements CommunityClient{ +public class CommunityApiCore {// implements CommunityClient{ - private static final Log log = LogFactory.getLog(CommunityApiCore.class); + private static final Log log = LogFactory.getLog(CommunityApiCore.class); - @Autowired - private CommunityClient cci; + @Autowired + private CommunityClient cci; - @Autowired - private ISClient isClient; + @Autowired + private ISClient isClient; - @Autowired - private CommunityCommon cc; + @Autowired + private CommunityCommon cc; + public List listCommunities() throws CommunityException { + return cc.listCommunities(); - public List listCommunities() throws CommunityException { - return cc.listCommunities(); + } - } + public CommunityDetails getCommunity(final String id) throws CommunityException, CommunityNotFoundException { + return cc.getCommunity(id); - public CommunityDetails getCommunity(final String id) throws CommunityException, CommunityNotFoundException { - return cc.getCommunity(id); + } - } + public void setCommunity(final String id, final CommunityWritableProperties details) throws CommunityException, CommunityNotFoundException { - public void setCommunity(final String id, final CommunityWritableProperties details) throws CommunityException, CommunityNotFoundException { + cc.getCommunity(id); // ensure the community exists. - cc.getCommunity(id); // ensure the community exists. + if (details.getShortName() != null) { + isClient.updateContextAttribute(id, CLABEL, details.getShortName()); - if(details.getShortName() != null) { - isClient.updateContextAttribute(id, CLABEL, details.getShortName()); + } + if (details.getName() != null) { + isClient.updateContextParam(id, CSUMMARY_NAME, details.getName()); - } - if (details.getName() != null){ - isClient.updateContextParam(id, CSUMMARY_NAME, details.getName()); + } + if (details.getDescription() != null) { + isClient.updateContextParam(id, CSUMMARY_DESCRIPTION, details.getDescription()); - } - if(details.getDescription() != null) { - isClient.updateContextParam(id, CSUMMARY_DESCRIPTION, details.getDescription()); + } + if (details.getLogoUrl() != null) { + isClient.updateContextParam(id, CSUMMARY_LOGOURL, details.getLogoUrl()); + + } + if (details.getStatus() != null) { + isClient.updateContextParam(id, CSUMMARY_STATUS, details.getStatus().name()); + + } + if (details.getSubjects() != null) { + isClient.updateContextParam(id, CPROFILE_SUBJECT, Joiner.on(CSV_DELIMITER).join(details.getSubjects())); + + } + if (details.getMainZenodoCommunity() != null) { + isClient.updateContextParam(id, CSUMMARY_ZENODOC, details.getMainZenodoCommunity()); + } + + cc.updateCommunity(id, details); + } + + public List getCommunityProjects(final String id) throws CommunityException, CommunityNotFoundException { + cc.getCommunity(id); // ensure the community exists. + return cc.getCommunityInfo(id, PROJECTS_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityProject(id, c)); + } + + public CommunityProject addCommunityProject(final String id, final CommunityProject project) throws CommunityException, CommunityNotFoundException { + if (!StringUtils.equalsIgnoreCase(id, project.getCommunityId())) { + throw new CommunityException("parameters 'id' and project.communityId must be coherent"); + } + + final TreeMap projects = getCommunityProjectMap(id); + final String project_id = project.getId(); + + if (project_id != null && projects.keySet().contains(Integer.valueOf(project_id))) { + if (project.getName() != null) { + isClient.updateConceptParam(id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project_id, CPROJECT_FULLNAME, project.getName()); + + } + if (project.getAcronym() != null) { + isClient.updateConceptParam(id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project_id, CPROJECT_ACRONYM, project.getAcronym()); + + } + if (project.getOpenaireId() != null) { + isClient.updateConceptParam(id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project_id, OPENAIRE_ID, project.getOpenaireId()); + + } + if (project.getFunder() != null) { + isClient.updateConceptParam(id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project_id, CPROJECT_FUNDER, project.getFunder()); - } - if(details.getLogoUrl()!=null){ - isClient.updateContextParam(id, CSUMMARY_LOGOURL, details.getLogoUrl()); + } + if (project.getGrantId() != null) { + isClient.updateConceptParam(id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project_id, CPROJECT_NUMBER, project.getGrantId()); - } - if (details.getStatus() != null) { - isClient.updateContextParam(id, CSUMMARY_STATUS, details.getStatus().name()); - - } - if (details.getSubjects() != null) { - isClient.updateContextParam(id, CPROFILE_SUBJECT, Joiner.on(CSV_DELIMITER).join(details.getSubjects())); + } + } else { + project.setId(nextId(projects != null && !projects.isEmpty() ? projects.lastKey() : 0)); - } - if (details.getMainZenodoCommunity() != null) { - isClient.updateContextParam(id, CSUMMARY_ZENODOC, details.getMainZenodoCommunity()); - } - - cc.updateCommunity(id, details); - } + isClient.addConcept(id, id + PROJECTS_ID_SUFFIX, CommunityMappingUtils.asProjectXML(id, project)); - public List getCommunityProjects(final String id) throws CommunityException, CommunityNotFoundException { - cc.getCommunity(id); // ensure the community exists. - return cc.getCommunityInfo(id, PROJECTS_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityProject(id, c)); - } - - public CommunityProject addCommunityProject(final String id, final CommunityProject project) throws CommunityException, CommunityNotFoundException { - if (!StringUtils.equalsIgnoreCase(id, project.getCommunityId())) { - throw new CommunityException("parameters 'id' and project.communityId must be coherent"); - } - - final TreeMap projects = getCommunityProjectMap(id); - String project_id = project.getId(); - - if (project_id != null && projects.keySet().contains(Integer.valueOf(project_id))){ - if (project.getName() != null) { - isClient.updateConceptParam(id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project_id, CPROJECT_FULLNAME,project.getName()); - - } - if(project.getAcronym()!= null){ - isClient.updateConceptParam(id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project_id, CPROJECT_ACRONYM,project.getAcronym()); - - } - if (project.getOpenaireId() != null){ - isClient.updateConceptParam(id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project_id, OPENAIRE_ID, project.getOpenaireId()); - - } - if (project.getFunder() != null){ - isClient.updateConceptParam(id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project_id, CPROJECT_FUNDER, project.getFunder()); - - } - if(project.getGrantId() != null){ - isClient.updateConceptParam(id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project_id, CPROJECT_NUMBER, project.getGrantId()); - - } - }else { - project.setId(nextId(projects != null && !projects.isEmpty() ? projects.lastKey() : 0)); - - isClient.addConcept(id, id + PROJECTS_ID_SUFFIX, CommunityMappingUtils.asProjectXML(id, project)); - - } - cc.updateProject(id, project ); - return project; - } - - private String nextId(final Integer id) { - return String.valueOf(id + 1); - } + } + cc.updateProject(id, project); + return project; + } - public void removeCommunityProject(final String id, final Integer projectId) throws CommunityException, CommunityNotFoundException { - final Map projects = getCommunityProjectMap(id); - if (!projects.containsKey(projectId)) { - throw new CommunityNotFoundException(String.format("project '%s' doesn't exist within context '%s'", projectId, id)); - } - isClient.removeConcept( - id, - id + PROJECTS_ID_SUFFIX, - id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + projectId); - cc.removeFromCategory(id, PROJECTS_ID_SUFFIX, String.valueOf(projectId)); - } + private String nextId(final Integer id) { + return String.valueOf(id + 1); + } - public List getCommunityContentproviders(final String id) throws CommunityException, CommunityNotFoundException { - cc.getCommunity(id); // ensure the community exists. - return cc.getCommunityInfo(id, CONTENTPROVIDERS_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityDataprovider(id, c)); - } + public void removeCommunityProject(final String id, final Integer projectId) throws CommunityException, CommunityNotFoundException { + final Map projects = getCommunityProjectMap(id); + if (!projects.containsKey(projectId)) { + throw new CommunityNotFoundException(String.format("project '%s' doesn't exist within context '%s'", projectId, id)); + } + isClient.removeConcept(id, id + PROJECTS_ID_SUFFIX, id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + projectId); + cc.removeFromCategory(id, PROJECTS_ID_SUFFIX, String.valueOf(projectId)); + } - public CommunityContentprovider addCommunityContentprovider(final String id, final CommunityContentprovider cp) throws CommunityException, CommunityNotFoundException { - log.info("content provider to add " + cp.toString()); - if (!StringUtils.equalsIgnoreCase(id, cp.getCommunityId())) { - throw new CommunityException("parameters 'id' and cp.communityId must be coherent"); - } - - final TreeMap cps = getCommunityContentproviderMap(id); - final String concept_id = cp.getId(); - if (concept_id != null && cps.keySet().contains(Integer.valueOf(concept_id))){ - if (cp.getName() != null) { - isClient.updateConceptParam(id + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + concept_id, CCONTENTPROVIDER_NAME,cp.getName()); - } - if(cp.getOfficialname()!= null){ - isClient.updateConceptParam(id + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + concept_id, CCONTENTPROVIDER_OFFICIALNAME,cp.getOfficialname()); - } - if (cp.getOpenaireId() != null){ - isClient.updateConceptParam(id + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + concept_id, OPENAIRE_ID,cp.getOpenaireId()); - } - if(cp.getSelectioncriteria() != null){ - isClient.updateConceptParamNoEscape(id + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + concept_id, CCONTENTPROVIDER_SELCRITERIA, cp.toXML()); - } - }else{ - log.info("adding new concept for community " + id); - cp.setId(nextId(!cps.isEmpty() ? cps.lastKey() : 0)); - - - isClient.addConcept(id, id + CONTENTPROVIDERS_ID_SUFFIX, CommunityMappingUtils.asContentProviderXML(id, cp)); - } - - cc.updateDatasource(id, cp); - return cp; - } - - public void removeCommunityContentProvider(final String id, final Integer contentproviderId) throws CommunityException, CommunityNotFoundException { - final Map providers = getCommunityContentproviderMap(id); - if (!providers.containsKey(contentproviderId)) { - throw new CommunityNotFoundException(String.format("content provider '%s' doesn't exist within context '%s'", contentproviderId, id)); - } - isClient.removeConcept( - id, - id + CONTENTPROVIDERS_ID_SUFFIX, - id + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + contentproviderId); - cc.removeFromCategory(id, CONTENTPROVIDERS_ID_SUFFIX, String.valueOf(contentproviderId)); - } + public List getCommunityContentproviders(final String id) throws CommunityException, CommunityNotFoundException { + cc.getCommunity(id); // ensure the community exists. + return cc.getCommunityInfo(id, CONTENTPROVIDERS_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityDataprovider(id, c)); + } - public void removeCommunityOrganization(String id, Integer organizationId) throws CommunityException, CommunityNotFoundException { - final Map organizations = getCommunityOrganizationMap(id); - if (!organizations.containsKey(organizationId)) { - throw new CommunityNotFoundException(String.format("organization '%s' doesn't exist within context '%s'", organizationId, id)); - } - isClient.removeConcept( - id, - id + ORGANIZATION_ID_SUFFIX, - id + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organizationId); - cc.removeFromCategory(id, ORGANIZATION_ID_SUFFIX, String.valueOf(organizationId)); - } - - public List getCommunityZenodoCommunities(final String id) throws CommunityException, CommunityNotFoundException { - - return cc.getCommunityZenodoCommunities(id); - } - - public List getCommunityOrganizations(final String id) throws CommunityException, CommunityNotFoundException { - cc.getCommunity(id); - return cc.getCommunityInfo(id,ORGANIZATION_ID_SUFFIX,c->CommunityMappingUtils.asCommunityOrganization(id,c)); - } - - - public CommunityDetails addCommunitySubjects(final String id, final List subjects) throws CommunityException, CommunityNotFoundException { - - final CommunityDetails cd = new CommunityDetails(); - - final Set current = Sets.newHashSet(cc.getCommunity(id).getSubjects()); - - current.addAll(subjects); - - cd.setSubjects(Lists.newArrayList(current)); - - setCommunity(id, CommunityWritableProperties.fromDetails(cd)); - - return cd; - } - - public CommunityDetails removeCommunitySubjects(final String id, final List subjects) throws CommunityException, CommunityNotFoundException { - - final CommunityDetails cd = new CommunityDetails(); - - final Set current = Sets.newHashSet(cc.getCommunity(id).getSubjects()); - - current.removeAll(subjects); - - cd.setSubjects(Lists.newArrayList(current)); - - setCommunity(id, CommunityWritableProperties.fromDetails(cd)); - - return cd; - } - - @CacheEvict(value="community-cache",allEntries = true) - public void removeCommunityZenodoCommunity(final String id, final Integer zenodoCommId) throws CommunityException, CommunityNotFoundException { - - final Map zcomms = getZenodoCommunityMap(id); - if (!zcomms.containsKey(zenodoCommId)) { - throw new CommunityNotFoundException(String.format("Zenodo community '%s' doesn't exist within context '%s'", zenodoCommId, id)); - } - isClient.removeConcept( - id, - id + ZENODOCOMMUNITY_ID_SUFFIX, - id + ZENODOCOMMUNITY_ID_SUFFIX + ID_SEPARATOR + zenodoCommId); - - - cc.removeFromCategory(id, ZENODOCOMMUNITY_ID_SUFFIX, String.valueOf(zenodoCommId)); - - } - - @CacheEvict(value="community-cache",allEntries = true) - public CommunityZenodoCommunity addCommunityZenodoCommunity(final String id, final CommunityZenodoCommunity zc) throws CommunityException, CommunityNotFoundException { - if (!StringUtils.equalsIgnoreCase(id, zc.getCommunityId())) { - throw new CommunityException("parameters 'id' and zc.communityId must be coherent"); - } - if(!StringUtils.isNotBlank(zc.getZenodoid())){ - throw new CommunityException("parameter zenodoid cannot be null or empty"); - } - final TreeMap zcs = getZenodoCommunityMap(id); - - for(CommunityZenodoCommunity czc : zcs.values()){ - if (czc.getZenodoid().equals(zc.getZenodoid())){ - throw new CommunityException("Zenodo community already associated to the RCD"); - } - } - - zc.setId(nextId(!zcs.isEmpty() ? zcs.lastKey() : 0)); - - isClient.addConcept(id, id + ZENODOCOMMUNITY_ID_SUFFIX, CommunityMappingUtils.asZenodoCommunityXML(id, zc)); - cc.updateZenodoCommunity(id, zc); - - return zc; - } - - public CommunityOpenAIRECommunities getOpenAIRECommunities(String zenodoId) throws CommunityException, CommunityNotFoundException { - - if(cci.getInverseZenodoCommunityMap().containsKey(zenodoId)) - return new CommunityOpenAIRECommunities().setZenodoid(zenodoId).setOpenAirecommunitylist(cci.getInverseZenodoCommunityMap().get(zenodoId).stream().collect(Collectors.toList())); - return new CommunityOpenAIRECommunities(); - - } - - // HELPERS - - private TreeMap getCommunityProjectMap(final String id) throws CommunityException, CommunityNotFoundException { - return getCommunityProjects(id).stream() - .collect(Collectors.toMap( - p -> Integer.valueOf(p.getId()), - Functions.identity(), - (p1, p2) -> { - log.warn(String.format("duplicate project found: '%s'", p1.getId())); - return p2; - }, - TreeMap::new)); - } + public CommunityContentprovider addCommunityContentprovider(final String id, final CommunityContentprovider cp) + throws CommunityException, CommunityNotFoundException { + log.info("content provider to add " + cp.toString()); + if (!StringUtils.equalsIgnoreCase(id, cp.getCommunityId())) { throw new CommunityException("parameters 'id' and cp.communityId must be coherent"); } - private TreeMap getCommunityContentproviderMap(final String id) throws CommunityException, CommunityNotFoundException { - log.info("getting community content provider map"); - return getCommunityContentproviders(id).stream() - .collect(Collectors.toMap( - cp -> Integer.valueOf(cp.getId()), - Functions.identity(), - (cp1, cp2) -> { - log.warn(String.format("duplicate content provider found: '%s'", cp1.getId())); - return cp2; - }, - TreeMap::new)); - } - - private TreeMap getZenodoCommunityMap(final String id) throws CommunityException, CommunityNotFoundException{ - return getCommunityZenodoCommunities(id).stream() - .collect(Collectors.toMap( - cp -> Integer.valueOf(cp.getId()), - Functions.identity(), - (cp1, cp2) -> { - log.warn(String.format("duplicate Zenodo community found: '%s'", cp1.getId())); - return cp2; - }, - TreeMap::new)); - } - - private TreeMap getCommunityOrganizationMap(final String id) throws CommunityException, CommunityNotFoundException { - return getCommunityOrganizations(id).stream() - .collect(Collectors.toMap( - o -> Integer.valueOf(o.getId()), - Functions.identity(), - (o1, o2) -> { - log.warn(String.format("duplicate content provider found: '%s'", o1.getId())); - return o2; - }, - TreeMap::new)); - } - - private Map getContextMap() throws CommunityException { - try { - return isClient.getCommunityContextMap(); - } catch (IOException e) { - throw new CommunityException(e); - } - } - - public CommunityOrganization addCommunityOrganization(String id, CommunityOrganization organization) throws CommunityException, CommunityNotFoundException { - if (!StringUtils.equalsIgnoreCase(id, organization.getCommunityId())) { - throw new CommunityException("parameters 'id' and organization.communityId must be coherent"); - } - - final TreeMap cps = getCommunityOrganizationMap(id); - - final String organization_id = organization.getId(); - if (organization_id != null && cps.keySet().contains(Integer.valueOf(organization_id))){ - if (organization.getName() != null) { - isClient.updateConceptParam(id + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organization_id, CORGANIZATION_NAME,organization.getName()); - } - if(organization.getLogo_url()!= null){ - isClient.updateConceptParam(id + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organization_id, CORGANIZATION_LOGOURL, Base64.getEncoder().encodeToString(organization.getLogo_url().getBytes())); - } - if (organization.getWebsite_url() != null){ - isClient.updateConceptParam(id + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organization_id, CORGANIZATION_WEBSITEURL,Base64.getEncoder().encodeToString(organization.getWebsite_url().getBytes())); - } - }else{ - organization.setId(nextId(!cps.isEmpty() ? cps.lastKey() : 0)); - isClient.addConcept(id, id + ORGANIZATION_ID_SUFFIX, CommunityMappingUtils.asOrganizationXML(id, organization)); - } - - - cc.updateOrganization(id, organization); - return organization; - } - - - - -} \ No newline at end of file + final TreeMap cps = getCommunityContentproviderMap(id); + final String concept_id = cp.getId(); + if (concept_id != null && cps.keySet().contains(Integer.valueOf(concept_id))) { + if (cp.getName() != null) { + isClient.updateConceptParam(id + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + concept_id, CCONTENTPROVIDER_NAME, cp.getName()); + } + if (cp.getOfficialname() != null) { + isClient.updateConceptParam(id + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + concept_id, CCONTENTPROVIDER_OFFICIALNAME, cp.getOfficialname()); + } + if (cp.getOpenaireId() != null) { + isClient.updateConceptParam(id + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + concept_id, OPENAIRE_ID, cp.getOpenaireId()); + } + if (cp.getSelectioncriteria() != null) { + isClient.updateConceptParamNoEscape(id + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + concept_id, CCONTENTPROVIDER_SELCRITERIA, cp.toXML()); + } + } else { + log.info("adding new concept for community " + id); + cp.setId(nextId(!cps.isEmpty() ? cps.lastKey() : 0)); + + isClient.addConcept(id, id + CONTENTPROVIDERS_ID_SUFFIX, CommunityMappingUtils.asContentProviderXML(id, cp)); + } + + cc.updateDatasource(id, cp); + return cp; + } + + public void removeCommunityContentProvider(final String id, final Integer contentproviderId) throws CommunityException, CommunityNotFoundException { + final Map providers = getCommunityContentproviderMap(id); + if (!providers.containsKey(contentproviderId)) { + throw new CommunityNotFoundException(String.format("content provider '%s' doesn't exist within context '%s'", contentproviderId, id)); + } + isClient.removeConcept(id, id + CONTENTPROVIDERS_ID_SUFFIX, id + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + contentproviderId); + cc.removeFromCategory(id, CONTENTPROVIDERS_ID_SUFFIX, String.valueOf(contentproviderId)); + } + + public void removeCommunityOrganization(final String id, final Integer organizationId) throws CommunityException, CommunityNotFoundException { + final Map organizations = getCommunityOrganizationMap(id); + if (!organizations.containsKey(organizationId)) { + throw new CommunityNotFoundException(String.format("organization '%s' doesn't exist within context '%s'", organizationId, id)); + } + isClient.removeConcept(id, id + ORGANIZATION_ID_SUFFIX, id + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organizationId); + cc.removeFromCategory(id, ORGANIZATION_ID_SUFFIX, String.valueOf(organizationId)); + } + + public List getCommunityZenodoCommunities(final String id) throws CommunityException, CommunityNotFoundException { + + return cc.getCommunityZenodoCommunities(id); + } + + public List getCommunityOrganizations(final String id) throws CommunityException, CommunityNotFoundException { + cc.getCommunity(id); + return cc.getCommunityInfo(id, ORGANIZATION_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityOrganization(id, c)); + } + + public CommunityDetails addCommunitySubjects(final String id, final List subjects) throws CommunityException, CommunityNotFoundException { + + final CommunityDetails cd = new CommunityDetails(); + + final Set current = Sets.newHashSet(cc.getCommunity(id).getSubjects()); + + current.addAll(subjects); + + cd.setSubjects(Lists.newArrayList(current)); + + setCommunity(id, CommunityWritableProperties.fromDetails(cd)); + + return cd; + } + + public CommunityDetails removeCommunitySubjects(final String id, final List subjects) throws CommunityException, CommunityNotFoundException { + + final CommunityDetails cd = new CommunityDetails(); + + final Set current = Sets.newHashSet(cc.getCommunity(id).getSubjects()); + + current.removeAll(subjects); + + cd.setSubjects(Lists.newArrayList(current)); + + setCommunity(id, CommunityWritableProperties.fromDetails(cd)); + + return cd; + } + + @CacheEvict(value = "community-cache", allEntries = true) + public void removeCommunityZenodoCommunity(final String id, final Integer zenodoCommId) throws CommunityException, CommunityNotFoundException { + + final Map zcomms = getZenodoCommunityMap(id); + if (!zcomms.containsKey(zenodoCommId)) { + throw new CommunityNotFoundException(String.format("Zenodo community '%s' doesn't exist within context '%s'", zenodoCommId, id)); + } + isClient.removeConcept(id, id + ZENODOCOMMUNITY_ID_SUFFIX, id + ZENODOCOMMUNITY_ID_SUFFIX + ID_SEPARATOR + zenodoCommId); + + cc.removeFromCategory(id, ZENODOCOMMUNITY_ID_SUFFIX, String.valueOf(zenodoCommId)); + + } + + @CacheEvict(value = "community-cache", allEntries = true) + public CommunityZenodoCommunity addCommunityZenodoCommunity(final String id, final CommunityZenodoCommunity zc) + throws CommunityException, CommunityNotFoundException { + if (!StringUtils.equalsIgnoreCase(id, zc.getCommunityId())) { throw new CommunityException("parameters 'id' and zc.communityId must be coherent"); } + if (!StringUtils.isNotBlank(zc.getZenodoid())) { throw new CommunityException("parameter zenodoid cannot be null or empty"); } + final TreeMap zcs = getZenodoCommunityMap(id); + + for (final CommunityZenodoCommunity czc : zcs.values()) { + if (czc.getZenodoid().equals(zc.getZenodoid())) { throw new CommunityException("Zenodo community already associated to the RCD"); } + } + + zc.setId(nextId(!zcs.isEmpty() ? zcs.lastKey() : 0)); + + isClient.addConcept(id, id + ZENODOCOMMUNITY_ID_SUFFIX, CommunityMappingUtils.asZenodoCommunityXML(id, zc)); + cc.updateZenodoCommunity(id, zc); + + return zc; + } + + public CommunityOpenAIRECommunities getOpenAIRECommunities(final String zenodoId) throws CommunityException, CommunityNotFoundException { + + if (cci.getInverseZenodoCommunityMap().containsKey(zenodoId)) { + return new CommunityOpenAIRECommunities().setZenodoid(zenodoId) + .setOpenAirecommunitylist(cci.getInverseZenodoCommunityMap().get(zenodoId).stream().collect(Collectors.toList())); + } + return new CommunityOpenAIRECommunities(); + + } + + // HELPERS + + private TreeMap getCommunityProjectMap(final String id) throws CommunityException, CommunityNotFoundException { + return getCommunityProjects(id).stream() + .collect(Collectors.toMap(p -> Integer.valueOf(p.getId()), Functions.identity(), (p1, p2) -> { + log.warn(String.format("duplicate project found: '%s'", p1.getId())); + return p2; + }, TreeMap::new)); + } + + private TreeMap getCommunityContentproviderMap(final String id) throws CommunityException, CommunityNotFoundException { + log.info("getting community content provider map"); + return getCommunityContentproviders(id).stream() + .collect(Collectors.toMap(cp -> Integer.valueOf(cp.getId()), Functions.identity(), (cp1, cp2) -> { + log.warn(String.format("duplicate content provider found: '%s'", cp1.getId())); + return cp2; + }, TreeMap::new)); + } + + private TreeMap getZenodoCommunityMap(final String id) throws CommunityException, CommunityNotFoundException { + return getCommunityZenodoCommunities(id).stream() + .collect(Collectors.toMap(cp -> Integer.valueOf(cp.getId()), Functions.identity(), (cp1, cp2) -> { + log.warn(String.format("duplicate Zenodo community found: '%s'", cp1.getId())); + return cp2; + }, TreeMap::new)); + } + + private TreeMap getCommunityOrganizationMap(final String id) throws CommunityException, CommunityNotFoundException { + return getCommunityOrganizations(id).stream() + .collect(Collectors.toMap(o -> Integer.valueOf(o.getId()), Functions.identity(), (o1, o2) -> { + log.warn(String.format("duplicate content provider found: '%s'", o1.getId())); + return o2; + }, TreeMap::new)); + } + + public CommunityOrganization addCommunityOrganization(final String id, final CommunityOrganization organization) + throws CommunityException, CommunityNotFoundException { + if (!StringUtils.equalsIgnoreCase(id, organization.getCommunityId())) { + throw new CommunityException("parameters 'id' and organization.communityId must be coherent"); + } + + final TreeMap cps = getCommunityOrganizationMap(id); + + final String organization_id = organization.getId(); + if (organization_id != null && cps.keySet().contains(Integer.valueOf(organization_id))) { + if (organization.getName() != null) { + isClient.updateConceptParam(id + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organization_id, CORGANIZATION_NAME, organization.getName()); + } + if (organization.getLogo_url() != null) { + isClient.updateConceptParam(id + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organization_id, CORGANIZATION_LOGOURL, Base64.getEncoder() + .encodeToString(organization.getLogo_url().getBytes())); + } + if (organization.getWebsite_url() != null) { + isClient.updateConceptParam(id + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organization_id, CORGANIZATION_WEBSITEURL, Base64.getEncoder() + .encodeToString(organization.getWebsite_url().getBytes())); + } + } else { + organization.setId(nextId(!cps.isEmpty() ? cps.lastKey() : 0)); + isClient.addConcept(id, id + ORGANIZATION_ID_SUFFIX, CommunityMappingUtils.asOrganizationXML(id, organization)); + } + + cc.updateOrganization(id, organization); + return organization; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityContentprovider.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityContentprovider.java index 9d470dc7..334a3db1 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityContentprovider.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityContentprovider.java @@ -4,16 +4,13 @@ import javax.validation.constraints.NotNull; import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.google.gson.Gson; + import eu.dnetlib.openaire.community.selectioncriteria.SelectionCriteria; import io.swagger.annotations.ApiModelProperty; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; @JsonAutoDetect public class CommunityContentprovider { - private static final Log log = LogFactory.getLog(CommunityContentprovider.class); - @ApiModelProperty(value = "OpenAIRE identifier for this content provider, if available", required = false) private String openaireId; @@ -32,7 +29,7 @@ public class CommunityContentprovider { @ApiModelProperty(value = "content provider official name", required = true) private String officialname; - //@NotNull + // @NotNull @ApiModelProperty(value = "content provider selection criteria", required = false) private SelectionCriteria selectioncriteria; @@ -78,28 +75,26 @@ public class CommunityContentprovider { public SelectionCriteria getSelectioncriteria() { - return this.selectioncriteria; + return this.selectioncriteria; } - public void setSelectioncriteria(SelectionCriteria selectioncriteria) { + public void setSelectioncriteria(final SelectionCriteria selectioncriteria) { this.selectioncriteria = selectioncriteria; } - public String toString(){ - return String.format("id %s, name %s, selection criteria %s" , this.id, this.name, toJson()); + @Override + public String toString() { + return String.format("id %s, name %s, selection criteria %s", this.id, this.name, toJson()); } - public String toJson() { - if (selectioncriteria == null) - return ""; + if (selectioncriteria == null) { return ""; } return new Gson().toJson(selectioncriteria); } public String toXML() { - if (selectioncriteria == null) - return ""; - return ""; + if (selectioncriteria == null) { return ""; } + return ""; } } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityException.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityException.java index 9b2c2746..db2cc2bd 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityException.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityException.java @@ -10,6 +10,11 @@ import org.springframework.web.bind.annotation.ResponseStatus; @ResponseStatus(value = HttpStatus.INTERNAL_SERVER_ERROR) public class CommunityException extends Exception { + /** + * + */ + private static final long serialVersionUID = -4961233580574761346L; + public CommunityException(final String message) { super(message); } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityNotFoundException.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityNotFoundException.java index 3138d64a..53eb016b 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityNotFoundException.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityNotFoundException.java @@ -8,6 +8,11 @@ import org.springframework.web.bind.annotation.ResponseStatus; @ResponseStatus(value = HttpStatus.NOT_FOUND) public class CommunityNotFoundException extends Exception { + /** + * + */ + private static final long serialVersionUID = -5605421323034135778L; + public CommunityNotFoundException(final String msg) { super(msg); } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunitySummary.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunitySummary.java index c3453e0e..e504583e 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunitySummary.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunitySummary.java @@ -1,9 +1,9 @@ package eu.dnetlib.openaire.community; import java.util.Date; -import java.util.List; import com.fasterxml.jackson.annotation.JsonAutoDetect; + import io.swagger.annotations.ApiModelProperty; @JsonAutoDetect @@ -42,21 +42,20 @@ public class CommunitySummary { @ApiModelProperty("Zenodo community associated to this community") protected String zenodoCommunity; - public CommunitySummary() { - } + public CommunitySummary() {} public CommunitySummary( - final String id, - final String queryId, - final String type, - final String name, - final String shortName, - final Date creationDate, - final Date lastUpdateDate, - final String description, - final String logoUrl, - final CommunityStatus status, - final String zenodoCommunity) { + final String id, + final String queryId, + final String type, + final String name, + final String shortName, + final Date creationDate, + final Date lastUpdateDate, + final String description, + final String logoUrl, + final CommunityStatus status, + final String zenodoCommunity) { this.id = id; this.queryId = queryId; this.type = type; @@ -72,16 +71,16 @@ public class CommunitySummary { public CommunitySummary(final CommunitySummary summary) { this(summary.getId(), - summary.getQueryId(), - summary.getType(), - summary.getName(), - summary.getShortName(), - summary.getCreationDate(), - summary.getLastUpdateDate(), - summary.getDescription(), - summary.getLogoUrl(), - summary.getStatus(), - summary.getZenodoCommunity()); + summary.getQueryId(), + summary.getType(), + summary.getName(), + summary.getShortName(), + summary.getCreationDate(), + summary.getLastUpdateDate(), + summary.getDescription(), + summary.getLogoUrl(), + summary.getStatus(), + summary.getZenodoCommunity()); } public String getId() { @@ -124,13 +123,21 @@ public class CommunitySummary { this.shortName = shortName; } - public Date getCreationDate() { return creationDate; } + public Date getCreationDate() { + return creationDate; + } - public void setCreationDate(final Date creationDate) { this.creationDate = creationDate; } + public void setCreationDate(final Date creationDate) { + this.creationDate = creationDate; + } - public Date getLastUpdateDate() { return lastUpdateDate; } + public Date getLastUpdateDate() { + return lastUpdateDate; + } - public void setLastUpdateDate(final Date lastUpdateDate) { this.lastUpdateDate = lastUpdateDate; } + public void setLastUpdateDate(final Date lastUpdateDate) { + this.lastUpdateDate = lastUpdateDate; + } public String getDescription() { return description; @@ -160,7 +167,7 @@ public class CommunitySummary { return zenodoCommunity; } - public void setZenodoCommunity(String zenodoCommunity) { + public void setZenodoCommunity(final String zenodoCommunity) { this.zenodoCommunity = zenodoCommunity; } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/selectioncriteria/Constraint.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/selectioncriteria/Constraint.java index 9f31965c..31c71535 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/selectioncriteria/Constraint.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/selectioncriteria/Constraint.java @@ -1,44 +1,45 @@ package eu.dnetlib.openaire.community.selectioncriteria; -import com.fasterxml.jackson.annotation.JsonAutoDetect; - import java.io.Serializable; +import com.fasterxml.jackson.annotation.JsonAutoDetect; @JsonAutoDetect public class Constraint implements Serializable { - private String verb; - private String field; - private String value; + /** + * + */ + private static final long serialVersionUID = -5996232267609464747L; - public Constraint() { - } + private String verb; + private String field; + private String value; - public String getVerb() { - return verb; - } + public Constraint() {} - public void setVerb(String verb) { - this.verb = verb; - } + public String getVerb() { + return verb; + } - public String getField() { - return field; - } + public void setVerb(final String verb) { + this.verb = verb; + } - public void setField(String field) { - this.field = field; - } + public String getField() { + return field; + } - public String getValue() { - return value; - } - - public void setValue(String value) { - this.value = value; - } + public void setField(final String field) { + this.field = field; + } + public String getValue() { + return value; + } + public void setValue(final String value) { + this.value = value; + } } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/selectioncriteria/Constraints.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/selectioncriteria/Constraints.java index 863d97c8..a899095f 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/selectioncriteria/Constraints.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/selectioncriteria/Constraints.java @@ -1,23 +1,27 @@ package eu.dnetlib.openaire.community.selectioncriteria; -import com.fasterxml.jackson.annotation.JsonAutoDetect; - import java.io.Serializable; import java.util.List; +import com.fasterxml.jackson.annotation.JsonAutoDetect; + @JsonAutoDetect public class Constraints implements Serializable { - private List constraint; + /** + * + */ + private static final long serialVersionUID = 2694950017620361195L; + private List constraint; - public Constraints() { - } - public List getConstraint() { - return constraint; - } + public Constraints() {} - public void setConstraint(List constraint) { - this.constraint = constraint; - } + public List getConstraint() { + return constraint; + } + + public void setConstraint(final List constraint) { + this.constraint = constraint; + } } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/selectioncriteria/SelectionCriteria.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/selectioncriteria/SelectionCriteria.java index 3b0c09f1..0214e41a 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/selectioncriteria/SelectionCriteria.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/selectioncriteria/SelectionCriteria.java @@ -1,30 +1,33 @@ package eu.dnetlib.openaire.community.selectioncriteria; -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.google.gson.Gson; import java.io.Serializable; import java.util.List; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.google.gson.Gson; + @JsonAutoDetect public class SelectionCriteria implements Serializable { - private List criteria; - public SelectionCriteria() { - } + /** + * + */ + private static final long serialVersionUID = 4303936216579280542L; + private List criteria; - public List getCriteria() { - return criteria; - } + public SelectionCriteria() {} - public void setCriteria(List criteria) { - this.criteria = criteria; - } + public List getCriteria() { + return criteria; + } + public void setCriteria(final List criteria) { + this.criteria = criteria; + } + public static SelectionCriteria fromJson(final String json) { + return new Gson().fromJson(json, SelectionCriteria.class); - public static SelectionCriteria fromJson(final String json) { - return new Gson().fromJson(json, SelectionCriteria.class); - - } + } } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextApiCore.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextApiCore.java index 95dea205..9cd2e859 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextApiCore.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextApiCore.java @@ -6,102 +6,96 @@ import java.util.Map; import java.util.stream.Collectors; import java.util.stream.Stream; -import com.google.common.base.Splitter; -import com.google.common.collect.Lists; -import eu.dnetlib.openaire.common.ISClient; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; +import com.google.common.base.Splitter; +import com.google.common.collect.Lists; + +import eu.dnetlib.openaire.common.ISClient; + @Component @ConditionalOnProperty(value = "openaire.exporter.enable.context", havingValue = "true") public class ContextApiCore { - private static final Log log = LogFactory.getLog(ContextApiCore.class); private static final String SEPARATOR = "::"; @Autowired private ISClient isClient; public List listContexts(final List type) throws ContextException { - return getContextMap(type).values().stream() - .map(c -> new ContextSummary() - .setId(c.getId()) - .setType(c.getType()) - .setLabel(c.getLabel()) - .setStatus(c.getParams().containsKey("status") ? c.getParams().get("status").get(0).getValue() : "")) - .collect(Collectors.toList()); + return getContextMap(type).values() + .stream() + .map(c -> new ContextSummary() + .setId(c.getId()) + .setType(c.getType()) + .setLabel(c.getLabel()) + .setStatus(c.getParams().containsKey("status") ? c.getParams().get("status").get(0).getValue() : "")) + .collect(Collectors.toList()); } - public List listCategories(final String contextId, Boolean all) throws ContextException { + public List listCategories(final String contextId, final Boolean all) throws ContextException { final Stream categories = getContextMap().get(contextId).getCategories().values().stream(); return all ? asCategorySummaries(categories) : asCategorySummaries(categories.filter(Category::isClaim)); } - private List asCategorySummaries(Stream categories) { + private List asCategorySummaries(final Stream categories) { return categories - .map(c -> new CategorySummary() - .setId(c.getId()) - .setLabel(c.getLabel()) - .setHasConcept(c.hasConcepts())) - .collect(Collectors.toList()); + .map(c -> new CategorySummary() + .setId(c.getId()) + .setLabel(c.getLabel()) + .setHasConcept(c.hasConcepts())) + .collect(Collectors.toList()); } - public List listConcepts(final String categoryId, Boolean all) throws ContextException { + public List listConcepts(final String categoryId, final Boolean all) throws ContextException { final String contextId = StringUtils.substringBefore(categoryId, SEPARATOR); final Stream concepts = getContextMap().get(contextId) - .getCategories() - .get(categoryId) - .getConcepts() - .stream(); + .getCategories() + .get(categoryId) + .getConcepts() + .stream(); return all ? asConceptSummaries(concepts) : asConceptSummaries(concepts.filter(Concept::isClaim)); } - private List asConceptSummaries(Stream concepts) { + private List asConceptSummaries(final Stream concepts) { return concepts - .map(c -> new ConceptSummary() - .setId(c.getId()) - .setLabel(c.getLabel()) - .setHasSubConcept(c.hasSubConcepts())) - .collect(Collectors.toList()); + .map(c -> new ConceptSummary() + .setId(c.getId()) + .setLabel(c.getLabel()) + .setHasSubConcept(c.hasSubConcepts())) + .collect(Collectors.toList()); } - public List listSubConcepts(final String conceptId, Boolean all) throws ContextException { + public List listSubConcepts(final String conceptId, final Boolean all) throws ContextException { final List ids = Splitter.on(SEPARATOR).splitToList(conceptId); - if (ids.size() < 3) { - throw new ContextException(""); - } + if (ids.size() < 3) { throw new ContextException(""); } final String contextId = ids.get(0); final String categoryId = contextId + SEPARATOR + ids.get(1); final Stream concepts = getContextMap().get(contextId) - .getCategories() - .get(categoryId) - .getConcepts() - .stream() - .filter(c -> conceptId.equals(c.getId())); + .getCategories() + .get(categoryId) + .getConcepts() + .stream() + .filter(c -> conceptId.equals(c.getId())); - return all ? - mapConcepts(concepts.filter(Concept::isClaim).collect(Collectors.toList())) : - mapConcepts(concepts.collect(Collectors.toList())); + return all ? mapConcepts(concepts.filter(Concept::isClaim).collect(Collectors.toList())) : mapConcepts(concepts.collect(Collectors.toList())); } private List mapConcepts(final List concepts) { - if (concepts == null || concepts.isEmpty()) { - return null; - } + if (concepts == null || concepts.isEmpty()) { return null; } return concepts.stream() - .map(c -> new ConceptSummary() - .setId(c.getId()) - .setLabel(c.getLabel()) - .setHasSubConcept(c.hasSubConcepts()) - .setConcept(mapConcepts(c.getConcepts()))) - .collect(Collectors.toList()); + .map(c -> new ConceptSummary() + .setId(c.getId()) + .setLabel(c.getLabel()) + .setHasSubConcept(c.hasSubConcepts()) + .setConcept(mapConcepts(c.getConcepts()))) + .collect(Collectors.toList()); } private Map getContextMap() throws ContextException { @@ -111,10 +105,9 @@ public class ContextApiCore { private Map getContextMap(final List type) throws ContextException { try { return isClient.getContextMap(type); - } catch (IOException e) { + } catch (final IOException e) { throw new ContextException(e); } } - } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextException.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextException.java index be1fb392..7d092b96 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextException.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextException.java @@ -10,6 +10,11 @@ import org.springframework.web.bind.annotation.ResponseStatus; @ResponseStatus(value = HttpStatus.INTERNAL_SERVER_ERROR) public class ContextException extends Exception { + /** + * + */ + private static final long serialVersionUID = -5489369676370127052L; + public ContextException(final String message) { super(message); } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextNotFoundException.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextNotFoundException.java index 212b5334..ba87774c 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextNotFoundException.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextNotFoundException.java @@ -8,6 +8,11 @@ import org.springframework.web.bind.annotation.ResponseStatus; @ResponseStatus(value = HttpStatus.NOT_FOUND) public class ContextNotFoundException extends Exception { + /** + * + */ + private static final long serialVersionUID = -2026506752817353752L; + public ContextNotFoundException(final String msg) { super(msg); } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmApiController.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmApiController.java index 1a5d7598..460a9970 100755 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmApiController.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmApiController.java @@ -162,9 +162,9 @@ public class DsmApiController extends AbstractExporterController { @ApiResponse(code = 200, message = "OK", response = DatasourceResponse.class), @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) - public SimpleResponse recentRegistered(@PathVariable final int size) throws Throwable { + public SimpleResponse recentRegistered(@PathVariable final int size) throws Throwable { final StopWatch stop = StopWatch.createStarted(); - final SimpleResponse rsp = dsmCore.searchRecentRegistered(size); + final SimpleResponse rsp = dsmCore.searchRecentRegistered(size); return prepareResponse(1, size, stop, rsp); } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmCore.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmCore.java index 437a58cb..f4721e08 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmCore.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmCore.java @@ -9,18 +9,11 @@ import java.nio.charset.Charset; import java.util.Collection; import java.util.HashSet; import java.util.List; -import java.util.Queue; import java.util.Set; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; -import javax.annotation.PostConstruct; - import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; @@ -33,37 +26,23 @@ import org.springframework.jdbc.core.BeanPropertyRowMapper; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Component; -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.common.util.concurrent.MoreExecutors; -import com.google.common.util.concurrent.ThreadFactoryBuilder; - -import eu.dnetlib.OpenaireExporterConfig; import eu.dnetlib.enabling.datasources.common.AggregationInfo; -import eu.dnetlib.enabling.datasources.common.AggregationStage; -import eu.dnetlib.enabling.datasources.common.Datasource; import eu.dnetlib.enabling.datasources.common.DsmException; import eu.dnetlib.enabling.datasources.common.DsmForbiddenException; import eu.dnetlib.enabling.datasources.common.DsmNotFoundException; import eu.dnetlib.openaire.common.ISClient; import eu.dnetlib.openaire.community.CommunityClient; import eu.dnetlib.openaire.dsm.dao.DatasourceDao; -import eu.dnetlib.openaire.dsm.dao.DatasourceIndexClient; import eu.dnetlib.openaire.dsm.dao.MongoLoggerClient; -import eu.dnetlib.openaire.dsm.dao.ObjectStoreClient; import eu.dnetlib.openaire.dsm.dao.ResponseUtils; import eu.dnetlib.openaire.dsm.dao.VocabularyClient; import eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils; -import eu.dnetlib.openaire.dsm.dao.utils.IndexDsInfo; -import eu.dnetlib.openaire.dsm.dao.utils.IndexRecordsInfo; import eu.dnetlib.openaire.dsm.domain.AggregationHistoryResponse; import eu.dnetlib.openaire.dsm.domain.ApiDetails; import eu.dnetlib.openaire.dsm.domain.ApiDetailsResponse; import eu.dnetlib.openaire.dsm.domain.DatasourceDetailResponse; import eu.dnetlib.openaire.dsm.domain.DatasourceDetails; import eu.dnetlib.openaire.dsm.domain.DatasourceDetailsUpdate; -import eu.dnetlib.openaire.dsm.domain.DatasourceInfo; import eu.dnetlib.openaire.dsm.domain.DatasourceSnippetResponse; import eu.dnetlib.openaire.dsm.domain.RegisteredDatasourceInfo; import eu.dnetlib.openaire.dsm.domain.RequestFilter; @@ -87,20 +66,11 @@ public class DsmCore { @Autowired private ISClient isClient; - @Autowired - private ObjectStoreClient objectStoreClient; - - @Autowired - private DatasourceIndexClient datasourceIndexClient; - @Autowired private VocabularyClient vocabularyClient; @Autowired - private DatasourceDao dsDao; - - @Autowired - private OpenaireExporterConfig config; + private DatasourceDao dsDao; @Autowired private JdbcTemplate jdbcTemplate; @@ -108,14 +78,6 @@ public class DsmCore { @Autowired private CommunityClient communityClient; - private ListeningExecutorService executor; - - @PostConstruct - public void init() { - executor = MoreExecutors.listeningDecorator(new ScheduledThreadPoolExecutor(config.getRequestWorkers(), - new ThreadFactoryBuilder().setNameFormat("dsm-client-%d").build())); - } - public List listCountries() throws DsmException { try { return dsDao.listCountries(); @@ -182,7 +144,7 @@ public class DsmCore { public ApiDetailsResponse getApis(final String dsId) throws DsmException { try { - final List apis = dsDao.getApis(dsId); + final List apis = dsDao.getApis(dsId); final List api = apis.stream() .map(DsmMappingUtils::asDetails) .collect(Collectors.toList()); @@ -218,25 +180,24 @@ public class DsmCore { public void updateDatasource(final DatasourceDetailsUpdate d) throws DsmException, DsmNotFoundException { try { // initialize with current values from DB - final Datasource ds = dsDao.getDs(d.getId()); - final DatasourceDbEntry dbEntry = (DatasourceDbEntry) ds; + final DatasourceDbEntry ds = dsDao.getDs(d.getId()); - if (dbEntry == null) { throw new DsmNotFoundException(String.format("ds '%s' does not exist", d.getId())); } + if (ds == null) { throw new DsmNotFoundException(String.format("ds '%s' does not exist", d.getId())); } final DatasourceDbEntry update = asDbEntry(d); if (d.getIdentities() != null) { final Set identities = new HashSet<>( - Stream.of(update.getIdentities(), dbEntry.getIdentities()) + Stream.of(update.getIdentities(), ds.getIdentities()) .flatMap(Collection::stream) .collect(Collectors.toMap(i -> i.getIssuertype() + i.getPid(), Function.identity(), (i1, i2) -> i1)) .values()); - copyNonNullProperties(update, dbEntry); - dbEntry.setIdentities(identities); + copyNonNullProperties(update, ds); + ds.setIdentities(identities); } else { - copyNonNullProperties(update, dbEntry); + copyNonNullProperties(update, ds); } - dsDao.saveDs(dbEntry); + dsDao.saveDs(ds); } catch (final Throwable e) { log.error(ExceptionUtils.getStackTrace(e)); throw e; @@ -287,112 +248,7 @@ public class DsmCore { // HELPERS ////////////// - private DatasourceInfo enrichDatasourceInfo(final DatasourceDetails d, final CountDownLatch outerLatch, final Queue errors) { - final DatasourceInfo dsInfo = new DatasourceInfo().setDatasource(d); - getAggregationHistory(d.getId(), outerLatch, errors, dsInfo); - getIndexDsInfo(d.getId(), outerLatch, errors, dsInfo); - return dsInfo; - } - - private void getAggregationHistory(final String dsId, - final CountDownLatch outerLatch, - final Queue errors, - final DatasourceInfo datasourceInfo) { - Futures.addCallback(executor.submit(() -> mongoLoggerClient.getAggregationHistory(dsId)), new FutureCallback>() { - - @Override - public void onSuccess(final List info) { - setAggregationHistory(datasourceInfo, info); - outerLatch.countDown(); - } - - @Override - public void onFailure(final Throwable e) { - log.error(ExceptionUtils.getStackTrace(e)); - errors.offer(e); - outerLatch.countDown(); - } - }, executor); - } - - private void setAggregationHistory(final DatasourceInfo datasourceInfo, final List info) { - datasourceInfo.setAggregationHistory(info); - if (!info.isEmpty()) { - datasourceInfo - .setLastCollection(info.stream().filter(a -> AggregationStage.COLLECT.equals(a.getAggregationStage())).findFirst().get()) - .setLastTransformation(info.stream().filter(a -> AggregationStage.TRANSFORM.equals(a.getAggregationStage())).findFirst().get()); - } - } - - private void getIndexDsInfo(final String dsId, - final CountDownLatch outerLatch, - final Queue errors, - final DatasourceInfo datasourceInfo) { - Futures.addCallback(executor.submit(() -> isClient.calculateCurrentIndexDsInfo()), new FutureCallback() { - - @Override - public void onSuccess(final IndexDsInfo info) { - - final CountDownLatch innerLatch = new CountDownLatch(2); - - Futures.addCallback(executor.submit(() -> datasourceIndexClient.getIndexInfo(dsId, info, errors)), new FutureCallback() { - - @Override - public void onSuccess(final IndexRecordsInfo info) { - datasourceInfo - .setIndexRecords(info.getTotal()) - .setFundedContent(info.getFunded()) - .setLastIndexingDate(info.getDate()); - innerLatch.countDown(); - } - - @Override - public void onFailure(final Throwable e) { - errors.offer(e); - innerLatch.countDown(); - } - }, executor); - - Futures.addCallback(executor.submit(() -> objectStoreClient.getObjectStoreSize(isClient.getObjectStoreId(dsId))), new FutureCallback() { - - @Override - public void onSuccess(final Long objectStoreSize) { - datasourceInfo.setFulltexts(objectStoreSize); - innerLatch.countDown(); - } - - @Override - public void onFailure(final Throwable e) { - errors.offer(e); - innerLatch.countDown(); - } - }, executor); - - waitLatch(innerLatch, errors, config.getRequestTimeout()); - - outerLatch.countDown(); - } - - @Override - public void onFailure(final Throwable e) { - // log.error(ExceptionUtils.getStackTrace(e)); - errors.offer(e); - outerLatch.countDown(); - } - }, executor); - } - - private void waitLatch(final CountDownLatch latch, final Queue errors, final int waitSeconds) { - try { - if (!latch.await(waitSeconds, TimeUnit.SECONDS)) { - errors.offer(new TimeoutException("Waiting for requests to complete has timed out.")); - } - } catch (final InterruptedException e) { - errors.offer(e); - } - } - - public SimpleResponse searchRecentRegistered(final int size) throws Throwable { + public SimpleResponse searchRecentRegistered(final int size) throws Throwable { try { final String sql = IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/recent_registered_datasources.sql.st"), Charset.defaultCharset()); @@ -413,17 +269,13 @@ public class DsmCore { IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate_typology.st.sql"), Charset .defaultCharset()); - return jdbcTemplate.queryForObject(sql, new Object[] { - fromDate, typologyFilter + "%" - }, Long.class); + return jdbcTemplate.queryForObject(sql, Long.class, fromDate, typologyFilter + "%"); } else { final String sql = IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate.st.sql"), Charset .defaultCharset()); - return jdbcTemplate.queryForObject(sql, new Object[] { - fromDate - }, Long.class); + return jdbcTemplate.queryForObject(sql, Long.class, fromDate); } } catch (final Throwable e) { diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDaoImpl.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDaoImpl.java index cb53f505..09fae5bc 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDaoImpl.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDaoImpl.java @@ -1,19 +1,20 @@ package eu.dnetlib.openaire.dsm.dao; -import com.google.common.collect.Lists; -import eu.dnetlib.OpenaireExporterConfig; -import eu.dnetlib.enabling.datasources.common.DsmException; -import eu.dnetlib.enabling.datasources.common.DsmForbiddenException; -import eu.dnetlib.enabling.datasources.common.DsmNotFoundException; -import eu.dnetlib.openaire.dsm.domain.RequestFilter; -import eu.dnetlib.openaire.dsm.domain.RequestSort; -import eu.dnetlib.openaire.dsm.domain.RequestSortOrder; -import eu.dnetlib.openaire.dsm.domain.db.ApiDbEntry; -import eu.dnetlib.openaire.dsm.domain.db.ApiParamDbEntry; -import eu.dnetlib.openaire.dsm.domain.db.DatasourceApiDbEntry; -import eu.dnetlib.openaire.dsm.domain.db.DatasourceDbEntry; -import eu.dnetlib.openaire.vocabularies.Country; -import eu.dnetlib.openaire.vocabularies.Vocabulary; +import static eu.dnetlib.openaire.common.ExporterConstants.OAI; +import static eu.dnetlib.openaire.common.ExporterConstants.SET; +import static eu.dnetlib.openaire.dsm.dao.DatasourceSpecs.apiSpec; +import static eu.dnetlib.openaire.dsm.dao.DatasourceSpecs.dsRegisteredbyNotNullSpec; +import static eu.dnetlib.openaire.dsm.dao.DatasourceSpecs.dsSpec; + +import java.sql.Date; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +import javax.persistence.EntityNotFoundException; + import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -26,20 +27,21 @@ import org.springframework.data.jpa.domain.Specification; import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Transactional; -import javax.persistence.EntityNotFoundException; -import java.sql.Date; -import java.util.HashSet; -import java.util.List; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; - -import static eu.dnetlib.openaire.common.ExporterConstants.OAI; -import static eu.dnetlib.openaire.common.ExporterConstants.SET; -import static eu.dnetlib.openaire.dsm.dao.DatasourceSpecs.apiSpec; -import static eu.dnetlib.openaire.dsm.dao.DatasourceSpecs.dsSpec; -import static eu.dnetlib.openaire.dsm.dao.DatasourceSpecs.dsRegisteredbyNotNullSpec; +import com.google.common.collect.Lists; +import eu.dnetlib.DnetOpenaireExporterProperties; +import eu.dnetlib.enabling.datasources.common.DsmException; +import eu.dnetlib.enabling.datasources.common.DsmForbiddenException; +import eu.dnetlib.enabling.datasources.common.DsmNotFoundException; +import eu.dnetlib.openaire.dsm.domain.RequestFilter; +import eu.dnetlib.openaire.dsm.domain.RequestSort; +import eu.dnetlib.openaire.dsm.domain.RequestSortOrder; +import eu.dnetlib.openaire.dsm.domain.db.ApiDbEntry; +import eu.dnetlib.openaire.dsm.domain.db.ApiParamDbEntry; +import eu.dnetlib.openaire.dsm.domain.db.DatasourceApiDbEntry; +import eu.dnetlib.openaire.dsm.domain.db.DatasourceDbEntry; +import eu.dnetlib.openaire.vocabularies.Country; +import eu.dnetlib.openaire.vocabularies.Vocabulary; /** * Created by claudio on 20/10/2016. @@ -51,7 +53,7 @@ public class DatasourceDaoImpl implements DatasourceDao listCountries() throws DsmException { final List countries = Lists.newArrayList(); final Vocabulary v = vocabularyClient.getCountries(); - countries.addAll(countryTermRepository.findAll().stream() - .filter(Objects::nonNull) - .map(t -> new Country(t.getTerm(), v.getEnglishName(t.getTerm()))) - .collect(Collectors.toList())); + countries.addAll(countryTermRepository.findAll() + .stream() + .filter(Objects::nonNull) + .map(t -> new Country(t.getTerm(), v.getEnglishName(t.getTerm()))) + .collect(Collectors.toList())); return countries; } @Override - public Page search(final RequestSort requestSortBy, RequestSortOrder order, RequestFilter requestFilter, final int page, final int size) - throws DsmException { + public Page search(final RequestSort requestSortBy, + final RequestSortOrder order, + final RequestFilter requestFilter, + final int page, + final int size) + throws DsmException { final Specification spec = dsSpec(requestSortBy, order, requestFilter); return dsRepository.findAll(spec, PageRequest.of(page, size)); } - @Override - public Page searchRegistered(final RequestSort requestSortBy, RequestSortOrder order, RequestFilter requestFilter, final int page, final int size) - throws DsmException { + public Page searchRegistered(final RequestSort requestSortBy, + final RequestSortOrder order, + final RequestFilter requestFilter, + final int page, + final int size) + throws DsmException { final Specification spec = dsSpec(requestSortBy, order, requestFilter).and(dsRegisteredbyNotNullSpec()); return dsRepository.findAll(spec, PageRequest.of(page, size)); @@ -98,7 +108,7 @@ public class DatasourceDaoImpl implements DatasourceDao new DsmException("Datasource not found. ID: " + dsId)); } @Override @@ -114,7 +124,7 @@ public class DatasourceDaoImpl implements DatasourceDao new DsmNotFoundException("Api not found. ID: " + apiId)); try { - if (!api.getRemovable()) { - throw new DsmForbiddenException(HttpStatus.SC_UNAUTHORIZED, "api is not removable"); - } + if (!api.getRemovable()) { throw new DsmForbiddenException(HttpStatus.SC_UNAUTHORIZED, "api is not removable"); } apiRepository.deleteById(apiId); log.info(String.format("deleted api '%s'", apiId)); - } catch (EntityNotFoundException e) { + } catch (final EntityNotFoundException e) { throw new DsmNotFoundException(HttpStatus.SC_NOT_FOUND, "api not found"); } } @@ -167,7 +175,7 @@ public class DatasourceDaoImpl implements DatasourceDao new DsmNotFoundException("Api not found. ID: " + apiId)); if (OAI.equalsIgnoreCase(api.getProtocol())) { final Set apiParams = api.getApiParams(); @@ -211,10 +219,12 @@ public class DatasourceDaoImpl implements DatasourceDao findApiBaseURLs(final RequestFilter requestFilter, final int page, final int size) throws DsmException { final PageRequest pageable = PageRequest.of(page, size); final Specification spec = apiSpec(requestFilter); - final Set set = dsApiRepository.findAll(spec, pageable).getContent().stream() - .map(DatasourceApiDbEntry::getBaseurl) - .filter(StringUtils::isNotBlank) - .collect(Collectors.toCollection(HashSet::new)); + final Set set = dsApiRepository.findAll(spec, pageable) + .getContent() + .stream() + .map(DatasourceApiDbEntry::getBaseurl) + .filter(StringUtils::isNotBlank) + .collect(Collectors.toCollection(HashSet::new)); return Lists.newArrayList(set); } @@ -228,11 +238,9 @@ public class DatasourceDaoImpl implements DatasourceDao, JpaSpecificationExecutor { - DatasourceDbEntry findOneById(String id); - @Query("select d.managed from #{#entityName} d where d.id = ?1") boolean isManaged(String id); diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceIndexClientImpl.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceIndexClientImpl.java index 0788bed1..659c67a0 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceIndexClientImpl.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceIndexClientImpl.java @@ -12,7 +12,7 @@ import javax.annotation.PreDestroy; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.util.concurrent.*; -import eu.dnetlib.OpenaireExporterConfig; +import eu.dnetlib.DnetOpenaireExporterProperties; import eu.dnetlib.enabling.datasources.common.DsmException; import eu.dnetlib.miscutils.functional.hash.Hashing; import eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils; @@ -46,7 +46,7 @@ public class DatasourceIndexClientImpl implements DatasourceIndexClient { public static final String DSVERSION = "__dsversion"; @Autowired - private OpenaireExporterConfig config; + private DnetOpenaireExporterProperties config; private ListeningExecutorService executor; diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/MongoLoggerClientImpl.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/MongoLoggerClientImpl.java index 978ccefa..997e936b 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/MongoLoggerClientImpl.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/MongoLoggerClientImpl.java @@ -16,8 +16,8 @@ import com.mongodb.BasicDBObject; import com.mongodb.MongoClient; import com.mongodb.client.FindIterable; import com.mongodb.client.MongoCollection; -import eu.dnetlib.OpenaireExporterConfig; -import eu.dnetlib.OpenaireExporterConfig.Datasource; +import eu.dnetlib.DnetOpenaireExporterProperties; +import eu.dnetlib.DnetOpenaireExporterProperties.Datasource; import eu.dnetlib.enabling.datasources.common.AggregationInfo; import eu.dnetlib.enabling.datasources.common.AggregationStage; import eu.dnetlib.enabling.datasources.common.DsmException; @@ -57,7 +57,7 @@ public class MongoLoggerClientImpl implements MongoLoggerClient { private MongoClient datasourcePublisherMongoClient; @Autowired - private OpenaireExporterConfig config; + private DnetOpenaireExporterProperties config; private final static String LOADTIME = "loadtime"; private final LoadingCache loadingCache = CacheBuilder.newBuilder() diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ResponseUtils.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ResponseUtils.java index b417a754..a2d422b8 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ResponseUtils.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ResponseUtils.java @@ -5,7 +5,16 @@ import java.util.Queue; import com.google.common.collect.Lists; -import eu.dnetlib.openaire.dsm.domain.*; +import eu.dnetlib.openaire.dsm.domain.ApiDetails; +import eu.dnetlib.openaire.dsm.domain.ApiDetailsResponse; +import eu.dnetlib.openaire.dsm.domain.DatasourceDetailResponse; +import eu.dnetlib.openaire.dsm.domain.DatasourceDetails; +import eu.dnetlib.openaire.dsm.domain.DatasourceInfo; +import eu.dnetlib.openaire.dsm.domain.DatasourceSearchResponse; +import eu.dnetlib.openaire.dsm.domain.DatasourceSnippetExtended; +import eu.dnetlib.openaire.dsm.domain.DatasourceSnippetResponse; +import eu.dnetlib.openaire.dsm.domain.Header; +import eu.dnetlib.openaire.dsm.domain.SimpleResponse; public class ResponseUtils { @@ -33,18 +42,17 @@ public class ResponseUtils { return rsp; } - public static Header header(final Queue errors, final long total) { return Header.newInsance() - .setExceptions(errors) - .setTotal(total); + .setExceptions(errors) + .setTotal(total); } public static Header header(final long total) { return header(Lists.newLinkedList(), total); } - public static SimpleResponse simpleResponse(final List list) { + public static SimpleResponse simpleResponse(final List list) { final SimpleResponse rsp = new SimpleResponse().setResponse(list);; rsp.setHeader(header(Lists.newLinkedList(), list.size())); return rsp; diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/VocabularyClientImpl.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/VocabularyClientImpl.java index 82fa2fb7..3a9ad1b8 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/VocabularyClientImpl.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/VocabularyClientImpl.java @@ -1,6 +1,6 @@ package eu.dnetlib.openaire.dsm.dao; -import eu.dnetlib.OpenaireExporterConfig; +import eu.dnetlib.DnetOpenaireExporterProperties; import eu.dnetlib.enabling.datasources.common.DsmException; import eu.dnetlib.openaire.vocabularies.Vocabulary; import org.apache.commons.logging.Log; @@ -22,7 +22,7 @@ public class VocabularyClientImpl implements VocabularyClient { private static final Log log = LogFactory.getLog(VocabularyClientImpl.class); @Autowired - private OpenaireExporterConfig config; + private DnetOpenaireExporterProperties config; @Override @Cacheable("vocabularies-cache") diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/RequestFilter.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/RequestFilter.java index 51858e94..4275db07 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/RequestFilter.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/RequestFilter.java @@ -3,12 +3,18 @@ package eu.dnetlib.openaire.dsm.domain; import java.util.HashMap; import com.fasterxml.jackson.annotation.JsonAutoDetect; + import io.swagger.annotations.ApiModel; @JsonAutoDetect @ApiModel(value = "Request filter", description = "field name and value pairs") public class RequestFilter extends HashMap { + /** + * + */ + private static final long serialVersionUID = 5501969842482508379L; + public RequestFilter() {} } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/SimpleResponse.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/SimpleResponse.java index f9a1d782..cb79eb9c 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/SimpleResponse.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/SimpleResponse.java @@ -18,7 +18,7 @@ public class SimpleResponse extends Response { return response; } - public SimpleResponse setResponse(final List response) { + public SimpleResponse setResponse(final List response) { this.response = response; return this; } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/ApiDbEntry.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/ApiDbEntry.java index 0f64a3e4..04a7b7e9 100755 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/ApiDbEntry.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/ApiDbEntry.java @@ -20,7 +20,7 @@ public class ApiDbEntry extends Api { return compatibilityOverride; } - public Api setCompatibilityOverride(final String compatibilityOverride) { + public Api setCompatibilityOverride(final String compatibilityOverride) { this.compatibilityOverride = compatibilityOverride; return this; } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/ApiParamKeyDbEntry.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/ApiParamKeyDbEntry.java index bbb51391..7ffa6e08 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/ApiParamKeyDbEntry.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/ApiParamKeyDbEntry.java @@ -4,6 +4,7 @@ import javax.persistence.Embeddable; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + import eu.dnetlib.enabling.datasources.common.ApiParamKey; /** @@ -13,6 +14,11 @@ import eu.dnetlib.enabling.datasources.common.ApiParamKey; @JsonIgnoreProperties(ignoreUnknown = true) public class ApiParamKeyDbEntry extends ApiParamKey { + /** + * + */ + private static final long serialVersionUID = 1L; + @Override @JsonIgnore public ApiDbEntry getApi() { diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/DatasourceApiDbEntry.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/DatasourceApiDbEntry.java index 618072c6..5b719b1f 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/DatasourceApiDbEntry.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/DatasourceApiDbEntry.java @@ -172,4 +172,12 @@ public class DatasourceApiDbEntry { public void setId(String id) { this.id = id; } + + public String getCollectedfrom() { + return collectedfrom; + } + + public void setCollectedfrom(String collectedfrom) { + this.collectedfrom = collectedfrom; + } } \ No newline at end of file diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderDao.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderDao.java index 31f65b4d..7c53b2c3 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderDao.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderDao.java @@ -3,43 +3,41 @@ package eu.dnetlib.openaire.funders; import java.util.List; import java.util.stream.Collectors; -import eu.dnetlib.openaire.funders.domain.ConversionUtils; -import eu.dnetlib.openaire.funders.domain.ExtendedFunderDetails; -import eu.dnetlib.openaire.funders.domain.FunderDetails; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.data.domain.PageRequest; import org.springframework.stereotype.Component; +import eu.dnetlib.openaire.funders.domain.ConversionUtils; +import eu.dnetlib.openaire.funders.domain.ExtendedFunderDetails; +import eu.dnetlib.openaire.funders.domain.FunderDetails; + @Component @ConditionalOnProperty(value = "openaire.exporter.enable.funders", havingValue = "true") public class FunderDao { - private static final Log log = LogFactory.getLog(FunderDao.class); - @Autowired private FunderRepository funderRepository; public ExtendedFunderDetails getExtendedFunderDetails(final String funderId) throws FundersApiException { - return ConversionUtils.asExtendedFunderDetails(funderRepository.getOne(funderId)); + return ConversionUtils + .asExtendedFunderDetails(funderRepository.findById(funderId).orElseThrow(() -> new FundersApiException("Funder not found. ID: " + funderId))); } public List listFunderDetails(final int page, final int size) throws FundersApiException { return funderRepository.findAll(PageRequest.of(page, size)) - .getContent() - .stream() - .map(ConversionUtils::asFunderDetails) - .collect(Collectors.toList()); + .getContent() + .stream() + .map(ConversionUtils::asFunderDetails) + .collect(Collectors.toList()); } public List listFunderIds(final int page, final int size) throws FundersApiException { return funderRepository.findAll(PageRequest.of(page, size)) - .getContent() - .stream() - .map(f -> f.getId()) - .collect(Collectors.toList()); + .getContent() + .stream() + .map(f -> f.getId()) + .collect(Collectors.toList()); } } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiException.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiException.java index 62103670..c8f2f024 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiException.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiException.java @@ -2,8 +2,12 @@ package eu.dnetlib.openaire.funders; public class FundersApiException extends Exception { - public FundersApiException() { - } + /** + * + */ + private static final long serialVersionUID = 842353818131133522L; + + public FundersApiException() {} public FundersApiException(final String message) { super(message); diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/info/InfoController.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/info/InfoController.java index e7fc9eb8..a3027500 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/info/InfoController.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/info/InfoController.java @@ -1,24 +1,32 @@ package eu.dnetlib.openaire.info; - -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import eu.dnetlib.openaire.common.AbstractExporterController; -import eu.dnetlib.openaire.common.ExporterConstants; -import io.swagger.annotations.*; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.format.annotation.DateTimeFormat; -import org.springframework.web.bind.annotation.*; import java.time.LocalDate; import java.util.List; import java.util.Map; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.web.bind.annotation.CrossOrigin; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +import eu.dnetlib.openaire.common.AbstractExporterController; +import eu.dnetlib.openaire.common.ExporterConstants; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiResponse; +import io.swagger.annotations.ApiResponses; @RestController -@CrossOrigin(origins = { "*" }) +@CrossOrigin(origins = { + "*" +}) @ConditionalOnProperty(value = "openaire.exporter.enable.info", havingValue = "true") @io.swagger.annotations.Api(tags = "OpenAIRE Info API", description = "the OpenAIRE info API") public class InfoController extends AbstractExporterController { @@ -30,51 +38,66 @@ public class InfoController extends AbstractExporterController { @Autowired private JdbcInfoDao jdbcInfoDao; - @RequestMapping(value = "/info/{infoKey}", produces = { "application/json" }, method = RequestMethod.GET) - @ApiOperation(value = "get info date", notes = "get info date", tags = { ExporterConstants.R }, response = LocalDate.class) + @RequestMapping(value = "/info/{infoKey}", produces = { + "application/json" + }, method = RequestMethod.GET) + @ApiOperation(value = "get info date", notes = "get info date", tags = { + ExporterConstants.R + }, response = LocalDate.class) @ApiResponses(value = { - @ApiResponse(code = 200, message = "OK", response = LocalDate.class), - @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) - public LocalDate getDate(@PathVariable final String infoKey){ - JdbcInfoDao.DATE_INFO info = JdbcInfoDao.DATE_INFO.valueOf(infoKey); - if(info == null) throw new RuntimeException(infoKey + " not recognized"); + @ApiResponse(code = 200, message = "OK", response = LocalDate.class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) + }) + public LocalDate getDate(@PathVariable final String infoKey) { + final JdbcInfoDao.DATE_INFO info = JdbcInfoDao.DATE_INFO.valueOf(infoKey); + if (info == null) { throw new RuntimeException(infoKey + " not recognized"); } return jdbcInfoDao.getDate(info); } - @RequestMapping(value = "/info", produces = { "application/json" }, method = RequestMethod.GET) - @ApiOperation(value = "get all the info date", notes = "get all the info date", tags = { ExporterConstants.R }, response = Map.class) + @RequestMapping(value = "/info", produces = { + "application/json" + }, method = RequestMethod.GET) + @ApiOperation(value = "get all the info date", notes = "get all the info date", tags = { + ExporterConstants.R + }, response = Map.class) @ApiResponses(value = { - @ApiResponse(code = 200, message = "OK", response = LocalDate.class), - @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) - public Map listInfo(){ - Map map = Maps.newHashMap(); - for(JdbcInfoDao.DATE_INFO dateInfo : JdbcInfoDao.DATE_INFO.values()){ + @ApiResponse(code = 200, message = "OK", response = LocalDate.class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) + }) + public Map listInfo() { + final Map map = Maps.newHashMap(); + for (final JdbcInfoDao.DATE_INFO dateInfo : JdbcInfoDao.DATE_INFO.values()) { map.put(dateInfo.name(), jdbcInfoDao.getDate(dateInfo)); } return map; } - @RequestMapping(value = "/info/keys", produces = { "application/json" }, method = RequestMethod.GET) - @ApiOperation(value = "get the available keys", notes = "get the available keys", tags = { ExporterConstants.R }, - response = String.class, responseContainer = "List") + @RequestMapping(value = "/info/keys", produces = { + "application/json" + }, method = RequestMethod.GET) + @ApiOperation(value = "get the available keys", notes = "get the available keys", tags = { + ExporterConstants.R + }, response = String.class, responseContainer = "List") @ApiResponses(value = { - @ApiResponse(code = 200, message = "OK", response = LocalDate.class), - @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) - public List listInfoKeys(){ - List keys = Lists.newArrayList(); - for(JdbcInfoDao.DATE_INFO dateInfo : JdbcInfoDao.DATE_INFO.values()){ + @ApiResponse(code = 200, message = "OK", response = LocalDate.class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) + }) + public List listInfoKeys() { + final List keys = Lists.newArrayList(); + for (final JdbcInfoDao.DATE_INFO dateInfo : JdbcInfoDao.DATE_INFO.values()) { keys.add(dateInfo.name()); } return keys; } - - @RequestMapping(value = "/info/dropCache", produces = { "application/json" }, method = RequestMethod.GET) - @ApiOperation(value = "Drops the info cache", notes = "Drops the info cache", tags = { ExporterConstants.R }) - public void dropCache(){ + @RequestMapping(value = "/info/dropCache", produces = { + "application/json" + }, method = RequestMethod.GET) + @ApiOperation(value = "Drops the info cache", notes = "Drops the info cache", tags = { + ExporterConstants.R + }) + public void dropCache() { jdbcInfoDao.dropCache(); } - - } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/info/JdbcInfoDao.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/info/JdbcInfoDao.java index 1e3a6abc..0898d85e 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/info/JdbcInfoDao.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/info/JdbcInfoDao.java @@ -1,15 +1,6 @@ package eu.dnetlib.openaire.info; -import eu.dnetlib.openaire.project.dao.ValueCleaner; -import org.antlr.stringtemplate.StringTemplate; - -import java.io.IOException; -import java.io.OutputStream; -import java.sql.SQLException; import java.time.LocalDate; -import java.util.Date; -import java.util.Map; -import java.util.zip.ZipOutputStream; public interface JdbcInfoDao { diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/info/JdbcInfoDaoImpl.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/info/JdbcInfoDaoImpl.java index 63142558..344728d9 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/info/JdbcInfoDaoImpl.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/info/JdbcInfoDaoImpl.java @@ -1,20 +1,13 @@ package eu.dnetlib.openaire.info; -import com.google.common.base.Joiner; -import com.google.common.base.Splitter; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; -import eu.dnetlib.OpenaireExporterConfig; -import eu.dnetlib.openaire.project.dao.JdbcApiDao; -import eu.dnetlib.openaire.project.dao.ProjectTsvRepository; -import eu.dnetlib.openaire.project.dao.ValueCleaner; -import eu.dnetlib.openaire.project.domain.Project; -import eu.dnetlib.openaire.project.domain.db.ProjectDetails; -import eu.dnetlib.openaire.project.domain.db.ProjectTsv; -import org.antlr.stringtemplate.StringTemplate; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.time.DateFormatUtils; -import org.apache.commons.lang3.time.DateUtils; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.time.LocalDate; + +import javax.sql.DataSource; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -24,31 +17,12 @@ import org.springframework.cache.annotation.Cacheable; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; -import javax.sql.DataSource; -import java.io.BufferedOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.nio.charset.Charset; -import java.sql.*; -import java.time.Duration; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; -import java.time.format.FormatStyle; -import java.util.Arrays; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.zip.GZIPOutputStream; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; +import eu.dnetlib.DnetOpenaireExporterProperties; /** * Created by alessia on 29/04/2020 * - * Get and set info dates via JDBC. Dates are expected to be in a table named 'info' with two columns: - * key - see JdbcInfoDao.DATE_INFO enum + * Get and set info dates via JDBC. Dates are expected to be in a table named 'info' with two columns: key - see JdbcInfoDao.DATE_INFO enum * value - the date (LocalDate, no time) * */ @@ -59,7 +33,7 @@ public class JdbcInfoDaoImpl implements JdbcInfoDao { private static final Log log = LogFactory.getLog(JdbcInfoDaoImpl.class); @Autowired - private OpenaireExporterConfig config; + private DnetOpenaireExporterProperties config; @Autowired private DataSource dataSource; @@ -71,11 +45,11 @@ public class JdbcInfoDaoImpl implements JdbcInfoDao { LocalDate date = null; try (final Connection con = getConn(); final PreparedStatement stm = getStm(sql, con, dateInfo.name()); final ResultSet rs = getRs(stm)) { - log.info("loading info "+dateInfo+" Query: "+stm.toString()); - if(rs.next()) { + log.info("loading info " + dateInfo + " Query: " + stm.toString()); + if (rs.next()) { date = rs.getObject("value", LocalDate.class); } - } catch (SQLException e) { + } catch (final SQLException e) { throw new RuntimeException(e); } @@ -83,7 +57,9 @@ public class JdbcInfoDaoImpl implements JdbcInfoDao { } @Override - @CacheEvict(cacheNames = { "info" }, allEntries = true) + @CacheEvict(cacheNames = { + "info" + }, allEntries = true) @Scheduled(fixedDelayString = "${openaire.exporter.cache.ttl}") public void dropCache() { log.debug("dropped info cache"); diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/ProjectQueryParams.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/ProjectQueryParams.java index 16ff1d5a..376b0ade 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/ProjectQueryParams.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/ProjectQueryParams.java @@ -2,9 +2,6 @@ package eu.dnetlib.openaire.project; import java.util.regex.Pattern; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - public class ProjectQueryParams { private final Pattern patternFundingStream = Pattern.compile("(\\w*(::|%| )*)*"); @@ -20,8 +17,6 @@ public class ProjectQueryParams { private String endFrom = null; private String endUntil = null; - private static final Log log = LogFactory.getLog(ProjectQueryParams.class); // NOPMD by marko on 11/24/08 5:02 PM - public String getFundingProgramme() { return fundingProgramme; } @@ -71,15 +66,16 @@ public class ProjectQueryParams { } protected String verifyParam(final String p) { - if ((p != null) && !patternFundingStream.matcher(p).matches()) { + if (p != null && !patternFundingStream.matcher(p).matches()) { throw new IllegalArgumentException(String.format("Parameter '%s' contains an invalid character", p)); } return p; } protected String verifyDateParam(final String date) { - if ((date != null) && !patternDate.matcher(date).matches()) { - throw new IllegalArgumentException(String.format("Parameter date '%s' contains an invalid character. Accepted pattern is %s", date, patternDate.toString())); + if (date != null && !patternDate.matcher(date).matches()) { + throw new IllegalArgumentException( + String.format("Parameter date '%s' contains an invalid character. Accepted pattern is %s", date, patternDate.toString())); } return date; } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/ProjectsController.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/ProjectsController.java index 7c8070b3..00abbcf2 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/ProjectsController.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/ProjectsController.java @@ -13,8 +13,8 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import com.google.common.xml.XmlEscapers; -import eu.dnetlib.OpenaireExporterConfig; -import eu.dnetlib.OpenaireExporterConfig.Project; +import eu.dnetlib.DnetOpenaireExporterProperties; +import eu.dnetlib.DnetOpenaireExporterProperties.Project; import eu.dnetlib.openaire.common.AbstractExporterController; import eu.dnetlib.openaire.common.ExporterConstants; import eu.dnetlib.openaire.project.domain.db.ProjectTsv; @@ -49,7 +49,7 @@ public class ProjectsController extends AbstractExporterController { public final static String UTF8 = "UTF-8"; @Autowired - private OpenaireExporterConfig config; + private DnetOpenaireExporterProperties config; @Autowired private JdbcApiDao dao; diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/dao/JdbcApiDaoImpl.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/dao/JdbcApiDaoImpl.java index f2f2b258..49a0279f 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/dao/JdbcApiDaoImpl.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/dao/JdbcApiDaoImpl.java @@ -21,7 +21,7 @@ import com.google.common.base.Joiner; import com.google.common.base.Splitter; import com.google.common.collect.Maps; import com.google.common.collect.Sets; -import eu.dnetlib.OpenaireExporterConfig; +import eu.dnetlib.DnetOpenaireExporterProperties; import eu.dnetlib.openaire.project.domain.Project; import eu.dnetlib.openaire.project.domain.db.ProjectDetails; import eu.dnetlib.openaire.project.domain.db.ProjectTsv; @@ -48,7 +48,7 @@ public class JdbcApiDaoImpl implements JdbcApiDao { private static final Log log = LogFactory.getLog(JdbcApiDaoImpl.class); @Autowired - private OpenaireExporterConfig config; + private DnetOpenaireExporterProperties config; @Autowired private DataSource dataSource; diff --git a/apps/dnet-exporter-api/src/main/resources/dnet_dsm.data.sql b/apps/dnet-exporter-api/src/main/resources/dnet_dsm.data.sql new file mode 100644 index 00000000..ee3ec218 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/dnet_dsm.data.sql @@ -0,0 +1,176 @@ +INSERT INTO dsm_identities(pid, issuertype) VALUES ('77', 'roar'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('680', 'roar'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('260', 'roar'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('637', 'roar'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('528', 'roar'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('577', 'roar'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('566', 'roar'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('241', 'roar'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('491', 'roar'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('324', 'roar'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('375', 'roar'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('152', 'roar'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('1025', 'roar'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('10.13039/100004440', 'doi'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('10.13039/100000001', 'doi'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('10.13039/100000002', 'doi'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('10.13039/501100007601', 'doi'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('10.13039/501100001871', 'doi'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('10.13039/501100008982', 'doi'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('10.13039/501100000925', 'doi'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('10.13039/501100000923', 'doi'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('10.13039/501100001602', 'doi'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('10.13039/501100002428', 'doi'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('10.13039/501100000690', 'doi'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('10.13039/501100002341', 'doi'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 2242 8989', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0004 5900 900X', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 2169 9189', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 2181 2823', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0004 0427 7672', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 1958 7073', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 2169 1945', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 2154 0709', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 2176 1982', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 0789 9694', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 2228 3249', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 2176 7727', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0004 0611 9213', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0004 0609 4140', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0004 4663 8325', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 1092 7772', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0004 0452 5752', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 2297 5165', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 0672 3101', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 1957 0992', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 1091 8438', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 0507 0997', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 0665 7300', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 0685 2712', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0004 0647 6886', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 0943 9683', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 2096 9829', 'isni'); +INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 1957 9997', 'isni'); + +INSERT INTO funder_identity(funder, pid) VALUES ('H2020', '680'); +INSERT INTO funder_identity(funder, pid) VALUES ('H2020', '10.13039/501100007601'); +INSERT INTO funder_identity(funder, pid) VALUES ('H2020', '0000 0001 2242 8989'); +INSERT INTO funder_identity(funder, pid) VALUES ('FP7', '0000 0004 5900 900X'); +INSERT INTO funder_identity(funder, pid) VALUES ('FCT', '260'); +INSERT INTO funder_identity(funder, pid) VALUES ('FCT', '10.13039/501100001871'); +INSERT INTO funder_identity(funder, pid) VALUES ('FCT', '0000 0001 2169 9189'); +INSERT INTO funder_identity(funder, pid) VALUES ('FCT', '0000 0001 2181 2823'); +INSERT INTO funder_identity(funder, pid) VALUES ('WT', '637'); +INSERT INTO funder_identity(funder, pid) VALUES ('WT', '10.13039/100004440'); +INSERT INTO funder_identity(funder, pid) VALUES ('WT', '0000 0004 0427 7672'); +INSERT INTO funder_identity(funder, pid) VALUES ('NSF', '528'); +INSERT INTO funder_identity(funder, pid) VALUES ('NSF', '10.13039/501100008982'); +INSERT INTO funder_identity(funder, pid) VALUES ('NSF', '10.13039/100000001'); +INSERT INTO funder_identity(funder, pid) VALUES ('NSF', '0000 0001 1958 7073'); +INSERT INTO funder_identity(funder, pid) VALUES ('NSF', '0000 0001 2169 1945'); +INSERT INTO funder_identity(funder, pid) VALUES ('NSF', '0000 0001 2154 0709'); +INSERT INTO funder_identity(funder, pid) VALUES ('NHMRC', '577'); +INSERT INTO funder_identity(funder, pid) VALUES ('NHMRC', '10.13039/501100000925'); +INSERT INTO funder_identity(funder, pid) VALUES ('NHMRC', '0000 0001 2176 1982'); +INSERT INTO funder_identity(funder, pid) VALUES ('NHMRC', '0000 0001 0789 9694'); +INSERT INTO funder_identity(funder, pid) VALUES ('NHMRC', '0000 0001 2228 3249'); +INSERT INTO funder_identity(funder, pid) VALUES ('NHMRC', '0000 0001 2176 7727'); +INSERT INTO funder_identity(funder, pid) VALUES ('NHMRC', '0000 0001 0789 9694'); +INSERT INTO funder_identity(funder, pid) VALUES ('NHMRC', '0000 0001 2228 3249'); +INSERT INTO funder_identity(funder, pid) VALUES ('ARC', '566'); +INSERT INTO funder_identity(funder, pid) VALUES ('ARC', '10.13039/501100000923'); +INSERT INTO funder_identity(funder, pid) VALUES ('ARC', '0000 0004 0611 9213'); +INSERT INTO funder_identity(funder, pid) VALUES ('MSES', '0000 0004 0609 4140'); +INSERT INTO funder_identity(funder, pid) VALUES ('CSF', '0000 0004 4663 8325'); +INSERT INTO funder_identity(funder, pid) VALUES ('NWO', '241'); +INSERT INTO funder_identity(funder, pid) VALUES ('NWO', '0000 0001 1092 7772'); +INSERT INTO funder_identity(funder, pid) VALUES ('SFI', '10.13039/501100001602'); +INSERT INTO funder_identity(funder, pid) VALUES ('SFI', '0000 0004 0452 5752'); +INSERT INTO funder_identity(funder, pid) VALUES ('NIH', '10.13039/100000002'); +INSERT INTO funder_identity(funder, pid) VALUES ('NIH', '0000 0001 2297 5165'); +INSERT INTO funder_identity(funder, pid) VALUES ('NIH', '491'); +INSERT INTO funder_identity(funder, pid) VALUES ('SNSF', '324'); +INSERT INTO funder_identity(funder, pid) VALUES ('SNSF', '0000 0001 0672 3101'); +INSERT INTO funder_identity(funder, pid) VALUES ('SNSF', '0000 0001 1957 0992'); +INSERT INTO funder_identity(funder, pid) VALUES ('FWF', '77'); +INSERT INTO funder_identity(funder, pid) VALUES ('FWF', '10.13039/501100002428'); +INSERT INTO funder_identity(funder, pid) VALUES ('FWF', '0000 0001 1091 8438'); +INSERT INTO funder_identity(funder, pid) VALUES ('RCUK', '375'); +INSERT INTO funder_identity(funder, pid) VALUES ('RCUK', '10.13039/501100000690'); +INSERT INTO funder_identity(funder, pid) VALUES ('RCUK', '0000 0001 0507 0997'); +INSERT INTO funder_identity(funder, pid) VALUES ('TBT', '0000 0001 0665 7300'); +INSERT INTO funder_identity(funder, pid) VALUES ('TBT', '0000 0001 0685 2712'); +INSERT INTO funder_identity(funder, pid) VALUES ('AFF', '1025'); +INSERT INTO funder_identity(funder, pid) VALUES ('AFF', '10.13039/501100002341'); +INSERT INTO funder_identity(funder, pid) VALUES ('AFF', '0000 0004 0647 6886'); +INSERT INTO funder_identity(funder, pid) VALUES ('CONICYT', '0000 0001 0943 9683'); +INSERT INTO funder_identity(funder, pid) VALUES ('DFG', '152'); +INSERT INTO funder_identity(funder, pid) VALUES ('DFG', '0000 0001 2096 9829'); +INSERT INTO funder_identity(funder, pid) VALUES ('DFG', '0000 0001 1957 9997'); + + +INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) +VALUES ('H2020', 'European Commission - Horizon 2020', 'H2020', 'EU', 'http://ec.europa.eu/research/participants/data/ref/h2020/grants_manual/hi/oa_pilot/h2020-hi-oa-pilot-guide_en.pdf', 'OA mandate for publications; OA to research data by default', '2015-08-24'); + +INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) +VALUES ('FP7', 'European Commission - 7th Framework program', 'FP7', 'EU', 'http://ec.europa.eu/research/fp7', 'OA advised for publications; 2012 – 2013 (FP7)Special Clause 39 ERC on Open Access applies', '2013-05-07'); + +INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) +VALUES ('FCT', 'Portuguese Foundation for Science and Technology', 'FCT', 'PT', 'https://www.fct.pt/acessoaberto/index.phtml.en', 'OA mandate for publications; OA to research data advised', '2015-02-27'); + +INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) +VALUES ('WT', 'Wellcome Trust', 'WT', '', 'https://wellcome.ac.uk/funding/managing-grant/open-access-policy', 'OA mandate for articles, monographs and book chapters; CC-BY licence required', '2013-05-07'); + +INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) +VALUES ('NSF', 'National Science Foundation', 'NSF', 'US', 'https://www.nsf.gov/news/special_reports/public_access', 'Version of record or AAM of articles, conferences and data deposited into the repository; persistent identifier to full text on publisher’s website', '2016-03-04'); + +INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) +VALUES ('NHMRC', 'National Health and Medical Research Council', 'NHMRC', 'AU', 'https://www.nhmrc.gov.au/grants-funding/policy/nhmrc-open-access-policy', 'OA mandate for publications', '2015-08-24'); + +INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) +VALUES ('ARC', 'Australian Research Council', 'ARC', 'AU', 'http://www.arc.gov.au/arc-open-access-policy', 'OA mandate for research outputs, except for research data', '2015-08-24'); + +INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) +VALUES ('MESTD', 'Ministry of Education, Science and Technological Development', 'MESTD', 'RS', 'NOAD', 'NOAD', '2017-01-23'); + +INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) +VALUES ('MSES', 'Ministry of Science Education and Sport', 'MSES/MZOS', 'HR', 'NOAD', 'NOAD', '2015-09-14'); + +INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) +VALUES ('CSF', 'Croatian Science Foundation', 'CSF/HRZZ', 'HR', 'NOAD', 'NOAD', '2015-09-14'); + +INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) +VALUES ('NWO', 'Netherlands Organisation for Scientific Research', 'NWO', 'NL', 'NOAD', 'NOAD', '2016-06-23'); + +INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) +VALUES ('SFI', 'Science Foundation Ireland', 'SFI', 'IE', 'http://www.sfi.ie/resources/open-access-dec-10.pdf', 'OA mandate to publications, conference proceedings and technical reports', '2015-07-21'); + +INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) +VALUES ('NIH', 'National Institute of Health', 'NIH', 'US', 'https://publicaccess.nih.gov/policy.htm', 'OA to articles', '2017-09-21'); + +INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) +VALUES ('SNSF', 'Swiss National Science Foundation', 'SNSF', 'CH', 'NOAD', 'NOAD', '2016-11-16'); + +INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) +VALUES ('FWF', 'Austrian Science Fund', 'FWF', 'AT', 'NOAD', 'NOAD', '2017-09-21'); + +INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) +VALUES ('RCUK', 'Research Council UK', 'RCUK', 'UK', 'NOAD', 'NOAD', '2017-11-02'); + +INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) +VALUES ('TBT', 'Scientific and Technological Research Council of Turkey', 'Tubitak', 'TR', 'NOAD', 'NOAD', '2017-11-02'); + +INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) +VALUES ('DFG', 'German Research Foundation', 'DFG', 'DE', 'NOAD', 'NOAD', '2017-09-29'); + +INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) +VALUES ('TARA', 'Tara Expedition Foundation', 'TARA', 'FR', 'NOAD', 'NOAD', '2016-12-02'); + +INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) +VALUES ('AFF', 'Academy of Finland', 'AFF', 'FI', 'NOAD', 'NOAD', '2017-09-21'); + +INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) +VALUES ('CONICYT', 'Comisión Nacional de Investigación Científica y Tecnológica', 'CONICYT', 'CL', 'http://www.lareferencia.info', '', '2017-09-08'); + +INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) +VALUES ('SGOV', 'Ministry of Economy, Industry and Competitiveness', 'SGOV', 'ES', 'NOAD', 'NOAD', '2017-09-14'); diff --git a/apps/dnet-exporter-api/src/main/resources/dnet_dsm.sql b/apps/dnet-exporter-api/src/main/resources/dnet_dsm.sql new file mode 100644 index 00000000..b40ee542 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/dnet_dsm.sql @@ -0,0 +1,510 @@ + +CREATE TABLE dsm_api ( + id character varying(255) NOT NULL, + protocol character varying(255), + datasource character varying(255), + contentdescription character varying(255) DEFAULT 'metadata'::character varying, + active boolean DEFAULT false, + removable boolean DEFAULT false, + typology character varying(255) DEFAULT 'UNKNOWN'::character varying, + compatibility character varying(255) DEFAULT 'UNKNOWN'::character varying, + metadata_identifier_path character varying(512) DEFAULT NULL::character varying, + last_collection_total integer, + last_collection_date timestamp without time zone, + last_collection_mdid character varying(255) DEFAULT NULL::character varying, + last_aggregation_total integer, + last_aggregation_date timestamp without time zone, + last_aggregation_mdid character varying(255) DEFAULT NULL::character varying, + last_download_total integer, + last_download_date timestamp without time zone, + last_download_objid character varying(255) DEFAULT NULL::character varying, + last_validation_job character varying(255) DEFAULT NULL::character varying, + baseurl text, + _dnet_resource_identifier_ character varying(2048) DEFAULT ((('temp_'::text || md5((clock_timestamp())::text)) || '_'::text) || md5((random())::text)), + compatibility_override character varying(255) DEFAULT NULL::character varying +); + + +ALTER TABLE public.dsm_api OWNER TO dnetapi; + + +CREATE TABLE dsm_datasource_organization ( + datasource character varying(255) NOT NULL, + organization character varying(255) NOT NULL, + _dnet_resource_identifier_ character varying(2048) DEFAULT ((('temp_'::text || md5((clock_timestamp())::text)) || '_'::text) || md5((random())::text)) +); + + +ALTER TABLE public.dsm_datasource_organization OWNER TO dnetapi; + +-- +-- Name: dsm_datasources; Type: TABLE; Schema: public; Owner: dnetapi; Tablespace: +-- + +CREATE TABLE dsm_datasources ( + id character varying(255) NOT NULL, + officialname character varying(512) NOT NULL, + englishname character varying(512), + websiteurl character varying(255), + logourl character varying(255), + contactemail character varying(255), + latitude double precision DEFAULT 0.0, + longitude double precision DEFAULT 0.0, + timezone character varying(10) DEFAULT '0.0'::character varying, + namespaceprefix character(12) NOT NULL, + languages text, + od_contenttypes text, + collectedfrom character varying(255), + dateofvalidation date, + optional1 character varying(255), + optional2 character varying(255), + typology character varying(255) NOT NULL, + provenanceaction character varying(255) DEFAULT 'UNKNOWN'::character varying, + dateofcollection date DEFAULT ('now'::text)::date NOT NULL, + platform character varying(255), + activationid character varying(255), + description text, + releasestartdate date, + releaseenddate date, + missionstatementurl character varying(512), + dataprovider boolean, + serviceprovider boolean, + databaseaccesstype character varying(64), + datauploadtype character varying(64), + databaseaccessrestriction character varying(64), + datauploadrestriction character varying(64), + versioning boolean, + citationguidelineurl character varying(512), + qualitymanagementkind character varying(64), + pidsystems text, + certificates text, + aggregator character varying(64) DEFAULT 'OPENAIRE'::character varying NOT NULL, + issn character varying(20), + eissn character varying(20), + lissn character varying(20), + registeredby character varying(255), + subjects text, + managed boolean DEFAULT false, + registrationdate date, + consentTermsOfUse boolean default null, + fullTextDownload boolean default null, + _dnet_resource_identifier_ character varying(2048) DEFAULT ((('temp_'::text || md5((clock_timestamp())::text)) || '_'::text) || md5((random())::text)) +); + + +ALTER TABLE public.dsm_datasources OWNER TO dnetapi; + +-- +-- Name: dsm_organizations; Type: TABLE; Schema: public; Owner: dnetapi; Tablespace: +-- + +CREATE TABLE dsm_organizations ( + id character varying(255) NOT NULL, + legalshortname character varying(255), + legalname character varying(255), + websiteurl text, + logourl character varying(255), + ec_legalbody boolean DEFAULT false, + ec_legalperson boolean DEFAULT false, + ec_nonprofit boolean DEFAULT false, + ec_researchorganization boolean DEFAULT false, + ec_highereducation boolean DEFAULT false, + ec_internationalorganizationeurinterests boolean DEFAULT false, + ec_internationalorganization boolean DEFAULT false, + ec_enterprise boolean DEFAULT false, + ec_smevalidated boolean DEFAULT false, + ec_nutscode boolean DEFAULT false, + country character varying(255), + collectedfrom character varying(255), + optional1 character varying(255), + optional2 character varying(255), + dateofcollection date DEFAULT ('now'::text)::date NOT NULL, + provenanceaction character varying(255) DEFAULT 'UNKNOWN'::character varying, + _dnet_resource_identifier_ character varying(2048) DEFAULT ((('temp_'::text || md5((clock_timestamp())::text)) || '_'::text) || md5((random())::text)), + lastupdate date DEFAULT ('now'::text)::date NOT NULL, + trust double precision DEFAULT 0.9 +); + + +ALTER TABLE public.dsm_organizations OWNER TO dnetapi; + +-- +-- Name: browse_countries; Type: VIEW; Schema: public; Owner: dnet +-- + +CREATE VIEW browse_countries AS + SELECT o.country AS term, + count(*) AS total + FROM (((dsm_api a + LEFT JOIN dsm_datasources d ON (((a.datasource)::text = (d.id)::text))) + LEFT JOIN dsm_datasource_organization dao ON (((d.id)::text = (dao.datasource)::text))) + LEFT JOIN dsm_organizations o ON (((dao.organization)::text = (o.id)::text))) + GROUP BY o.country + ORDER BY count(*) DESC; + + +ALTER TABLE public.browse_countries OWNER TO dnet; + + + +CREATE TABLE dsm_apiparams ( + param character varying(255) NOT NULL, + value text DEFAULT ''::character varying NOT NULL, + api character varying(255) NOT NULL, + _dnet_resource_identifier_ character varying(2048) DEFAULT ((('temp_'::text || md5((clock_timestamp())::text)) || '_'::text) || md5((random())::text)) +); + + +ALTER TABLE public.dsm_apiparams OWNER TO dnetapi; + +-- +-- Name: dsm_datasource_api; Type: VIEW; Schema: public; Owner: dnet +-- + +CREATE OR REPLACE VIEW dsm_datasource_api AS + SELECT + row_number() OVER (ORDER BY a.id) AS rowid, + d.id, + d.officialname, + d.englishname, + d.websiteurl, + d.contactemail, + d.collectedfrom, + d.typology, + d.platform, + d.registeredby, + d.managed, + a.protocol, + a.contentdescription, + a.active, + a.removable, + a.typology AS apitypology, + a.compatibility, + a.baseurl + FROM (dsm_datasources d + LEFT JOIN dsm_api a ON (((d.id)::text = (a.datasource)::text))); + +-- +-- Name: dsm_datasourcepids; Type: TABLE; Schema: public; Owner: dnetapi; Tablespace: +-- + +CREATE TABLE dsm_datasourcepids ( + datasource character varying(255) NOT NULL, + pid character varying(255) NOT NULL, + _dnet_resource_identifier_ character varying(2048) DEFAULT ((('temp_'::text || md5((clock_timestamp())::text)) || '_'::text) || md5((random())::text)) +); + + +ALTER TABLE public.dsm_datasourcepids OWNER TO dnetapi; + +-- +-- Name: dsm_identities; Type: TABLE; Schema: public; Owner: dnetapi; Tablespace: +-- + +CREATE TABLE dsm_identities ( + pid character varying(255) NOT NULL, + issuertype character varying(255), + _dnet_resource_identifier_ character varying(2048) DEFAULT ((('temp_'::text || md5((clock_timestamp())::text)) || '_'::text) || md5((random())::text)) +); + + +ALTER TABLE public.dsm_identities OWNER TO dnetapi; + +-- +-- Name: dsm_organizationpids; Type: TABLE; Schema: public; Owner: dnetapi; Tablespace: +-- + +CREATE TABLE dsm_organizationpids ( + organization character varying(255) NOT NULL, + pid character varying(255) NOT NULL, + _dnet_resource_identifier_ character varying(2048) DEFAULT ((('temp_'::text || md5((clock_timestamp())::text)) || '_'::text) || md5((random())::text)) +); + + +ALTER TABLE public.dsm_organizationpids OWNER TO dnetapi; + +-- +-- Name: funder_identity; Type: TABLE; Schema: public; Owner: dnet; Tablespace: +-- + +CREATE TABLE funder_identity ( + funder character varying(255), + pid character varying(255) +); + + +ALTER TABLE public.funder_identity OWNER TO dnet; + +-- +-- Name: funders; Type: TABLE; Schema: public; Owner: dnet; Tablespace: +-- + +CREATE TABLE funders ( + id character varying(255) NOT NULL, + name character varying(255), + shortname character varying(255), + jurisdiction character varying(255), + websiteurl text, + policy character varying(255), + registrationdate date DEFAULT ('now'::text)::date NOT NULL, + lastupdatedate date +); + + +ALTER TABLE public.funders OWNER TO dnet; + +-- +-- Name: fundingpaths; Type: TABLE; Schema: public; Owner: dnet; Tablespace: +-- + + + + + + + + + + + + + + +-- +-- Name: project_organization; Type: TABLE; Schema: public; Owner: dnet; Tablespace: +-- + +--CREATE TABLE project_organization ( +-- participantnumber integer, +-- project character varying(255) NOT NULL, +-- resporganization character varying(255) NOT NULL, +-- semanticclass character varying(255) DEFAULT 'UNKNOWN'::character varying, +-- trust double precision DEFAULT 0.9, +-- _dnet_resource_identifier_ character varying(2048) DEFAULT ((('temp_'::text || md5((clock_timestamp())::text)) || '_'::text) || md5((random())::text)) +--); + + +--ALTER TABLE public.project_organization OWNER TO dnet; + + + + +-- +-- Name: dsm_api_pkey; Type: CONSTRAINT; Schema: public; Owner: dnetapi; Tablespace: +-- + +ALTER TABLE ONLY dsm_api + ADD CONSTRAINT dsm_api_pkey PRIMARY KEY (id); + + +-- +-- Name: dsm_apicollection_pkey; Type: CONSTRAINT; Schema: public; Owner: dnetapi; Tablespace: +-- + +ALTER TABLE ONLY dsm_apiparams + ADD CONSTRAINT dsm_apicollection_pkey PRIMARY KEY (api, param); + + +-- +-- Name: dsm_datasourcepids_pkey; Type: CONSTRAINT; Schema: public; Owner: dnetapi; Tablespace: +-- + +ALTER TABLE ONLY dsm_datasourcepids + ADD CONSTRAINT dsm_datasourcepids_pkey PRIMARY KEY (datasource, pid); + + +-- +-- Name: dsm_datasources_namespaceprefix_key; Type: CONSTRAINT; Schema: public; Owner: dnetapi; Tablespace: +-- + +ALTER TABLE ONLY dsm_datasources + ADD CONSTRAINT dsm_datasources_namespaceprefix_key UNIQUE (namespaceprefix); + + +-- +-- Name: dsm_datasources_pkey; Type: CONSTRAINT; Schema: public; Owner: dnetapi; Tablespace: +-- + +ALTER TABLE ONLY dsm_datasources + ADD CONSTRAINT dsm_datasources_pkey PRIMARY KEY (id); + + +-- +-- Name: dsm_identities_pkey; Type: CONSTRAINT; Schema: public; Owner: dnetapi; Tablespace: +-- + +ALTER TABLE ONLY dsm_identities + ADD CONSTRAINT dsm_identities_pkey PRIMARY KEY (pid); + + +-- +-- Name: dsm_organization_datasource_pkey; Type: CONSTRAINT; Schema: public; Owner: dnetapi; Tablespace: +-- + +ALTER TABLE ONLY dsm_datasource_organization + ADD CONSTRAINT dsm_organization_datasource_pkey PRIMARY KEY (datasource, organization); + + +-- +-- Name: dsm_organizationpids_pkey; Type: CONSTRAINT; Schema: public; Owner: dnetapi; Tablespace: +-- + +ALTER TABLE ONLY dsm_organizationpids + ADD CONSTRAINT dsm_organizationpids_pkey PRIMARY KEY (organization, pid); + + +-- +-- Name: dsm_organizations_pkey; Type: CONSTRAINT; Schema: public; Owner: dnetapi; Tablespace: +-- + +ALTER TABLE ONLY dsm_organizations + ADD CONSTRAINT dsm_organizations_pkey PRIMARY KEY (id); + + + + + + +-- +-- Name: dsm_datasources_contactemail_idx; Type: INDEX; Schema: public; Owner: dnetapi; Tablespace: +-- + +CREATE INDEX dsm_datasources_contactemail_idx ON dsm_datasources USING btree (contactemail); + + +-- +-- Name: dsm_datasources_englishname_idx; Type: INDEX; Schema: public; Owner: dnetapi; Tablespace: +-- + +CREATE INDEX dsm_datasources_englishname_idx ON dsm_datasources USING btree (englishname); + + +-- +-- Name: dsm_datasources_managed_idx; Type: INDEX; Schema: public; Owner: dnetapi; Tablespace: +-- + +CREATE INDEX dsm_datasources_managed_idx ON dsm_datasources USING btree (managed); + + +-- +-- Name: dsm_datasources_officialname_idx; Type: INDEX; Schema: public; Owner: dnetapi; Tablespace: +-- + +CREATE INDEX dsm_datasources_officialname_idx ON dsm_datasources USING btree (officialname); + + +-- +-- Name: dsm_datasources_registeredby_idx; Type: INDEX; Schema: public; Owner: dnetapi; Tablespace: +-- + +CREATE INDEX dsm_datasources_registeredby_idx ON dsm_datasources USING btree (registeredby); + + +-- +-- Name: dsm_organizations_country_idx; Type: INDEX; Schema: public; Owner: dnetapi; Tablespace: +-- + +CREATE INDEX dsm_organizations_country_idx ON dsm_organizations USING btree (country); + + + + + + + + +ALTER TABLE ONLY dsm_api + ADD CONSTRAINT dsm_api_datasource_fkey FOREIGN KEY (datasource) REFERENCES dsm_datasources(id) ON DELETE CASCADE; + + +-- +-- Name: dsm_apicollections_api_fkey; Type: FK CONSTRAINT; Schema: public; Owner: dnetapi +-- + +ALTER TABLE ONLY dsm_apiparams + ADD CONSTRAINT dsm_apicollections_api_fkey FOREIGN KEY (api) REFERENCES dsm_api(id); + + +-- +-- Name: dsm_datasource_organization_datasource_fkey; Type: FK CONSTRAINT; Schema: public; Owner: dnetapi +-- + +ALTER TABLE ONLY dsm_datasource_organization + ADD CONSTRAINT dsm_datasource_organization_datasource_fkey FOREIGN KEY (datasource) REFERENCES dsm_datasources(id); + + +-- +-- Name: dsm_datasource_organization_organization_fkey; Type: FK CONSTRAINT; Schema: public; Owner: dnetapi +-- + +ALTER TABLE ONLY dsm_datasource_organization + ADD CONSTRAINT dsm_datasource_organization_organization_fkey FOREIGN KEY (organization) REFERENCES dsm_organizations(id) ON DELETE CASCADE; + + +-- +-- Name: dsm_datasourcepids_datasource_fkey; Type: FK CONSTRAINT; Schema: public; Owner: dnetapi +-- + +ALTER TABLE ONLY dsm_datasourcepids + ADD CONSTRAINT dsm_datasourcepids_datasource_fkey FOREIGN KEY (datasource) REFERENCES dsm_datasources(id); + + +-- +-- Name: dsm_datasourcepids_pid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: dnetapi +-- + +ALTER TABLE ONLY dsm_datasourcepids + ADD CONSTRAINT dsm_datasourcepids_pid_fkey FOREIGN KEY (pid) REFERENCES dsm_identities(pid); + + +-- +-- Name: dsm_datasources_collectedfrom_fkey; Type: FK CONSTRAINT; Schema: public; Owner: dnetapi +-- + +ALTER TABLE ONLY dsm_datasources + ADD CONSTRAINT dsm_datasources_collectedfrom_fkey FOREIGN KEY (collectedfrom) REFERENCES dsm_datasources(id); + + +-- +-- Name: dsm_organizationpids_organization_fkey; Type: FK CONSTRAINT; Schema: public; Owner: dnetapi +-- + +ALTER TABLE ONLY dsm_organizationpids + ADD CONSTRAINT dsm_organizationpids_organization_fkey FOREIGN KEY (organization) REFERENCES dsm_organizations(id); + + +-- +-- Name: dsm_organizationpids_pid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: dnetapi +-- + +ALTER TABLE ONLY dsm_organizationpids + ADD CONSTRAINT dsm_organizationpids_pid_fkey FOREIGN KEY (pid) REFERENCES dsm_identities(pid); + + +-- +-- Name: dsm_organizations_collectedfrom_fkey; Type: FK CONSTRAINT; Schema: public; Owner: dnetapi +-- + +ALTER TABLE ONLY dsm_organizations + ADD CONSTRAINT dsm_organizations_collectedfrom_fkey FOREIGN KEY (collectedfrom) REFERENCES dsm_datasources(id); + + + +REVOKE ALL ON SCHEMA public FROM PUBLIC; +REVOKE ALL ON SCHEMA public FROM postgres; +GRANT ALL ON SCHEMA public TO postgres; +GRANT ALL ON SCHEMA public TO PUBLIC; + + +REVOKE ALL ON TABLE browse_countries FROM PUBLIC; +REVOKE ALL ON TABLE browse_countries FROM dnet; +GRANT ALL ON TABLE browse_countries TO dnet; +GRANT ALL ON TABLE browse_countries TO dnetapi; + + +REVOKE ALL ON TABLE dsm_datasource_api FROM PUBLIC; +REVOKE ALL ON TABLE dsm_datasource_api FROM dnet; +GRANT ALL ON TABLE dsm_datasource_api TO dnet; +GRANT SELECT ON TABLE dsm_datasource_api TO dnetapi; + + diff --git a/apps/dnet-exporter-api/src/main/resources/dnet_info.sql b/apps/dnet-exporter-api/src/main/resources/dnet_info.sql new file mode 100644 index 00000000..23ddb680 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/dnet_info.sql @@ -0,0 +1,21 @@ + +CREATE TABLE info ( + key character varying(255) NOT NULL PRIMARY KEY, + value DATE +); + +GRANT ALL ON TABLE info TO dnet; +GRANT ALL ON TABLE info TO dnetapi; + +INSERT INTO info(key) VALUES ('oaf_load_date'); +INSERT INTO info(key) VALUES ('odf_load_date'); +INSERT INTO info(key) VALUES ('inference_date'); +INSERT INTO info(key) VALUES ('claim_load_date'); +INSERT INTO info(key) VALUES ('stats_update_date'); + +INSERT INTO info(key) VALUES ('crossref_update_date'); +INSERT INTO info(key) VALUES ('unpaywall_update_date'); +INSERT INTO info(key) VALUES ('orcid_update_date'); +INSERT INTO info(key) VALUES ('mag_update_date'); + + -- 2.17.1 From 33bd9c396feaed9171b5ff122b34f02198abc287 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Mon, 7 Feb 2022 10:33:09 +0100 Subject: [PATCH 04/19] move vs dnet_services --- .../src/main/resources/dnet_dsm.data.sql | 176 ------ .../src/main/resources/dnet_dsm.sql | 510 ------------------ .../src/main/resources/dnet_info.sql | 21 - .../sql/recent_registered_datasources.sql.st | 6 +- ...ent_registered_datasources_fromDate.st.sql | 6 +- ...tered_datasources_fromDate_typology.st.sql | 6 +- 6 files changed, 9 insertions(+), 716 deletions(-) delete mode 100644 apps/dnet-exporter-api/src/main/resources/dnet_dsm.data.sql delete mode 100644 apps/dnet-exporter-api/src/main/resources/dnet_dsm.sql delete mode 100644 apps/dnet-exporter-api/src/main/resources/dnet_info.sql diff --git a/apps/dnet-exporter-api/src/main/resources/dnet_dsm.data.sql b/apps/dnet-exporter-api/src/main/resources/dnet_dsm.data.sql deleted file mode 100644 index ee3ec218..00000000 --- a/apps/dnet-exporter-api/src/main/resources/dnet_dsm.data.sql +++ /dev/null @@ -1,176 +0,0 @@ -INSERT INTO dsm_identities(pid, issuertype) VALUES ('77', 'roar'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('680', 'roar'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('260', 'roar'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('637', 'roar'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('528', 'roar'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('577', 'roar'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('566', 'roar'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('241', 'roar'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('491', 'roar'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('324', 'roar'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('375', 'roar'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('152', 'roar'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('1025', 'roar'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('10.13039/100004440', 'doi'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('10.13039/100000001', 'doi'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('10.13039/100000002', 'doi'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('10.13039/501100007601', 'doi'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('10.13039/501100001871', 'doi'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('10.13039/501100008982', 'doi'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('10.13039/501100000925', 'doi'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('10.13039/501100000923', 'doi'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('10.13039/501100001602', 'doi'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('10.13039/501100002428', 'doi'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('10.13039/501100000690', 'doi'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('10.13039/501100002341', 'doi'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 2242 8989', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0004 5900 900X', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 2169 9189', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 2181 2823', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0004 0427 7672', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 1958 7073', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 2169 1945', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 2154 0709', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 2176 1982', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 0789 9694', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 2228 3249', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 2176 7727', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0004 0611 9213', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0004 0609 4140', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0004 4663 8325', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 1092 7772', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0004 0452 5752', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 2297 5165', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 0672 3101', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 1957 0992', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 1091 8438', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 0507 0997', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 0665 7300', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 0685 2712', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0004 0647 6886', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 0943 9683', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 2096 9829', 'isni'); -INSERT INTO dsm_identities(pid, issuertype) VALUES ('0000 0001 1957 9997', 'isni'); - -INSERT INTO funder_identity(funder, pid) VALUES ('H2020', '680'); -INSERT INTO funder_identity(funder, pid) VALUES ('H2020', '10.13039/501100007601'); -INSERT INTO funder_identity(funder, pid) VALUES ('H2020', '0000 0001 2242 8989'); -INSERT INTO funder_identity(funder, pid) VALUES ('FP7', '0000 0004 5900 900X'); -INSERT INTO funder_identity(funder, pid) VALUES ('FCT', '260'); -INSERT INTO funder_identity(funder, pid) VALUES ('FCT', '10.13039/501100001871'); -INSERT INTO funder_identity(funder, pid) VALUES ('FCT', '0000 0001 2169 9189'); -INSERT INTO funder_identity(funder, pid) VALUES ('FCT', '0000 0001 2181 2823'); -INSERT INTO funder_identity(funder, pid) VALUES ('WT', '637'); -INSERT INTO funder_identity(funder, pid) VALUES ('WT', '10.13039/100004440'); -INSERT INTO funder_identity(funder, pid) VALUES ('WT', '0000 0004 0427 7672'); -INSERT INTO funder_identity(funder, pid) VALUES ('NSF', '528'); -INSERT INTO funder_identity(funder, pid) VALUES ('NSF', '10.13039/501100008982'); -INSERT INTO funder_identity(funder, pid) VALUES ('NSF', '10.13039/100000001'); -INSERT INTO funder_identity(funder, pid) VALUES ('NSF', '0000 0001 1958 7073'); -INSERT INTO funder_identity(funder, pid) VALUES ('NSF', '0000 0001 2169 1945'); -INSERT INTO funder_identity(funder, pid) VALUES ('NSF', '0000 0001 2154 0709'); -INSERT INTO funder_identity(funder, pid) VALUES ('NHMRC', '577'); -INSERT INTO funder_identity(funder, pid) VALUES ('NHMRC', '10.13039/501100000925'); -INSERT INTO funder_identity(funder, pid) VALUES ('NHMRC', '0000 0001 2176 1982'); -INSERT INTO funder_identity(funder, pid) VALUES ('NHMRC', '0000 0001 0789 9694'); -INSERT INTO funder_identity(funder, pid) VALUES ('NHMRC', '0000 0001 2228 3249'); -INSERT INTO funder_identity(funder, pid) VALUES ('NHMRC', '0000 0001 2176 7727'); -INSERT INTO funder_identity(funder, pid) VALUES ('NHMRC', '0000 0001 0789 9694'); -INSERT INTO funder_identity(funder, pid) VALUES ('NHMRC', '0000 0001 2228 3249'); -INSERT INTO funder_identity(funder, pid) VALUES ('ARC', '566'); -INSERT INTO funder_identity(funder, pid) VALUES ('ARC', '10.13039/501100000923'); -INSERT INTO funder_identity(funder, pid) VALUES ('ARC', '0000 0004 0611 9213'); -INSERT INTO funder_identity(funder, pid) VALUES ('MSES', '0000 0004 0609 4140'); -INSERT INTO funder_identity(funder, pid) VALUES ('CSF', '0000 0004 4663 8325'); -INSERT INTO funder_identity(funder, pid) VALUES ('NWO', '241'); -INSERT INTO funder_identity(funder, pid) VALUES ('NWO', '0000 0001 1092 7772'); -INSERT INTO funder_identity(funder, pid) VALUES ('SFI', '10.13039/501100001602'); -INSERT INTO funder_identity(funder, pid) VALUES ('SFI', '0000 0004 0452 5752'); -INSERT INTO funder_identity(funder, pid) VALUES ('NIH', '10.13039/100000002'); -INSERT INTO funder_identity(funder, pid) VALUES ('NIH', '0000 0001 2297 5165'); -INSERT INTO funder_identity(funder, pid) VALUES ('NIH', '491'); -INSERT INTO funder_identity(funder, pid) VALUES ('SNSF', '324'); -INSERT INTO funder_identity(funder, pid) VALUES ('SNSF', '0000 0001 0672 3101'); -INSERT INTO funder_identity(funder, pid) VALUES ('SNSF', '0000 0001 1957 0992'); -INSERT INTO funder_identity(funder, pid) VALUES ('FWF', '77'); -INSERT INTO funder_identity(funder, pid) VALUES ('FWF', '10.13039/501100002428'); -INSERT INTO funder_identity(funder, pid) VALUES ('FWF', '0000 0001 1091 8438'); -INSERT INTO funder_identity(funder, pid) VALUES ('RCUK', '375'); -INSERT INTO funder_identity(funder, pid) VALUES ('RCUK', '10.13039/501100000690'); -INSERT INTO funder_identity(funder, pid) VALUES ('RCUK', '0000 0001 0507 0997'); -INSERT INTO funder_identity(funder, pid) VALUES ('TBT', '0000 0001 0665 7300'); -INSERT INTO funder_identity(funder, pid) VALUES ('TBT', '0000 0001 0685 2712'); -INSERT INTO funder_identity(funder, pid) VALUES ('AFF', '1025'); -INSERT INTO funder_identity(funder, pid) VALUES ('AFF', '10.13039/501100002341'); -INSERT INTO funder_identity(funder, pid) VALUES ('AFF', '0000 0004 0647 6886'); -INSERT INTO funder_identity(funder, pid) VALUES ('CONICYT', '0000 0001 0943 9683'); -INSERT INTO funder_identity(funder, pid) VALUES ('DFG', '152'); -INSERT INTO funder_identity(funder, pid) VALUES ('DFG', '0000 0001 2096 9829'); -INSERT INTO funder_identity(funder, pid) VALUES ('DFG', '0000 0001 1957 9997'); - - -INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) -VALUES ('H2020', 'European Commission - Horizon 2020', 'H2020', 'EU', 'http://ec.europa.eu/research/participants/data/ref/h2020/grants_manual/hi/oa_pilot/h2020-hi-oa-pilot-guide_en.pdf', 'OA mandate for publications; OA to research data by default', '2015-08-24'); - -INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) -VALUES ('FP7', 'European Commission - 7th Framework program', 'FP7', 'EU', 'http://ec.europa.eu/research/fp7', 'OA advised for publications; 2012 – 2013 (FP7)Special Clause 39 ERC on Open Access applies', '2013-05-07'); - -INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) -VALUES ('FCT', 'Portuguese Foundation for Science and Technology', 'FCT', 'PT', 'https://www.fct.pt/acessoaberto/index.phtml.en', 'OA mandate for publications; OA to research data advised', '2015-02-27'); - -INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) -VALUES ('WT', 'Wellcome Trust', 'WT', '', 'https://wellcome.ac.uk/funding/managing-grant/open-access-policy', 'OA mandate for articles, monographs and book chapters; CC-BY licence required', '2013-05-07'); - -INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) -VALUES ('NSF', 'National Science Foundation', 'NSF', 'US', 'https://www.nsf.gov/news/special_reports/public_access', 'Version of record or AAM of articles, conferences and data deposited into the repository; persistent identifier to full text on publisher’s website', '2016-03-04'); - -INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) -VALUES ('NHMRC', 'National Health and Medical Research Council', 'NHMRC', 'AU', 'https://www.nhmrc.gov.au/grants-funding/policy/nhmrc-open-access-policy', 'OA mandate for publications', '2015-08-24'); - -INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) -VALUES ('ARC', 'Australian Research Council', 'ARC', 'AU', 'http://www.arc.gov.au/arc-open-access-policy', 'OA mandate for research outputs, except for research data', '2015-08-24'); - -INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) -VALUES ('MESTD', 'Ministry of Education, Science and Technological Development', 'MESTD', 'RS', 'NOAD', 'NOAD', '2017-01-23'); - -INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) -VALUES ('MSES', 'Ministry of Science Education and Sport', 'MSES/MZOS', 'HR', 'NOAD', 'NOAD', '2015-09-14'); - -INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) -VALUES ('CSF', 'Croatian Science Foundation', 'CSF/HRZZ', 'HR', 'NOAD', 'NOAD', '2015-09-14'); - -INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) -VALUES ('NWO', 'Netherlands Organisation for Scientific Research', 'NWO', 'NL', 'NOAD', 'NOAD', '2016-06-23'); - -INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) -VALUES ('SFI', 'Science Foundation Ireland', 'SFI', 'IE', 'http://www.sfi.ie/resources/open-access-dec-10.pdf', 'OA mandate to publications, conference proceedings and technical reports', '2015-07-21'); - -INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) -VALUES ('NIH', 'National Institute of Health', 'NIH', 'US', 'https://publicaccess.nih.gov/policy.htm', 'OA to articles', '2017-09-21'); - -INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) -VALUES ('SNSF', 'Swiss National Science Foundation', 'SNSF', 'CH', 'NOAD', 'NOAD', '2016-11-16'); - -INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) -VALUES ('FWF', 'Austrian Science Fund', 'FWF', 'AT', 'NOAD', 'NOAD', '2017-09-21'); - -INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) -VALUES ('RCUK', 'Research Council UK', 'RCUK', 'UK', 'NOAD', 'NOAD', '2017-11-02'); - -INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) -VALUES ('TBT', 'Scientific and Technological Research Council of Turkey', 'Tubitak', 'TR', 'NOAD', 'NOAD', '2017-11-02'); - -INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) -VALUES ('DFG', 'German Research Foundation', 'DFG', 'DE', 'NOAD', 'NOAD', '2017-09-29'); - -INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) -VALUES ('TARA', 'Tara Expedition Foundation', 'TARA', 'FR', 'NOAD', 'NOAD', '2016-12-02'); - -INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) -VALUES ('AFF', 'Academy of Finland', 'AFF', 'FI', 'NOAD', 'NOAD', '2017-09-21'); - -INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) -VALUES ('CONICYT', 'Comisión Nacional de Investigación Científica y Tecnológica', 'CONICYT', 'CL', 'http://www.lareferencia.info', '', '2017-09-08'); - -INSERT INTO funders(id, name, shortname, jurisdiction, websiteurl, policy, registrationdate) -VALUES ('SGOV', 'Ministry of Economy, Industry and Competitiveness', 'SGOV', 'ES', 'NOAD', 'NOAD', '2017-09-14'); diff --git a/apps/dnet-exporter-api/src/main/resources/dnet_dsm.sql b/apps/dnet-exporter-api/src/main/resources/dnet_dsm.sql deleted file mode 100644 index b40ee542..00000000 --- a/apps/dnet-exporter-api/src/main/resources/dnet_dsm.sql +++ /dev/null @@ -1,510 +0,0 @@ - -CREATE TABLE dsm_api ( - id character varying(255) NOT NULL, - protocol character varying(255), - datasource character varying(255), - contentdescription character varying(255) DEFAULT 'metadata'::character varying, - active boolean DEFAULT false, - removable boolean DEFAULT false, - typology character varying(255) DEFAULT 'UNKNOWN'::character varying, - compatibility character varying(255) DEFAULT 'UNKNOWN'::character varying, - metadata_identifier_path character varying(512) DEFAULT NULL::character varying, - last_collection_total integer, - last_collection_date timestamp without time zone, - last_collection_mdid character varying(255) DEFAULT NULL::character varying, - last_aggregation_total integer, - last_aggregation_date timestamp without time zone, - last_aggregation_mdid character varying(255) DEFAULT NULL::character varying, - last_download_total integer, - last_download_date timestamp without time zone, - last_download_objid character varying(255) DEFAULT NULL::character varying, - last_validation_job character varying(255) DEFAULT NULL::character varying, - baseurl text, - _dnet_resource_identifier_ character varying(2048) DEFAULT ((('temp_'::text || md5((clock_timestamp())::text)) || '_'::text) || md5((random())::text)), - compatibility_override character varying(255) DEFAULT NULL::character varying -); - - -ALTER TABLE public.dsm_api OWNER TO dnetapi; - - -CREATE TABLE dsm_datasource_organization ( - datasource character varying(255) NOT NULL, - organization character varying(255) NOT NULL, - _dnet_resource_identifier_ character varying(2048) DEFAULT ((('temp_'::text || md5((clock_timestamp())::text)) || '_'::text) || md5((random())::text)) -); - - -ALTER TABLE public.dsm_datasource_organization OWNER TO dnetapi; - --- --- Name: dsm_datasources; Type: TABLE; Schema: public; Owner: dnetapi; Tablespace: --- - -CREATE TABLE dsm_datasources ( - id character varying(255) NOT NULL, - officialname character varying(512) NOT NULL, - englishname character varying(512), - websiteurl character varying(255), - logourl character varying(255), - contactemail character varying(255), - latitude double precision DEFAULT 0.0, - longitude double precision DEFAULT 0.0, - timezone character varying(10) DEFAULT '0.0'::character varying, - namespaceprefix character(12) NOT NULL, - languages text, - od_contenttypes text, - collectedfrom character varying(255), - dateofvalidation date, - optional1 character varying(255), - optional2 character varying(255), - typology character varying(255) NOT NULL, - provenanceaction character varying(255) DEFAULT 'UNKNOWN'::character varying, - dateofcollection date DEFAULT ('now'::text)::date NOT NULL, - platform character varying(255), - activationid character varying(255), - description text, - releasestartdate date, - releaseenddate date, - missionstatementurl character varying(512), - dataprovider boolean, - serviceprovider boolean, - databaseaccesstype character varying(64), - datauploadtype character varying(64), - databaseaccessrestriction character varying(64), - datauploadrestriction character varying(64), - versioning boolean, - citationguidelineurl character varying(512), - qualitymanagementkind character varying(64), - pidsystems text, - certificates text, - aggregator character varying(64) DEFAULT 'OPENAIRE'::character varying NOT NULL, - issn character varying(20), - eissn character varying(20), - lissn character varying(20), - registeredby character varying(255), - subjects text, - managed boolean DEFAULT false, - registrationdate date, - consentTermsOfUse boolean default null, - fullTextDownload boolean default null, - _dnet_resource_identifier_ character varying(2048) DEFAULT ((('temp_'::text || md5((clock_timestamp())::text)) || '_'::text) || md5((random())::text)) -); - - -ALTER TABLE public.dsm_datasources OWNER TO dnetapi; - --- --- Name: dsm_organizations; Type: TABLE; Schema: public; Owner: dnetapi; Tablespace: --- - -CREATE TABLE dsm_organizations ( - id character varying(255) NOT NULL, - legalshortname character varying(255), - legalname character varying(255), - websiteurl text, - logourl character varying(255), - ec_legalbody boolean DEFAULT false, - ec_legalperson boolean DEFAULT false, - ec_nonprofit boolean DEFAULT false, - ec_researchorganization boolean DEFAULT false, - ec_highereducation boolean DEFAULT false, - ec_internationalorganizationeurinterests boolean DEFAULT false, - ec_internationalorganization boolean DEFAULT false, - ec_enterprise boolean DEFAULT false, - ec_smevalidated boolean DEFAULT false, - ec_nutscode boolean DEFAULT false, - country character varying(255), - collectedfrom character varying(255), - optional1 character varying(255), - optional2 character varying(255), - dateofcollection date DEFAULT ('now'::text)::date NOT NULL, - provenanceaction character varying(255) DEFAULT 'UNKNOWN'::character varying, - _dnet_resource_identifier_ character varying(2048) DEFAULT ((('temp_'::text || md5((clock_timestamp())::text)) || '_'::text) || md5((random())::text)), - lastupdate date DEFAULT ('now'::text)::date NOT NULL, - trust double precision DEFAULT 0.9 -); - - -ALTER TABLE public.dsm_organizations OWNER TO dnetapi; - --- --- Name: browse_countries; Type: VIEW; Schema: public; Owner: dnet --- - -CREATE VIEW browse_countries AS - SELECT o.country AS term, - count(*) AS total - FROM (((dsm_api a - LEFT JOIN dsm_datasources d ON (((a.datasource)::text = (d.id)::text))) - LEFT JOIN dsm_datasource_organization dao ON (((d.id)::text = (dao.datasource)::text))) - LEFT JOIN dsm_organizations o ON (((dao.organization)::text = (o.id)::text))) - GROUP BY o.country - ORDER BY count(*) DESC; - - -ALTER TABLE public.browse_countries OWNER TO dnet; - - - -CREATE TABLE dsm_apiparams ( - param character varying(255) NOT NULL, - value text DEFAULT ''::character varying NOT NULL, - api character varying(255) NOT NULL, - _dnet_resource_identifier_ character varying(2048) DEFAULT ((('temp_'::text || md5((clock_timestamp())::text)) || '_'::text) || md5((random())::text)) -); - - -ALTER TABLE public.dsm_apiparams OWNER TO dnetapi; - --- --- Name: dsm_datasource_api; Type: VIEW; Schema: public; Owner: dnet --- - -CREATE OR REPLACE VIEW dsm_datasource_api AS - SELECT - row_number() OVER (ORDER BY a.id) AS rowid, - d.id, - d.officialname, - d.englishname, - d.websiteurl, - d.contactemail, - d.collectedfrom, - d.typology, - d.platform, - d.registeredby, - d.managed, - a.protocol, - a.contentdescription, - a.active, - a.removable, - a.typology AS apitypology, - a.compatibility, - a.baseurl - FROM (dsm_datasources d - LEFT JOIN dsm_api a ON (((d.id)::text = (a.datasource)::text))); - --- --- Name: dsm_datasourcepids; Type: TABLE; Schema: public; Owner: dnetapi; Tablespace: --- - -CREATE TABLE dsm_datasourcepids ( - datasource character varying(255) NOT NULL, - pid character varying(255) NOT NULL, - _dnet_resource_identifier_ character varying(2048) DEFAULT ((('temp_'::text || md5((clock_timestamp())::text)) || '_'::text) || md5((random())::text)) -); - - -ALTER TABLE public.dsm_datasourcepids OWNER TO dnetapi; - --- --- Name: dsm_identities; Type: TABLE; Schema: public; Owner: dnetapi; Tablespace: --- - -CREATE TABLE dsm_identities ( - pid character varying(255) NOT NULL, - issuertype character varying(255), - _dnet_resource_identifier_ character varying(2048) DEFAULT ((('temp_'::text || md5((clock_timestamp())::text)) || '_'::text) || md5((random())::text)) -); - - -ALTER TABLE public.dsm_identities OWNER TO dnetapi; - --- --- Name: dsm_organizationpids; Type: TABLE; Schema: public; Owner: dnetapi; Tablespace: --- - -CREATE TABLE dsm_organizationpids ( - organization character varying(255) NOT NULL, - pid character varying(255) NOT NULL, - _dnet_resource_identifier_ character varying(2048) DEFAULT ((('temp_'::text || md5((clock_timestamp())::text)) || '_'::text) || md5((random())::text)) -); - - -ALTER TABLE public.dsm_organizationpids OWNER TO dnetapi; - --- --- Name: funder_identity; Type: TABLE; Schema: public; Owner: dnet; Tablespace: --- - -CREATE TABLE funder_identity ( - funder character varying(255), - pid character varying(255) -); - - -ALTER TABLE public.funder_identity OWNER TO dnet; - --- --- Name: funders; Type: TABLE; Schema: public; Owner: dnet; Tablespace: --- - -CREATE TABLE funders ( - id character varying(255) NOT NULL, - name character varying(255), - shortname character varying(255), - jurisdiction character varying(255), - websiteurl text, - policy character varying(255), - registrationdate date DEFAULT ('now'::text)::date NOT NULL, - lastupdatedate date -); - - -ALTER TABLE public.funders OWNER TO dnet; - --- --- Name: fundingpaths; Type: TABLE; Schema: public; Owner: dnet; Tablespace: --- - - - - - - - - - - - - - - --- --- Name: project_organization; Type: TABLE; Schema: public; Owner: dnet; Tablespace: --- - ---CREATE TABLE project_organization ( --- participantnumber integer, --- project character varying(255) NOT NULL, --- resporganization character varying(255) NOT NULL, --- semanticclass character varying(255) DEFAULT 'UNKNOWN'::character varying, --- trust double precision DEFAULT 0.9, --- _dnet_resource_identifier_ character varying(2048) DEFAULT ((('temp_'::text || md5((clock_timestamp())::text)) || '_'::text) || md5((random())::text)) ---); - - ---ALTER TABLE public.project_organization OWNER TO dnet; - - - - --- --- Name: dsm_api_pkey; Type: CONSTRAINT; Schema: public; Owner: dnetapi; Tablespace: --- - -ALTER TABLE ONLY dsm_api - ADD CONSTRAINT dsm_api_pkey PRIMARY KEY (id); - - --- --- Name: dsm_apicollection_pkey; Type: CONSTRAINT; Schema: public; Owner: dnetapi; Tablespace: --- - -ALTER TABLE ONLY dsm_apiparams - ADD CONSTRAINT dsm_apicollection_pkey PRIMARY KEY (api, param); - - --- --- Name: dsm_datasourcepids_pkey; Type: CONSTRAINT; Schema: public; Owner: dnetapi; Tablespace: --- - -ALTER TABLE ONLY dsm_datasourcepids - ADD CONSTRAINT dsm_datasourcepids_pkey PRIMARY KEY (datasource, pid); - - --- --- Name: dsm_datasources_namespaceprefix_key; Type: CONSTRAINT; Schema: public; Owner: dnetapi; Tablespace: --- - -ALTER TABLE ONLY dsm_datasources - ADD CONSTRAINT dsm_datasources_namespaceprefix_key UNIQUE (namespaceprefix); - - --- --- Name: dsm_datasources_pkey; Type: CONSTRAINT; Schema: public; Owner: dnetapi; Tablespace: --- - -ALTER TABLE ONLY dsm_datasources - ADD CONSTRAINT dsm_datasources_pkey PRIMARY KEY (id); - - --- --- Name: dsm_identities_pkey; Type: CONSTRAINT; Schema: public; Owner: dnetapi; Tablespace: --- - -ALTER TABLE ONLY dsm_identities - ADD CONSTRAINT dsm_identities_pkey PRIMARY KEY (pid); - - --- --- Name: dsm_organization_datasource_pkey; Type: CONSTRAINT; Schema: public; Owner: dnetapi; Tablespace: --- - -ALTER TABLE ONLY dsm_datasource_organization - ADD CONSTRAINT dsm_organization_datasource_pkey PRIMARY KEY (datasource, organization); - - --- --- Name: dsm_organizationpids_pkey; Type: CONSTRAINT; Schema: public; Owner: dnetapi; Tablespace: --- - -ALTER TABLE ONLY dsm_organizationpids - ADD CONSTRAINT dsm_organizationpids_pkey PRIMARY KEY (organization, pid); - - --- --- Name: dsm_organizations_pkey; Type: CONSTRAINT; Schema: public; Owner: dnetapi; Tablespace: --- - -ALTER TABLE ONLY dsm_organizations - ADD CONSTRAINT dsm_organizations_pkey PRIMARY KEY (id); - - - - - - --- --- Name: dsm_datasources_contactemail_idx; Type: INDEX; Schema: public; Owner: dnetapi; Tablespace: --- - -CREATE INDEX dsm_datasources_contactemail_idx ON dsm_datasources USING btree (contactemail); - - --- --- Name: dsm_datasources_englishname_idx; Type: INDEX; Schema: public; Owner: dnetapi; Tablespace: --- - -CREATE INDEX dsm_datasources_englishname_idx ON dsm_datasources USING btree (englishname); - - --- --- Name: dsm_datasources_managed_idx; Type: INDEX; Schema: public; Owner: dnetapi; Tablespace: --- - -CREATE INDEX dsm_datasources_managed_idx ON dsm_datasources USING btree (managed); - - --- --- Name: dsm_datasources_officialname_idx; Type: INDEX; Schema: public; Owner: dnetapi; Tablespace: --- - -CREATE INDEX dsm_datasources_officialname_idx ON dsm_datasources USING btree (officialname); - - --- --- Name: dsm_datasources_registeredby_idx; Type: INDEX; Schema: public; Owner: dnetapi; Tablespace: --- - -CREATE INDEX dsm_datasources_registeredby_idx ON dsm_datasources USING btree (registeredby); - - --- --- Name: dsm_organizations_country_idx; Type: INDEX; Schema: public; Owner: dnetapi; Tablespace: --- - -CREATE INDEX dsm_organizations_country_idx ON dsm_organizations USING btree (country); - - - - - - - - -ALTER TABLE ONLY dsm_api - ADD CONSTRAINT dsm_api_datasource_fkey FOREIGN KEY (datasource) REFERENCES dsm_datasources(id) ON DELETE CASCADE; - - --- --- Name: dsm_apicollections_api_fkey; Type: FK CONSTRAINT; Schema: public; Owner: dnetapi --- - -ALTER TABLE ONLY dsm_apiparams - ADD CONSTRAINT dsm_apicollections_api_fkey FOREIGN KEY (api) REFERENCES dsm_api(id); - - --- --- Name: dsm_datasource_organization_datasource_fkey; Type: FK CONSTRAINT; Schema: public; Owner: dnetapi --- - -ALTER TABLE ONLY dsm_datasource_organization - ADD CONSTRAINT dsm_datasource_organization_datasource_fkey FOREIGN KEY (datasource) REFERENCES dsm_datasources(id); - - --- --- Name: dsm_datasource_organization_organization_fkey; Type: FK CONSTRAINT; Schema: public; Owner: dnetapi --- - -ALTER TABLE ONLY dsm_datasource_organization - ADD CONSTRAINT dsm_datasource_organization_organization_fkey FOREIGN KEY (organization) REFERENCES dsm_organizations(id) ON DELETE CASCADE; - - --- --- Name: dsm_datasourcepids_datasource_fkey; Type: FK CONSTRAINT; Schema: public; Owner: dnetapi --- - -ALTER TABLE ONLY dsm_datasourcepids - ADD CONSTRAINT dsm_datasourcepids_datasource_fkey FOREIGN KEY (datasource) REFERENCES dsm_datasources(id); - - --- --- Name: dsm_datasourcepids_pid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: dnetapi --- - -ALTER TABLE ONLY dsm_datasourcepids - ADD CONSTRAINT dsm_datasourcepids_pid_fkey FOREIGN KEY (pid) REFERENCES dsm_identities(pid); - - --- --- Name: dsm_datasources_collectedfrom_fkey; Type: FK CONSTRAINT; Schema: public; Owner: dnetapi --- - -ALTER TABLE ONLY dsm_datasources - ADD CONSTRAINT dsm_datasources_collectedfrom_fkey FOREIGN KEY (collectedfrom) REFERENCES dsm_datasources(id); - - --- --- Name: dsm_organizationpids_organization_fkey; Type: FK CONSTRAINT; Schema: public; Owner: dnetapi --- - -ALTER TABLE ONLY dsm_organizationpids - ADD CONSTRAINT dsm_organizationpids_organization_fkey FOREIGN KEY (organization) REFERENCES dsm_organizations(id); - - --- --- Name: dsm_organizationpids_pid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: dnetapi --- - -ALTER TABLE ONLY dsm_organizationpids - ADD CONSTRAINT dsm_organizationpids_pid_fkey FOREIGN KEY (pid) REFERENCES dsm_identities(pid); - - --- --- Name: dsm_organizations_collectedfrom_fkey; Type: FK CONSTRAINT; Schema: public; Owner: dnetapi --- - -ALTER TABLE ONLY dsm_organizations - ADD CONSTRAINT dsm_organizations_collectedfrom_fkey FOREIGN KEY (collectedfrom) REFERENCES dsm_datasources(id); - - - -REVOKE ALL ON SCHEMA public FROM PUBLIC; -REVOKE ALL ON SCHEMA public FROM postgres; -GRANT ALL ON SCHEMA public TO postgres; -GRANT ALL ON SCHEMA public TO PUBLIC; - - -REVOKE ALL ON TABLE browse_countries FROM PUBLIC; -REVOKE ALL ON TABLE browse_countries FROM dnet; -GRANT ALL ON TABLE browse_countries TO dnet; -GRANT ALL ON TABLE browse_countries TO dnetapi; - - -REVOKE ALL ON TABLE dsm_datasource_api FROM PUBLIC; -REVOKE ALL ON TABLE dsm_datasource_api FROM dnet; -GRANT ALL ON TABLE dsm_datasource_api TO dnet; -GRANT SELECT ON TABLE dsm_datasource_api TO dnetapi; - - diff --git a/apps/dnet-exporter-api/src/main/resources/dnet_info.sql b/apps/dnet-exporter-api/src/main/resources/dnet_info.sql deleted file mode 100644 index 23ddb680..00000000 --- a/apps/dnet-exporter-api/src/main/resources/dnet_info.sql +++ /dev/null @@ -1,21 +0,0 @@ - -CREATE TABLE info ( - key character varying(255) NOT NULL PRIMARY KEY, - value DATE -); - -GRANT ALL ON TABLE info TO dnet; -GRANT ALL ON TABLE info TO dnetapi; - -INSERT INTO info(key) VALUES ('oaf_load_date'); -INSERT INTO info(key) VALUES ('odf_load_date'); -INSERT INTO info(key) VALUES ('inference_date'); -INSERT INTO info(key) VALUES ('claim_load_date'); -INSERT INTO info(key) VALUES ('stats_update_date'); - -INSERT INTO info(key) VALUES ('crossref_update_date'); -INSERT INTO info(key) VALUES ('unpaywall_update_date'); -INSERT INTO info(key) VALUES ('orcid_update_date'); -INSERT INTO info(key) VALUES ('mag_update_date'); - - diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources.sql.st b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources.sql.st index a1d29624..d6e3a311 100644 --- a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources.sql.st +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources.sql.st @@ -10,9 +10,9 @@ select a.last_collection_date as "lastCollectionDate", a.last_collection_total as "lastCollectionTotal" from - dsm_datasources d - left outer join dsm_api a on (d.id = a.datasource) - left outer join dsm_datasource_organization dso on (d.id = dso.datasource) + dsm_services d + left outer join dsm_api a on (d.id = a.service) + left outer join dsm_service_organization dso on (d.id = dso.service) left outer join dsm_organizations o on (o.id = dso.organization) where d.registrationdate is not null diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate.st.sql b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate.st.sql index 057d5a2f..e014461c 100644 --- a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate.st.sql +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate.st.sql @@ -1,8 +1,8 @@ select count(d.id) as count from - dsm_datasources d - left outer join dsm_api a on (d.id = a.datasource) - left outer join dsm_datasource_organization dso on (d.id = dso.datasource) + dsm_services d + left outer join dsm_api a on (d.id = a.service) + left outer join dsm_service_organization dso on (d.id = dso.service) left outer join dsm_organizations o on (o.id = dso.organization) where d.registrationdate >= cast(? as date) diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate_typology.st.sql b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate_typology.st.sql index ae5de98c..76a8fb45 100644 --- a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate_typology.st.sql +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate_typology.st.sql @@ -1,8 +1,8 @@ select count(d.id) as count from - dsm_datasources d - left outer join dsm_api a on (d.id = a.datasource) - left outer join dsm_datasource_organization dso on (d.id = dso.datasource) + dsm_services d + left outer join dsm_api a on (d.id = a.service) + left outer join dsm_service_organization dso on (d.id = dso.service) left outer join dsm_organizations o on (o.id = dso.organization) where d.registrationdate >= cast(? as date) -- 2.17.1 From 37b662ac205699946da960806cbb98e6fcb74489 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Mon, 7 Feb 2022 10:35:41 +0100 Subject: [PATCH 05/19] move vs dnet_services --- .../dsm/domain/db/DatasourceApiDbEntry.java | 13 ++++++++----- .../openaire/dsm/domain/db/DatasourceDbEntry.java | 5 +++-- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/DatasourceApiDbEntry.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/DatasourceApiDbEntry.java index 5b719b1f..59acddeb 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/DatasourceApiDbEntry.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/DatasourceApiDbEntry.java @@ -1,14 +1,17 @@ package eu.dnetlib.openaire.dsm.domain.db; -import javax.persistence.*; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Table; import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonIgnore; + import io.swagger.annotations.ApiModel; @Entity @JsonAutoDetect -@Table(name = "dsm_datasource_api") +@Table(name = "dsm_service_api") @ApiModel(value = "DatasourceApi model", description = "describes a joint view between datasources and their API (1:N)") public class DatasourceApiDbEntry { @@ -169,7 +172,7 @@ public class DatasourceApiDbEntry { return id; } - public void setId(String id) { + public void setId(final String id) { this.id = id; } @@ -177,7 +180,7 @@ public class DatasourceApiDbEntry { return collectedfrom; } - public void setCollectedfrom(String collectedfrom) { + public void setCollectedfrom(final String collectedfrom) { this.collectedfrom = collectedfrom; } -} \ No newline at end of file +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/DatasourceDbEntry.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/DatasourceDbEntry.java index 490c7af4..32d72a77 100755 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/DatasourceDbEntry.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/DatasourceDbEntry.java @@ -4,17 +4,18 @@ import javax.persistence.Entity; import javax.persistence.Table; import javax.persistence.Transient; -import eu.dnetlib.enabling.datasources.common.Datasource; import org.hibernate.annotations.DynamicUpdate; import org.hibernate.annotations.SelectBeforeUpdate; +import eu.dnetlib.enabling.datasources.common.Datasource; + /** * Datasource */ @Entity @DynamicUpdate @SelectBeforeUpdate -@Table(name = "dsm_datasources") +@Table(name = "dsm_services") public class DatasourceDbEntry extends Datasource { @Transient -- 2.17.1 From 41d4846027e1bd650a2b4ccaa15cf7b2afe1c87d Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Tue, 15 Feb 2022 08:50:59 +0100 Subject: [PATCH 06/19] claening --- .../src/main/resources/sql/importDedupEvents.sql | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/apps/dnet-orgs-database-application/src/main/resources/sql/importDedupEvents.sql b/apps/dnet-orgs-database-application/src/main/resources/sql/importDedupEvents.sql index 3be235ce..0ce42afc 100644 --- a/apps/dnet-orgs-database-application/src/main/resources/sql/importDedupEvents.sql +++ b/apps/dnet-orgs-database-application/src/main/resources/sql/importDedupEvents.sql @@ -137,14 +137,4 @@ DELETE FROM oa_duplicates d USING oa_duplicates d1 WHERE d.oa_original_id = d1.oa_original_id AND d.reltype = 'suggested' AND d1.reltype = 'is_similar'; - - - - - - - - - - COMMIT; -- 2.17.1 From 3b898f6a3e8bc7809109c24213928b718f52defd Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Tue, 22 Mar 2022 08:58:20 +0100 Subject: [PATCH 07/19] fixed some problem with EOSC model --- apps/dnet-exporter-api/pom.xml | 2 +- .../java/eu/dnetlib/openaire/dsm/dao/DatasourceDaoImpl.java | 6 +++++- .../openaire/dsm/dao/DatasourceDbEntryRepository.java | 3 +++ .../openaire/dsm/domain/db/DatasourceApiDbEntry.java | 2 +- .../src/main/resources/application.properties | 1 + apps/dnet-exporter-api/src/main/resources/global.properties | 2 +- 6 files changed, 12 insertions(+), 4 deletions(-) diff --git a/apps/dnet-exporter-api/pom.xml b/apps/dnet-exporter-api/pom.xml index 9d1433d6..f1cb0be9 100644 --- a/apps/dnet-exporter-api/pom.xml +++ b/apps/dnet-exporter-api/pom.xml @@ -123,7 +123,7 @@ eu.dnetlib dnet-datasource-manager-common - [1.1.0,2.0.0) + 1.2.0-EOSC-SNAPSHOT diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDaoImpl.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDaoImpl.java index 09fae5bc..f16ab315 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDaoImpl.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDaoImpl.java @@ -111,6 +111,11 @@ public class DatasourceDaoImpl implements DatasourceDao new DsmException("Datasource not found. ID: " + dsId)); } + @Override + public DatasourceDbEntry getDsByNsPrefix(final String prefix) throws DsmException { + return dsRepository.findByNamespaceprefix(prefix).orElseThrow(() -> new DsmException("Datasource not found. NS Prefix: " + prefix)); + } + @Override public void setManaged(final String id, final boolean managed) { log.info(String.format("setting managed = '%s' for ds '%s'", managed, id)); @@ -163,7 +168,6 @@ public class DatasourceDaoImpl implements DatasourceDao, JpaSpecificationExecutor { + Optional findByNamespaceprefix(String namespaceprefix); + @Query("select d.managed from #{#entityName} d where d.id = ?1") boolean isManaged(String id); diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/DatasourceApiDbEntry.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/DatasourceApiDbEntry.java index 59acddeb..0a3e5ede 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/DatasourceApiDbEntry.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/DatasourceApiDbEntry.java @@ -11,7 +11,7 @@ import io.swagger.annotations.ApiModel; @Entity @JsonAutoDetect -@Table(name = "dsm_service_api") +@Table(name = "dsm_datasource_api") @ApiModel(value = "DatasourceApi model", description = "describes a joint view between datasources and their API (1:N)") public class DatasourceApiDbEntry { diff --git a/apps/dnet-exporter-api/src/main/resources/application.properties b/apps/dnet-exporter-api/src/main/resources/application.properties index 9c4e8b67..2b947038 100644 --- a/apps/dnet-exporter-api/src/main/resources/application.properties +++ b/apps/dnet-exporter-api/src/main/resources/application.properties @@ -31,4 +31,5 @@ openaire.exporter.enable.info = true # CACHE TTL, 12h openaire.exporter.cache.ttl = 43200000 +maven.pom.path = /META-INF/maven/eu.dnetlib.dhp/dnet-exporter-api/effective-pom.xml diff --git a/apps/dnet-exporter-api/src/main/resources/global.properties b/apps/dnet-exporter-api/src/main/resources/global.properties index 0067964f..61657adf 100644 --- a/apps/dnet-exporter-api/src/main/resources/global.properties +++ b/apps/dnet-exporter-api/src/main/resources/global.properties @@ -15,7 +15,7 @@ openaire.exporter.cxfClientConnectTimeout = 60000 openaire.exporter.cxfClientReceiveTimeout = 120000 # JDBC -openaire.exporter.jdbc.url = jdbc:postgresql://localhost:5432/dnet_openaire +openaire.exporter.jdbc.url = jdbc:postgresql://localhost:5432/dnet_openaireplus openaire.exporter.jdbc.user = dnetapi openaire.exporter.jdbc.pwd = dnetPwd openaire.exporter.jdbc.minIdle = 1 -- 2.17.1 From 1311afa99369e166128828b042f123b611b29ed7 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Tue, 22 Mar 2022 10:37:49 +0100 Subject: [PATCH 08/19] swagger: removed duplicated group --- .../java/eu/dnetlib/DNetOpenaireExporterApplication.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/DNetOpenaireExporterApplication.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/DNetOpenaireExporterApplication.java index 5f61230e..8d543c98 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/DNetOpenaireExporterApplication.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/DNetOpenaireExporterApplication.java @@ -69,11 +69,6 @@ public class DNetOpenaireExporterApplication extends AbstractDnetApp { return _docket("OpenAIRE Contexts", ContextApiController.class.getPackage().getName(), config.getSwaggerCommunities(), V1); } - @Bean - public Docket info() { - return _docket("OpenAIRE Info", InfoController.class.getPackage().getName(), config.getSwaggerInfo(), V1); - } - private Docket _docket(final String groupName, final String controllerPackage, final Swagger swag, final String version) { final Docket d = new Docket(DocumentationType.SWAGGER_2); configSwagger(d, groupName, controllerPackage, swag, version); -- 2.17.1 From 6e653e125876e1d41f1f66ee4506212c8d054a2d Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Wed, 23 Mar 2022 08:49:13 +0100 Subject: [PATCH 09/19] swagger home redirect --- .../src/main/java/eu/dnetlib/SwaggerController.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/SwaggerController.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/SwaggerController.java index c3cadff1..b53bbbc4 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/SwaggerController.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/SwaggerController.java @@ -1,7 +1,4 @@ package eu.dnetlib; -import java.io.IOException; - -import javax.servlet.http.HttpServletResponse; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.RequestMapping; @@ -10,10 +7,10 @@ import org.springframework.web.bind.annotation.RequestMapping; public class SwaggerController { @RequestMapping(value = { - "/", "/docs" + "/", "/docs", "swagger-ui.html" }) - public void index(final HttpServletResponse response) throws IOException { - response.sendRedirect("swagger-ui.html"); + public String index() { + return "redirect:swagger-ui/"; } } -- 2.17.1 From 485c865896e1245b767f5f512b7000eb52a66537 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Wed, 23 Mar 2022 09:58:02 +0100 Subject: [PATCH 10/19] fixed swagger error --- .../java/eu/dnetlib/DNetOpenaireExporterApplication.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/DNetOpenaireExporterApplication.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/DNetOpenaireExporterApplication.java index 8d543c98..8af6272c 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/DNetOpenaireExporterApplication.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/DNetOpenaireExporterApplication.java @@ -2,6 +2,8 @@ package eu.dnetlib; import static springfox.documentation.builders.RequestHandlerSelectors.basePackage; +import java.time.LocalDate; + import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; @@ -86,8 +88,7 @@ public class DNetOpenaireExporterApplication extends AbstractDnetApp { .select() .apis(basePackage(controllerPackage)) .build() - .directModelSubstitute(org.joda.time.LocalDate.class, java.sql.Date.class) - .directModelSubstitute(org.joda.time.DateTime.class, java.util.Date.class) + .directModelSubstitute(LocalDate.class, java.sql.Date.class) .apiInfo(apiInfo(swag, version)); } -- 2.17.1 From 00ed2922da1675267206ff4d73094f5dddcad333 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Thu, 24 Mar 2022 12:26:14 +0100 Subject: [PATCH 11/19] removed typology field --- .../java/eu/dnetlib/openaire/dsm/DsmCore.java | 6 +- .../openaire/dsm/dao/DatasourceDao.java | 6 +- .../openaire/dsm/dao/DatasourceDaoImpl.java | 8 +- .../dsm/dao/DatasourceDbEntryRepository.java | 4 +- .../dsm/dao/utils/DsmMappingUtils.java | 122 +----------------- .../openaire/dsm/domain/ApiDetails.java | 29 +++-- .../dsm/domain/DatasourceDetails.java | 10 +- .../dsm/domain/DatasourceDetailsUpdate.java | 10 +- .../dsm/domain/DatasourceSnippetExtended.java | 10 +- .../openaire/dsm/domain/FilterName.java | 51 +++++--- .../dsm/domain/RegisteredDatasourceInfo.java | 11 +- .../dsm/domain/db/DatasourceApiDbEntry.java | 23 ++-- .../sql/recent_registered_datasources.sql.st | 2 +- ...tered_datasources_fromDate_typology.st.sql | 2 +- .../src/main/resources/global.properties | 2 +- 15 files changed, 93 insertions(+), 203 deletions(-) diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmCore.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmCore.java index f4721e08..dedba147 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmCore.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmCore.java @@ -262,14 +262,14 @@ public class DsmCore { } } - public Long countRegisteredAfter(final String fromDate, final String typologyFilter) throws Throwable { + public Long countRegisteredAfter(final String fromDate, final String typeFilter) throws Throwable { try { - if (StringUtils.isNotBlank(typologyFilter)) { + if (StringUtils.isNotBlank(typeFilter)) { final String sql = IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate_typology.st.sql"), Charset .defaultCharset()); - return jdbcTemplate.queryForObject(sql, Long.class, fromDate, typologyFilter + "%"); + return jdbcTemplate.queryForObject(sql, Long.class, fromDate, typeFilter + "%"); } else { final String sql = IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate.st.sql"), Charset diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDao.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDao.java index 4fb71855..7ab7583e 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDao.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDao.java @@ -2,6 +2,8 @@ package eu.dnetlib.openaire.dsm.dao; import java.util.List; +import org.springframework.data.domain.Page; + import eu.dnetlib.enabling.datasources.common.Api; import eu.dnetlib.enabling.datasources.common.Datasource; import eu.dnetlib.enabling.datasources.common.DatasourceManagerCommon; @@ -10,7 +12,6 @@ import eu.dnetlib.openaire.dsm.domain.RequestFilter; import eu.dnetlib.openaire.dsm.domain.RequestSort; import eu.dnetlib.openaire.dsm.domain.RequestSortOrder; import eu.dnetlib.openaire.vocabularies.Country; -import org.springframework.data.domain.Page; public interface DatasourceDao, API extends Api> extends DatasourceManagerCommon { @@ -32,7 +33,7 @@ public interface DatasourceDao, API extends Api> void updateTimezone(String dsId, String timezone) throws DsmException; - void updateTypology(String dsId, String timezone) throws DsmException; + void updateEoscDatasourceType(String dsId, String timezone) throws DsmException; void updateRegisteringUser(String dsId, String registeredBy) throws DsmException; @@ -54,5 +55,6 @@ public interface DatasourceDao, API extends Api> void updateApiBaseUrl(String apiId, String baseUrl) throws DsmException; + @Override void addApi(final API api) throws DsmException; } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDaoImpl.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDaoImpl.java index f16ab315..0f9cf5aa 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDaoImpl.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDaoImpl.java @@ -238,15 +238,15 @@ public class DatasourceDaoImpl implements DatasourceDao { - accessProtocol.addAttribute(ap.getParam(), ap.getValue()); - }); - } - iface.addElement("BASE_URL").setText(api.getBaseurl()); - iface.addElement("INTERFACE_EXTRA_FIELD").addAttribute("name", "last_collection_date"); - iface.addElement("INTERFACE_EXTRA_FIELD").addAttribute("name", "last_collection_mdId"); - iface.addElement("INTERFACE_EXTRA_FIELD").addAttribute("name", "last_collection_total"); - iface.addElement("INTERFACE_EXTRA_FIELD").addAttribute("name", "last_aggregation_date"); - iface.addElement("INTERFACE_EXTRA_FIELD").addAttribute("name", "last_aggregation_mdId"); - iface.addElement("INTERFACE_EXTRA_FIELD").addAttribute("name", "last_aggregation_total"); - - final Element mdPathNode = iface.addElement("INTERFACE_EXTRA_FIELD"); - mdPathNode.addAttribute("name", "metadata_identifier_path"); - if (StringUtils.isNotBlank(api.getMetadataIdentifierPath())) { - mdPathNode.setText(api.getMetadataIdentifierPath()); - } - - return iface.asXML(); - } - // HELPERS private static T _convert(final Object o, final Class clazz) { @@ -217,17 +108,6 @@ public class DsmMappingUtils { return mapper.convertValue(o, clazz); } - private static OrganizationDetails getOrganization(final DatasourceDetails ds) { - if (ds.getOrganizations() != null && !ds.getOrganizations().isEmpty()) { return ds.getOrganizations().stream().findFirst().get(); } - return null; - } - - private static void addExtraField(final Element extraFields, final String field, final String value) { - final Element f = extraFields.addElement("FIELD"); - f.addElement("key").setText(field); - f.addElement("value").setText(value != null ? value : ""); - } - private static String asOpenaireId(final String id) { final String prefix = StringUtils.substringBefore(id, ID_SEPARATOR); final String md5 = StringUtils.substringAfter(id, ID_SEPARATOR); diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/ApiDetails.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/ApiDetails.java index 5a51df3d..d2ab101a 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/ApiDetails.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/ApiDetails.java @@ -25,7 +25,7 @@ public class ApiDetails extends ApiIgnoredProperties { private String contentdescription = null; @ApiModelProperty(position = 4) - private String typology = null; + private String eoscDatasourceType = null; @ApiModelProperty(position = 5) private String compatibility; @@ -79,10 +79,6 @@ public class ApiDetails extends ApiIgnoredProperties { return contentdescription; } - public String getTypology() { - return typology; - } - public String getCompatibility() { return compatibility; } @@ -135,11 +131,6 @@ public class ApiDetails extends ApiIgnoredProperties { return this; } - public ApiDetails setTypology(final String typology) { - this.typology = typology; - return this; - } - public ApiDetails setCompatibility(final String compatibility) { this.compatibility = compatibility; return this; @@ -192,23 +183,35 @@ public class ApiDetails extends ApiIgnoredProperties { return compatibilityOverride; } - public void setCompatibilityOverride(final String compatibilityOverride) { + public ApiDetails setCompatibilityOverride(final String compatibilityOverride) { this.compatibilityOverride = compatibilityOverride; + return this; } public Boolean getRemovable() { return removable; } - public void setRemovable(final Boolean removable) { + public ApiDetails setRemovable(final Boolean removable) { this.removable = removable; + return this; } public String getMetadataIdentifierPath() { return metadataIdentifierPath; } - public void setMetadataIdentifierPath(final String metadataIdentifierPath) { + public ApiDetails setMetadataIdentifierPath(final String metadataIdentifierPath) { this.metadataIdentifierPath = metadataIdentifierPath; + return this; + } + + public String getEoscDatasourceType() { + return eoscDatasourceType; + } + + public ApiDetails setEoscDatasourceType(final String eoscDatasourceType) { + this.eoscDatasourceType = eoscDatasourceType; + return this; } } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetails.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetails.java index deb52543..dd4967f2 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetails.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetails.java @@ -66,7 +66,7 @@ public class DatasourceDetails extends DatasourceIgnoredProperties { @NotBlank @ApiModelProperty(position = 13) - private String typology; + private String eoscDatasourceType; @ApiModelProperty(position = 14) private Date dateofcollection; @@ -172,8 +172,8 @@ public class DatasourceDetails extends DatasourceIgnoredProperties { return dateofvalidation; } - public String getTypology() { - return typology; + public String getEoscDatasourceType() { + return eoscDatasourceType; } public Date getDateofcollection() { @@ -305,8 +305,8 @@ public class DatasourceDetails extends DatasourceIgnoredProperties { return this; } - public DatasourceDetails setTypology(final String typology) { - this.typology = typology; + public DatasourceDetails setEoscDatasourceType(final String eoscDatasourceType) { + this.eoscDatasourceType = eoscDatasourceType; return this; } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetailsUpdate.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetailsUpdate.java index c3ddcb88..06c8ce5a 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetailsUpdate.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetailsUpdate.java @@ -51,7 +51,7 @@ public class DatasourceDetailsUpdate { @NotBlank @ApiModelProperty(position = 13) - private String typology; + private String eoscDatasourceType; @ApiModelProperty(position = 15) private String platform; @@ -114,8 +114,8 @@ public class DatasourceDetailsUpdate { return timezone; } - public String getTypology() { - return typology; + public String getEoscDatasourceType() { + return eoscDatasourceType; } public String getPlatform() { @@ -183,8 +183,8 @@ public class DatasourceDetailsUpdate { return this; } - public DatasourceDetailsUpdate setTypology(final String typology) { - this.typology = typology; + public DatasourceDetailsUpdate setEoscDatasourceType(final String eoscDatasourceType) { + this.eoscDatasourceType = eoscDatasourceType; return this; } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceSnippetExtended.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceSnippetExtended.java index 902e2c01..9c544658 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceSnippetExtended.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceSnippetExtended.java @@ -38,7 +38,7 @@ public class DatasourceSnippetExtended { private Date registrationdate; @ApiModelProperty(position = 7) - private String typology; + private String eoscDatasourceType; @ApiModelProperty(position = 8) private String logoUrl; @@ -106,12 +106,12 @@ public class DatasourceSnippetExtended { this.registrationdate = registrationdate; } - public String getTypology() { - return typology; + public String getEoscDatasourceType() { + return eoscDatasourceType; } - public void setTypology(final String typology) { - this.typology = typology; + public void setEoscDatasourceType(final String eoscDatasourceType) { + this.eoscDatasourceType = eoscDatasourceType; } public String getLogoUrl() { diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/FilterName.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/FilterName.java index 7e6cb524..f6e3de96 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/FilterName.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/FilterName.java @@ -1,34 +1,43 @@ package eu.dnetlib.openaire.dsm.domain; import com.fasterxml.jackson.annotation.JsonAutoDetect; + import io.swagger.annotations.ApiModel; @JsonAutoDetect @ApiModel(value = "Filter name", description = "List of the field names used to filter datasources") -public enum -FilterName { - id, managed, collectedfrom, // exact match - officialname, englishname, websiteurl, contactemail, registeredby, typology, platform, // like match +public enum FilterName { + + id, + managed, + collectedfrom, // exact match + officialname, + englishname, + websiteurl, + contactemail, + registeredby, + eoscDatasourceType, + platform, // like match country; // exact match on related organization - public static FilterType type(FilterName filterName) { + public static FilterType type(final FilterName filterName) { switch (filterName) { - case id: - case managed: - case collectedfrom: - return FilterType.exact; - case officialname: - case englishname: - case websiteurl: - case contactemail: - case registeredby: - case typology: - case platform: - return FilterType.search; - case country: - return FilterType.searchOrgs; - default: - throw new IllegalStateException("unmapped filter type for: " + filterName); + case id: + case managed: + case collectedfrom: + return FilterType.exact; + case officialname: + case englishname: + case websiteurl: + case contactemail: + case registeredby: + case eoscDatasourceType: + case platform: + return FilterType.search; + case country: + return FilterType.searchOrgs; + default: + throw new IllegalStateException("unmapped filter type for: " + filterName); } } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/RegisteredDatasourceInfo.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/RegisteredDatasourceInfo.java index 31e5e6b5..5bd17cc2 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/RegisteredDatasourceInfo.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/RegisteredDatasourceInfo.java @@ -6,7 +6,7 @@ public class RegisteredDatasourceInfo { private String officialName; private String englishName; private String organization; - private String typology; + private String eoscDatasourceType; private String registeredBy; private String registrationDate; private String compatibility; @@ -45,12 +45,12 @@ public class RegisteredDatasourceInfo { this.organization = organization; } - public String getTypology() { - return typology; + public String getEoscDatasourceType() { + return eoscDatasourceType; } - public void setTypology(final String typology) { - this.typology = typology; + public void setEoscDatasourceType(final String eoscDatasourceType) { + this.eoscDatasourceType = eoscDatasourceType; } public String getRegisteredBy() { @@ -92,4 +92,5 @@ public class RegisteredDatasourceInfo { public void setLastCollectionTotal(final long lastCollectionTotal) { this.lastCollectionTotal = lastCollectionTotal; } + } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/DatasourceApiDbEntry.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/DatasourceApiDbEntry.java index 0a3e5ede..064c4f41 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/DatasourceApiDbEntry.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/DatasourceApiDbEntry.java @@ -1,5 +1,6 @@ package eu.dnetlib.openaire.dsm.domain.db; +import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.Id; import javax.persistence.Table; @@ -26,7 +27,8 @@ public class DatasourceApiDbEntry { private String contactemail; private String collectedfrom; private String registeredby; - private String typology; + @Column(name = "eosc_datasource_type") + private String eoscDatasourceType; private String platform; private Boolean managed; @@ -34,7 +36,7 @@ public class DatasourceApiDbEntry { protected String contentdescription = null; protected Boolean active = false; protected Boolean removable = false; - protected String apitypology = null; + protected String compatibility; private String baseurl; @@ -96,12 +98,12 @@ public class DatasourceApiDbEntry { this.websiteurl = websiteurl; } - public String getTypology() { - return typology; + public String getEoscDatasourceType() { + return eoscDatasourceType; } - public void setTypology(final String typology) { - this.typology = typology; + public void setEoscDatasourceType(final String eoscDatasourceType) { + this.eoscDatasourceType = eoscDatasourceType; } public String getPlatform() { @@ -152,14 +154,6 @@ public class DatasourceApiDbEntry { this.removable = removable; } - public String getApitypology() { - return apitypology; - } - - public void setApitypology(final String apitypology) { - this.apitypology = apitypology; - } - public String getCompatibility() { return compatibility; } @@ -183,4 +177,5 @@ public class DatasourceApiDbEntry { public void setCollectedfrom(final String collectedfrom) { this.collectedfrom = collectedfrom; } + } diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources.sql.st b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources.sql.st index d6e3a311..56bff2be 100644 --- a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources.sql.st +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources.sql.st @@ -3,7 +3,7 @@ select d.officialname as "officialName", d.englishname as "englishName", o.legalname as organization, - d.typology as typology, + d.eosc_datasource_type as eosc_datasource_type, d.registeredby as "registeredBy", d.registrationdate::text as "registrationDate", a.compatibility as compatibility, diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate_typology.st.sql b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate_typology.st.sql index 76a8fb45..1af6acd7 100644 --- a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate_typology.st.sql +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate_typology.st.sql @@ -6,7 +6,7 @@ from left outer join dsm_organizations o on (o.id = dso.organization) where d.registrationdate >= cast(? as date) - and d.typology like ? + and d.eosc_datasource_type like ? and d.registrationdate < a.last_collection_date and d.registeredby is not null and d.managed = true diff --git a/apps/dnet-exporter-api/src/main/resources/global.properties b/apps/dnet-exporter-api/src/main/resources/global.properties index 61657adf..bef55ff0 100644 --- a/apps/dnet-exporter-api/src/main/resources/global.properties +++ b/apps/dnet-exporter-api/src/main/resources/global.properties @@ -102,4 +102,4 @@ openaire.exporter.swaggerInfo.apiContactEmail = ${openaire.exporter.swaggerD # VOCABULARIES openaire.exporter.vocabularies.baseUrl = http://localhost:8980/provision/mvc/vocabularies openaire.exporter.vocabularies.countriesEndpoint = ${openaire.exporter.vocabularies.baseUrl}/dnet:countries.json -openaire.exporter.vocabularies.datasourceTypologiesEndpoint = ${openaire.exporter.vocabularies.baseUrl}/dnet:datasource_typologies.json +openaire.exporter.vocabularies.datasourceTypologiesEndpoint = ${openaire.exporter.vocabularies.baseUrl}/dnet:eosc_datasource_types.json -- 2.17.1 From a6909dc253494ca4f8eac8a6a7d23370dffa5f8b Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Fri, 25 Mar 2022 10:52:19 +0100 Subject: [PATCH 12/19] fixed jpa query --- .../dnetlib/openaire/dsm/dao/DatasourceDbEntryRepository.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDbEntryRepository.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDbEntryRepository.java index 8b2328f8..b5adcd8d 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDbEntryRepository.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDbEntryRepository.java @@ -53,7 +53,7 @@ public interface DatasourceDbEntryRepository extends JpaRepository Date: Fri, 25 Mar 2022 14:42:47 +0100 Subject: [PATCH 13/19] set of field eoscDatasourceType in getApis() --- .../src/main/java/eu/dnetlib/openaire/dsm/DsmCore.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmCore.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmCore.java index dedba147..0d7e9107 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmCore.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmCore.java @@ -144,9 +144,11 @@ public class DsmCore { public ApiDetailsResponse getApis(final String dsId) throws DsmException { try { + final String eoscType = dsDao.getDs(dsId).getEoscDatasourceType(); final List apis = dsDao.getApis(dsId); final List api = apis.stream() .map(DsmMappingUtils::asDetails) + .map(a -> a.setEoscDatasourceType(eoscType)) .collect(Collectors.toList()); return ResponseUtils.apiResponse(api, api.size()); } catch (final Throwable e) { -- 2.17.1 From a6709a3878b739ade67d6ef94c5b73164bc0f8fd Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Thu, 7 Apr 2022 08:05:39 +0200 Subject: [PATCH 14/19] add PidSystemDbEntry --- .../openaire/dsm/dao/DatasourceDao.java | 2 +- .../dsm/domain/db/DatasourceDbEntry.java | 2 +- .../dsm/domain/db/PidSystemDbEntry.java | 57 +++++++++++++++++ .../dsm/domain/db/PidSystemKeyDbEntry.java | 62 +++++++++++++++++++ 4 files changed, 121 insertions(+), 2 deletions(-) create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/PidSystemDbEntry.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/PidSystemKeyDbEntry.java diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDao.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDao.java index 7ab7583e..5f40524d 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDao.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDao.java @@ -13,7 +13,7 @@ import eu.dnetlib.openaire.dsm.domain.RequestSort; import eu.dnetlib.openaire.dsm.domain.RequestSortOrder; import eu.dnetlib.openaire.vocabularies.Country; -public interface DatasourceDao, API extends Api> extends DatasourceManagerCommon { +public interface DatasourceDao, API extends Api> extends DatasourceManagerCommon { // DATASOURCE diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/DatasourceDbEntry.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/DatasourceDbEntry.java index 32d72a77..8dc269e1 100755 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/DatasourceDbEntry.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/DatasourceDbEntry.java @@ -16,7 +16,7 @@ import eu.dnetlib.enabling.datasources.common.Datasource; @DynamicUpdate @SelectBeforeUpdate @Table(name = "dsm_services") -public class DatasourceDbEntry extends Datasource { +public class DatasourceDbEntry extends Datasource { @Transient private String openaireId; diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/PidSystemDbEntry.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/PidSystemDbEntry.java new file mode 100644 index 00000000..42cee00f --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/PidSystemDbEntry.java @@ -0,0 +1,57 @@ +package eu.dnetlib.openaire.dsm.domain.db; + +import javax.persistence.EmbeddedId; +import javax.persistence.Entity; +import javax.persistence.Table; + +import org.hibernate.annotations.DynamicUpdate; +import org.hibernate.annotations.SelectBeforeUpdate; + +import com.fasterxml.jackson.annotation.JsonIgnore; + +import eu.dnetlib.enabling.datasources.common.PidSystem; + +/** + * Created by claudio on 13/04/2017. + */ + +@Entity +@DynamicUpdate +@SelectBeforeUpdate +@Table(name = "dsm_pid_systems") +public class PidSystemDbEntry implements PidSystem { + + @EmbeddedId + protected PidSystemKeyDbEntry id; + + public PidSystem setId(final PidSystemKeyDbEntry id) { + this.id = id; + return this; + } + + @JsonIgnore + public PidSystemKeyDbEntry getId() { + return id; + } + + @Override + public String getType() { + return id.getType(); + } + + @Override + public void setType(final String type) { + id.setType(type); + } + + @Override + public String getScheme() { + return id.getScheme(); + } + + @Override + public void setScheme(final String scheme) { + id.setScheme(scheme); + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/PidSystemKeyDbEntry.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/PidSystemKeyDbEntry.java new file mode 100644 index 00000000..995d5685 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/PidSystemKeyDbEntry.java @@ -0,0 +1,62 @@ +package eu.dnetlib.openaire.dsm.domain.db; + +import java.io.Serializable; +import java.util.Objects; + +import javax.persistence.Embeddable; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +import eu.dnetlib.enabling.datasources.common.PidSystem; + +@Embeddable +@JsonIgnoreProperties(ignoreUnknown = true) +public class PidSystemKeyDbEntry implements PidSystem, Serializable { + + private static final long serialVersionUID = 1L; + + private String service; + private String type; + private String scheme; + + public String getService() { + return service; + } + + public void setService(final String service) { + this.service = service; + } + + @Override + public String getType() { + return type; + } + + @Override + public void setType(final String type) { + this.type = type; + } + + @Override + public String getScheme() { + return scheme; + } + + @Override + public void setScheme(final String scheme) { + this.scheme = scheme; + } + + @Override + public int hashCode() { + return Objects.hash(scheme, service, type); + } + + @Override + public boolean equals(final Object obj) { + if (this == obj) { return true; } + if (!(obj instanceof PidSystemKeyDbEntry)) { return false; } + final PidSystemKeyDbEntry other = (PidSystemKeyDbEntry) obj; + return Objects.equals(scheme, other.scheme) && Objects.equals(service, other.service) && Objects.equals(type, other.type); + } +} -- 2.17.1 From 7078467e9711c4ca2af59c67f16fbf90be94912b Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Thu, 7 Apr 2022 08:32:48 +0200 Subject: [PATCH 15/19] added status field to response --- .../openaire/dsm/domain/DatasourceDetails.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetails.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetails.java index dd4967f2..f03f98bf 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetails.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetails.java @@ -120,6 +120,9 @@ public class DatasourceDetails extends DatasourceIgnoredProperties { @ApiModelProperty(position = 27) private Set identities; + @ApiModelProperty(position = 32) + private String status; + public String getId() { return id; } @@ -399,4 +402,13 @@ public class DatasourceDetails extends DatasourceIgnoredProperties { return this; } + public String getStatus() { + return status; + } + + public DatasourceDetails setStatus(final String status) { + this.status = status; + return this; + } + } -- 2.17.1 From 44e170946f432b211a5daa043dd8dd2d3ada05ae Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Tue, 12 Apr 2022 13:05:57 +0200 Subject: [PATCH 16/19] readded typology field for compatibility --- .../openaire/dsm/dao/utils/DsmMappingUtils.java | 1 + .../openaire/dsm/domain/DatasourceDetails.java | 14 ++++++++++++++ .../dsm/domain/DatasourceSnippetExtended.java | 14 ++++++++++++++ .../openaire/dsm/domain/db/DatasourceDbEntry.java | 15 +++++++++++++++ 4 files changed, 44 insertions(+) diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/utils/DsmMappingUtils.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/utils/DsmMappingUtils.java index 25f8ee99..67a8630c 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/utils/DsmMappingUtils.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/utils/DsmMappingUtils.java @@ -57,6 +57,7 @@ public class DsmMappingUtils { if (d.getOrganizations() != null) { ds.setOrganizations(d.getOrganizations().stream().map(DsmMappingUtils::asOrganizationDetail).collect(Collectors.toSet())); } + ds.setTypology(d.getTypology()); return ds; } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetails.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetails.java index f03f98bf..a5d27048 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetails.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetails.java @@ -123,6 +123,10 @@ public class DatasourceDetails extends DatasourceIgnoredProperties { @ApiModelProperty(position = 32) private String status; + @Deprecated + @ApiModelProperty(position = 32) + private String typology; + public String getId() { return id; } @@ -411,4 +415,14 @@ public class DatasourceDetails extends DatasourceIgnoredProperties { return this; } + @Deprecated + public String getTypology() { + return typology; + } + + @Deprecated + public void setTypology(final String typology) { + this.typology = typology; + } + } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceSnippetExtended.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceSnippetExtended.java index 9c544658..168a249f 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceSnippetExtended.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceSnippetExtended.java @@ -58,6 +58,10 @@ public class DatasourceSnippetExtended { @ApiModelProperty(position = 13) private Set organizations; + @Deprecated + @ApiModelProperty(position = 14) + private String typology; + public String getId() { return id; } @@ -167,4 +171,14 @@ public class DatasourceSnippetExtended { this.organizations = organizations; } + @Deprecated + public String getTypology() { + return typology; + } + + @Deprecated + public void setTypology(final String typology) { + this.typology = typology; + } + } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/DatasourceDbEntry.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/DatasourceDbEntry.java index 8dc269e1..fad3e190 100755 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/DatasourceDbEntry.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/DatasourceDbEntry.java @@ -1,5 +1,6 @@ package eu.dnetlib.openaire.dsm.domain.db; +import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.Table; import javax.persistence.Transient; @@ -21,6 +22,10 @@ public class DatasourceDbEntry extends Datasource Date: Wed, 13 Apr 2022 09:43:12 +0200 Subject: [PATCH 17/19] another fix for _typology_to_remove_ --- .../dsm/dao/utils/DsmMappingUtils.java | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/utils/DsmMappingUtils.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/utils/DsmMappingUtils.java index 67a8630c..8d0bf806 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/utils/DsmMappingUtils.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/utils/DsmMappingUtils.java @@ -95,9 +95,50 @@ public class DsmMappingUtils { o.setCollectedfrom(dbe.getCollectedfrom()); }); } + + _fix_typology(dbe, d); + return dbe; } + @Deprecated + private static void _fix_typology(final DatasourceDbEntry dbe, final DatasourceDetails d) { + if (StringUtils.isNotBlank(d.getTypology()) && StringUtils.isBlank(d.getEoscDatasourceType())) { + // THE ORDER IS IMPORTANT: DO NOT CHANGE IT + if (d.getTypology().startsWith("crissystem")) { + dbe.setEoscDatasourceType("CRIS system"); + } else if (d.getTypology().startsWith("entityregistry")) { + dbe.setEoscDatasourceType("Registry"); + } else if (d.getTypology().startsWith("pubscatalogue") || d.getTypology().equals("websource")) { + dbe.setEoscDatasourceType("Catalogue"); + } else if (d.getTypology().contains("journal")) { + dbe.setEoscDatasourceType("Journal archive"); + } else if (d.getTypology().startsWith("aggregator")) { + dbe.setEoscDatasourceType("Aggregator"); + } else if (d.getTypology().contains("repository")) { + dbe.setEoscDatasourceType("Repository"); + } else { + dbe.setEoscDatasourceType("Aggregator"); + } + } else if (StringUtils.isBlank(d.getTypology()) && StringUtils.isNotBlank(d.getEoscDatasourceType())) { + if (dbe.getEoscDatasourceType().equals("CRIS system")) { + dbe.setTypology("crissystem"); + } else if (dbe.getEoscDatasourceType().equals("Registry")) { + dbe.setTypology("entityregistry"); + } else if (dbe.getEoscDatasourceType().equals("Catalogue")) { + dbe.setTypology("pubscatalogue::unknown"); + } else if (dbe.getEoscDatasourceType().equals("Journal archive")) { + dbe.setTypology("pubsrepository::journal"); + } else if (dbe.getEoscDatasourceType().equals("Aggregator")) { + dbe.setTypology("aggregator"); + } else if (dbe.getEoscDatasourceType().equals("Repository")) { + dbe.setTypology("pubsrepository::unknown"); + } else { + dbe.setTypology("aggregator"); + } + } + } + public static DatasourceDbEntry asDbEntry(final DatasourceDetailsUpdate d) { return _convert(d, DatasourceDbEntry.class); } -- 2.17.1 From 892963fb688ec7ca7a11b3d208b5dbff1ee72ed8 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Wed, 20 Apr 2022 12:11:09 +0200 Subject: [PATCH 18/19] pom version --- apps/dnet-exporter-api/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/dnet-exporter-api/pom.xml b/apps/dnet-exporter-api/pom.xml index f1cb0be9..0a5a8369 100644 --- a/apps/dnet-exporter-api/pom.xml +++ b/apps/dnet-exporter-api/pom.xml @@ -3,7 +3,7 @@ eu.dnetlib.dhp apps - 3.2.4-SNAPSHOT + 3.2.6-SNAPSHOT ../ -- 2.17.1 From a15ececddd9dbcaf33c37057819f5c57c1f36029 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Fri, 22 Apr 2022 10:25:00 +0200 Subject: [PATCH 19/19] added field lastConsentTermsOfUseDate --- apps/dnet-exporter-api/pom.xml | 2 +- .../dsm/dao/utils/DsmMappingUtils.java | 1 + .../dsm/domain/DatasourceDetails.java | 17 ++++++++++++-- .../dsm/domain/DatasourceDetailsUpdate.java | 23 +++++++++++++++---- .../dsm/domain/DatasourceSnippetExtended.java | 22 ++++++++++++++---- 5 files changed, 54 insertions(+), 11 deletions(-) diff --git a/apps/dnet-exporter-api/pom.xml b/apps/dnet-exporter-api/pom.xml index 0a5a8369..122aed3d 100644 --- a/apps/dnet-exporter-api/pom.xml +++ b/apps/dnet-exporter-api/pom.xml @@ -123,7 +123,7 @@ eu.dnetlib dnet-datasource-manager-common - 1.2.0-EOSC-SNAPSHOT + [2.0.1,3.0.0) diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/utils/DsmMappingUtils.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/utils/DsmMappingUtils.java index 8d0bf806..86ce80cb 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/utils/DsmMappingUtils.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/utils/DsmMappingUtils.java @@ -53,6 +53,7 @@ public class DsmMappingUtils { ds.setDescription(d.getDescription()); ds.setConsentTermsOfUse(d.getConsentTermsOfUse()); ds.setConsentTermsOfUseDate(d.getConsentTermsOfUseDate()); + ds.setLastConsentTermsOfUseDate(d.getLastConsentTermsOfUseDate()); ds.setFullTextDownload(d.getFullTextDownload()); if (d.getOrganizations() != null) { ds.setOrganizations(d.getOrganizations().stream().map(DsmMappingUtils::asOrganizationDetail).collect(Collectors.toSet())); diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetails.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetails.java index a5d27048..5285fb6e 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetails.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetails.java @@ -114,6 +114,9 @@ public class DatasourceDetails extends DatasourceIgnoredProperties { @ApiModelProperty(position = 30) private Date consentTermsOfUseDate; + @ApiModelProperty(position = 31) + private Date lastConsentTermsOfUseDate; + @ApiModelProperty(position = 26) private Set organizations; @@ -124,7 +127,7 @@ public class DatasourceDetails extends DatasourceIgnoredProperties { private String status; @Deprecated - @ApiModelProperty(position = 32) + @ApiModelProperty(position = 33) private String typology; public String getId() { @@ -421,8 +424,18 @@ public class DatasourceDetails extends DatasourceIgnoredProperties { } @Deprecated - public void setTypology(final String typology) { + public DatasourceDetails setTypology(final String typology) { this.typology = typology; + return this; + } + + public Date getLastConsentTermsOfUseDate() { + return lastConsentTermsOfUseDate; + } + + public DatasourceDetails setLastConsentTermsOfUseDate(final Date lastConsentTermsOfUseDate) { + this.lastConsentTermsOfUseDate = lastConsentTermsOfUseDate; + return this; } } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetailsUpdate.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetailsUpdate.java index 06c8ce5a..311773a9 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetailsUpdate.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetailsUpdate.java @@ -75,7 +75,10 @@ public class DatasourceDetailsUpdate { @ApiModelProperty(position = 29) private Date consentTermsOfUseDate; - @ApiModelProperty(position = 30) + @ApiModelProperty(position = 29) + private Date lastConsentTermsOfUseDate; + + @ApiModelProperty(position = 31) private Boolean fullTextDownload; public String getId() { @@ -217,24 +220,36 @@ public class DatasourceDetailsUpdate { return consentTermsOfUse; } - public void setConsentTermsOfUse(final Boolean consentTermsOfUse) { + public DatasourceDetailsUpdate setConsentTermsOfUse(final Boolean consentTermsOfUse) { this.consentTermsOfUse = consentTermsOfUse; + return this; } public Date getConsentTermsOfUseDate() { return consentTermsOfUseDate; } - public void setConsentTermsOfUseDate(final Date consentTermsOfUseDate) { + public DatasourceDetailsUpdate setConsentTermsOfUseDate(final Date consentTermsOfUseDate) { this.consentTermsOfUseDate = consentTermsOfUseDate; + return this; } public Boolean getFullTextDownload() { return fullTextDownload; } - public void setFullTextDownload(final Boolean fullTextDownload) { + public DatasourceDetailsUpdate setFullTextDownload(final Boolean fullTextDownload) { this.fullTextDownload = fullTextDownload; + return this; + } + + public Date getLastConsentTermsOfUseDate() { + return lastConsentTermsOfUseDate; + } + + public DatasourceDetailsUpdate setLastConsentTermsOfUseDate(final Date lastConsentTermsOfUseDate) { + this.lastConsentTermsOfUseDate = lastConsentTermsOfUseDate; + return this; } } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceSnippetExtended.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceSnippetExtended.java index 168a249f..69dcab91 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceSnippetExtended.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceSnippetExtended.java @@ -53,13 +53,16 @@ public class DatasourceSnippetExtended { private Date consentTermsOfUseDate; @ApiModelProperty(position = 12) - private Boolean fullTextDownload; + private Date lastConsentTermsOfUseDate; @ApiModelProperty(position = 13) + private Boolean fullTextDownload; + + @ApiModelProperty(position = 14) private Set organizations; @Deprecated - @ApiModelProperty(position = 14) + @ApiModelProperty(position = 15) private String typology; public String getId() { @@ -167,8 +170,9 @@ public class DatasourceSnippetExtended { return organizations; } - public void setOrganizations(final Set organizations) { + public DatasourceSnippetExtended setOrganizations(final Set organizations) { this.organizations = organizations; + return this; } @Deprecated @@ -177,8 +181,18 @@ public class DatasourceSnippetExtended { } @Deprecated - public void setTypology(final String typology) { + public DatasourceSnippetExtended setTypology(final String typology) { this.typology = typology; + return this; + } + + public Date getLastConsentTermsOfUseDate() { + return lastConsentTermsOfUseDate; + } + + public DatasourceSnippetExtended setLastConsentTermsOfUseDate(final Date lastConsentTermsOfUseDate) { + this.lastConsentTermsOfUseDate = lastConsentTermsOfUseDate; + return this; } } -- 2.17.1