diff --git a/.env.example b/.env.example index 9c2399d..11dd700 100644 --- a/.env.example +++ b/.env.example @@ -5,21 +5,28 @@ APP_DIR=/srv/app # Host Ports CKAN_PORT_HOST=5000 -NGINX_PORT_HOST=81 -NGINX_SSLPORT_HOST=8443 -APACHE_PORT_HOST=81 +# Common proxy server for Apache or NGINX. +# Change all the PROXY_SERVER_PORT_HOST to PROXY_SERVER_HTTPS_PORT_HOST if you can use HTTPS instead of HTTP (Only for NGINX and not development compose) +PROXY_SERVER_PORT_HOST=81 +PROXY_SERVER_HTTPS_PORT_HOST=8443 +NGINX_PORT_HOST=${PROXY_SERVER_PORT_HOST} +NGINX_SSLPORT_HOST=${PROXY_SERVER_HTTPS_PORT_HOST} +APACHE_PORT_HOST=${PROXY_SERVER_PORT_HOST} PYCSW_PORT_HOST=8000 # Solr SOLR_IMAGE_VERSION=2.9-solr9-spatial SOLR_PORT=8983 -CKAN_SOLR_URL=http://solr:${SOLR_PORT}/solr/ckan -TEST_CKAN_SOLR_URL=http://solr:${SOLR_PORT}/solr/ckan +SOLR_CKAN_DATABASE=ckan +CKAN_SOLR_URL=http://solr:${SOLR_PORT}/solr/${SOLR_CKAN_DATABASE} +TEST_CKAN_SOLR_URL=http://solr:${SOLR_PORT}/solr/${SOLR_CKAN_DATABASE} # Redis REDIS_VERSION=7-alpine -CKAN_REDIS_URL=redis://redis:6379/1 -TEST_CKAN_REDIS_URL=redis://redis:6379/1 +REDIS_PORT=6379 +REDIS_CKAN_DATABASE=1 +CKAN_REDIS_URL=redis://redis:${REDIS_PORT}/${REDIS_CKAN_DATABASE} +TEST_CKAN_REDIS_URL=redis://redis:${REDIS_PORT}/${REDIS_CKAN_DATABASE} # NGINX NGINX_PORT=80 @@ -32,15 +39,16 @@ APACHE_PORT=80 APACHE_LOG_DIR=/var/log/apache #NGINX/APACHE -## Check CKAN__ROOT_PATH and CKANEXT__DCAT__BASE_URI and CKANEXT__SCHEMINGDCAT_GEOMETADATA_BASE_URI. If you don't need to use domain locations, it is better to use the nginx configuration. Leave blank or use the root `/`. +## Check CKAN__ROOT_PATH and CKANEXT__DCAT__BASE_URI and CKANEXT__SCHEMINGDCAT_GEOMETADATA_BASE_URI. If you don't need to use domain locations, it is better to use the nginx configuration. Leave blank or use the root `/`. If you dont need PROXY_SERVER_PORT_HOST (e.g. is 80), edit the PROXY_SERVER_URL and remove :${PROXY_SERVER_PORT_HOST} PROXY_SERVER_NAME=localhost +PROXY_SERVER_URL=http://${PROXY_SERVER_NAME}:${PROXY_SERVER_PORT_HOST} PROXY_CKAN_LOCATION=/catalog PROXY_PYCSW_LOCATION=/csw # pycsw PYCSW_PORT=8000 -CKAN_URL=http://localhost:81/catalog -PYCSW_URL=http://localhost:81/csw +CKAN_URL=${PROXY_SERVER_URL}${PROXY_CKAN_LOCATION} +PYCSW_URL=${PROXY_SERVER_URL}${PROXY_PYCSW_LOCATION} # SCHEMAS: ckan2pycsw/model/dataset.py - Dataset type PYCSW_CKAN_SCHEMA=iso19139_geodcatap PYCSW_OUPUT_SCHEMA=iso19139_inspire @@ -63,14 +71,16 @@ CKAN_DB=ckandb DATASTORE_READONLY_USER=datastore_ro DATASTORE_READONLY_PASSWORD=datastore DATASTORE_DB=datastore -CKAN_SQLALCHEMY_URL=postgresql://ckandbuser:ckandbpassword@db/ckandb -CKAN_DATASTORE_WRITE_URL=postgresql://ckandbuser:ckandbpassword@db/datastore -CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore +CKAN_SQLALCHEMY_URL=postgresql://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@${POSTGRES_HOST}/${CKAN_DB} +CKAN_DATASTORE_WRITE_URL=postgresql://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@${POSTGRES_HOST}/${DATASTORE_DB} +CKAN_DATASTORE_READ_URL=postgresql://${DATASTORE_READONLY_USER}:${DATASTORE_READONLY_PASSWORD}@${POSTGRES_HOST}/${DATASTORE_DB} # Test database connections -TEST_CKAN_SQLALCHEMY_URL=postgres://ckan:ckan@db/ckan_test -TEST_CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore_test -TEST_CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore_test +CKAN_TEST_DB=ckan_test +DATASTORE_TEST_DB=datastore_test +TEST_CKAN_SQLALCHEMY_URL=postgres://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@${POSTGRES_HOST}/${CKAN_TEST_DB} +TEST_CKAN_DATASTORE_WRITE_URL=postgresql://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@${POSTGRES_HOST}/${DATASTORE_TEST_DB} +TEST_CKAN_DATASTORE_READ_URL=postgresql://${DATASTORE_READONLY_USER}:${DATASTORE_READONLY_PASSWORD}@${POSTGRES_HOST}/${DATASTORE_TEST_DB} # Dev settings USE_HTTPS_FOR_DEV=false @@ -81,11 +91,12 @@ CKAN_DEV_COMPOSE_SERVICE=ckan-dev CKAN_VERSION=2.9.11 CKAN_SITE_ID=default # CKAN_SITE_URL = http:/ or https:/ + PROXY_SERVER_NAME. Optionally the APACHE_HOST_PORT if different from 80 -CKAN_SITE_URL=http://localhost:81 -CKAN__ROOT_PATH=/catalog/{{LANG}} -CKAN_PORT=5000 -CKAN__FAVICON=/catalog/base/images/ckan.ico +CKAN_SITE_URL=${PROXY_SERVER_URL} +CKAN__ROOT_PATH=${PROXY_CKAN_LOCATION}/{{LANG}} +CKAN__FAVICON=${PROXY_CKAN_LOCATION}/base/images/ckan.ico CKAN__SITE_LOGO=/images/default/ckan-logo.png +# Custom licenses that are aligned with DCAT-AP +CKAN___LICENSES_GROUP_URL=https://raw.githubusercontent.com/mjanez/ckanext-schemingdcat/main/ckanext/schemingdcat/public/static/licenses.json CKAN___BEAKER__SESSION__SECRET=CHANGE_ME # See https://docs.ckan.org/en/latest/maintaining/configuration.html#api-token-settings CKAN___API_TOKEN__JWT__ENCODE__SECRET=string:CHANGE_ME @@ -99,7 +110,7 @@ CKAN_SMTP_SERVER=smtp.corporateict.domain:25 CKAN_SMTP_STARTTLS=True CKAN_SMTP_USER=user CKAN_SMTP_PASSWORD=pass -CKAN_SMTP_MAIL_FROM=ckan@localhost +CKAN_SMTP_MAIL_FROM=ckan@${PROXY_SERVER_NAME} ## Customize which text formats the text_view plugin will show CKAN__PREVIEW__JSON_FORMATS="json jsonld" # html htm rdf+xml owl+xml xml n3 n-triples turtle plain atom csv tsv rss txt json @@ -127,24 +138,24 @@ CKAN__LOCALE_ORDER="en es pt_BR ja it cs_CZ ca fr el sv sr sr@latin no sk fi ru CKAN__LOCALES_OFFERED="en es pt_BR ja it cs_CZ ca fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv" # Extensions -CKAN__PLUGINS="envvars stats image_view text_view recline_view webpage_view resourcedictionary datastore xloader spatial_metadata spatial_query spatial_harvest_metadata_api csw_harvester waf_harvester doc_harvester resource_proxy geo_view geojson_view wmts_view shp_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface schemingdcat_datasets schemingdcat_groups schemingdcat_organizations schemingdcat schemingdcat_ckan_harvester schemingdcat_xls_harvester harvest pdf_view pages fluent" +CKAN__PLUGINS="stats image_view text_view recline_view webpage_view resourcedictionary datastore xloader spatial_metadata spatial_query spatial_harvest_metadata_api csw_harvester waf_harvester doc_harvester resource_proxy geo_view geojson_view wmts_view shp_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface schemingdcat_datasets schemingdcat_groups schemingdcat_organizations schemingdcat schemingdcat_ckan_harvester schemingdcat_xls_harvester harvest pdf_view pages fluent envvars" # ckanext-harvest CKAN__HARVEST__MQ__TYPE=redis CKAN__HARVEST__MQ__HOSTNAME=redis -CKAN__HARVEST__MQ__PORT=6379 -CKAN__HARVEST__MQ__REDIS_DB=1 +CKAN__HARVEST__MQ__PORT=${REDIS_PORT} +CKAN__HARVEST__MQ__REDIS_DB=${REDIS_CKAN_DATABASE} # Clean-up mechanism for the harvest log table. The default is 30 days. CKAN__HARVEST__LOG_TIMEFRAME=40 # ckanext-xloader CKANEXT__XLOADER__API_TOKEN=api_token -CKANEXT__XLOADER__JOBS__DB_URI=postgresql://ckandbuser:ckandbpassword@db/ckandb +CKANEXT__XLOADER__JOBS__DB_URI=postgresql://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@${POSTGRES_HOST}/${CKAN_DB} # ckanext-dcat CKANEXT__DCAT__BASE_URI=${CKAN_URL} -# Default profile(s). Instead of this envvar, it's possible to specify all the profile(s) availables to be used for serialization using the profiles parameter: http://localhost:5000/catalog.xml?profiles=euro_dcat_ap,spain_dcat -CKANEXT__DCAT__RDF_PROFILES='euro_dcat_ap_2' +# Default profile(s). Instead of this envvar, it's possible to specify all the profile(s) availables to be used for serialization using the profiles parameter: http://localhost:5000/catalog.xml?profiles=eu_dcat_ap_2,es_dcat +CKANEXT__DCAT__RDF_PROFILES='eu_dcat_ap_2 eu_dcat_ap_scheming' # The custom endpoint **must** start with a forward slash (`/`) and contain the `{_format}` placeholder. The endpoint is added to the CKAN_SITE_URL and CKAN__ROOT_PATH, example: http://localhost:5000/catalog/catalog.rdf CKANEXT__DCAT__DEFAULT_CATALOG_ENDPOINT='/catalog.{_format}' @@ -165,9 +176,9 @@ CKANEXT__GEOVIEW__SHP_VIEWER__ENCODING=UTF-8 ## CSW Endpoint for spatial metadata CKANEXT__SCHEMINGDCAT_GEOMETADATA_BASE_URI=${PYCSW_URL} ## Scheming: setup_scheming.sh -CKANEXT__SCHEMINGDCAT_DATASET_SCHEMA="ckanext.schemingdcat:schemas/geodcatap/geodcatap_dataset.yaml" -CKANEXT__SCHEMINGDCAT_GROUP_SCHEMAS="ckanext.schemingdcat:schemas/geodcatap/geodcatap_group.json" -CKANEXT__SCHEMINGDCAT_ORGANIZATION_SCHEMAS="ckanext.schemingdcat:schemas/geodcatap/geodcatap_org.json" +CKANEXT__SCHEMINGDCAT_DATASET_SCHEMA="ckanext.schemingdcat:schemas/geodcat_ap/eu_geodcat_ap_2.yaml ckanext.schemingdcat:schemas/resources/dcat_3_document.yaml" +CKANEXT__SCHEMINGDCAT_GROUP_SCHEMAS="ckanext.schemingdcat:schemas/geodcat_ap/eu_geodcat_ap_group.json" +CKANEXT__SCHEMINGDCAT_ORGANIZATION_SCHEMAS="ckanext.schemingdcat:schemas/geodcat_ap/eu_geodcat_ap_org.json" CKANEXT__SCHEMINGDCAT_PRESETS="ckanext.schemingdcat:schemas/default_presets.json ckanext.fluent:presets.json" ## Facets: setup_scheming.sh CKANEXT__SCHEMINGDCAT_FACET_LIST="dataset_scope theme groups theme_eu dcat_type groups publisher_name publisher_type spatial_uri owner_org res_format frequency tags tag_uri conforms_to" @@ -177,6 +188,7 @@ CKANEXT__SCHEMINGDCAT_DEFAULT_PACKAGE_ITEM_ICON="theme" CKANEXT__SCHEMINGDCAT_DEFAULT_PACKAGE_ITEM_SHOW_SPATIAL=True CKANEXT__SCHEMINGDCAT_SHOW_METADATA_TEMPLATES_TOOLBAR=False CKANEXT__METADATA_TEMPLATES_SEARCH_IDENTIFIER="schemingdcat_xls-template" +CKANEXT__SCHEMINGDCAT_ENDPOINTS_YAML="endpoints.yaml" # ckanext-pages CKANEXT__PAGES__ALOW_HTML=False @@ -184,8 +196,4 @@ CKANEXT__PAGES__ORGANIZATION=True CKANEXT__PAGES__GROUP=True CKANEXT__PAGES__ABOUT_MENU=False CKANEXT__PAGES__GROUP_MENU=True -CKANEXT__PAGES__ORGANIZATION_MENU=True - -# WIP: ckanext-sparql_interface -CKANEXT__SPARQL__ENDPOINT_URL=https://dbpedia.org/sparql -CKANEXT__SPARQL__HIDE_ENDPOINT_URL=False \ No newline at end of file +CKANEXT__PAGES__ORGANIZATION_MENU=True \ No newline at end of file diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 07ed077..eead1f3 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -14,9 +14,9 @@ on: env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} - TAG: ghcr.io/${{ github.repository }}:${{ github.head_ref }} CONTEXT: . BRANCH: ${{ github.head_ref }} + VERSION: ${{ github.head_ref }} DOCKERFILE_PATH: /ckan DOCKERFILE: Dockerfile @@ -43,6 +43,10 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract tag version from branch name + id: extract_tag_version + run: echo "VERSION=$(echo ${{ github.head_ref }} | sed 's/^ckan-//')" >> $GITHUB_ENV + - name: Extract Docker metadata id: meta uses: docker/metadata-action@v5 @@ -50,14 +54,18 @@ jobs: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} labels: | org.opencontainers.image.documentation=https://github.com/${{ github.repository }}/blob/${{ env.BRANCH }}/README.md - org.opencontainers.image.version=${{ env.BRANCH }} + org.opencontainers.image.version=${{ env.VERSION }} + annotations: | + org.opencontainers.image.description=This image contains CKAN based on a Docker Compose deployment. The container includes CKAN along with its dependencies and configurations for spatial data support. + org.opencontainers.image.source=https://github.com/${{ github.repository }} - name: Build and push - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: push: true - tags: ${{ env.TAG }} + tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.VERSION }} labels: ${{ steps.meta.outputs.labels }} + annotations: ${{ steps.meta.outputs.annotations }} context: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }} file: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}/${{ env.DOCKERFILE }} @@ -68,14 +76,14 @@ jobs: no-fail: true - name: Run Trivy container image vulnerability scanner - uses: aquasecurity/trivy-action@0.18.0 + uses: aquasecurity/trivy-action@0.24.0 with: - image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.BRANCH }} + image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.VERSION }} format: sarif output: trivy-results.sarif - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v2 + uses: github/codeql-action/upload-sarif@v3 if: always() with: sarif_file: trivy-results.sarif \ No newline at end of file diff --git a/.github/workflows/docker-manual.yml b/.github/workflows/docker-manual.yml index 9acbb46..44e2553 100644 --- a/.github/workflows/docker-manual.yml +++ b/.github/workflows/docker-manual.yml @@ -5,9 +5,9 @@ on: workflow_dispatch env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} - TAG: ghcr.io/${{ github.repository }}:${{ github.ref_name }} CONTEXT: . BRANCH: ${{ github.ref_name }} + VERSION: ${{ github.ref_name }} DOCKERFILE_PATH: /ckan DOCKERFILE: Dockerfile @@ -33,6 +33,10 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract tag version from branch name + id: extract_tag_version + run: echo "VERSION=$(echo ${{ github.head_ref }} | sed 's/^ckan-//')" >> $GITHUB_ENV + - name: Extract Docker metadata id: meta uses: docker/metadata-action@v5 @@ -40,14 +44,18 @@ jobs: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} labels: | org.opencontainers.image.documentation=https://github.com/${{ github.repository }}/blob/${{ env.BRANCH }}/README.md - org.opencontainers.image.version=${{ env.BRANCH }} + org.opencontainers.image.version=${{ env.VERSION }} + annotations: | + org.opencontainers.image.description=This image contains CKAN based on a Docker Compose deployment. The container includes CKAN along with its dependencies and configurations for spatial data support. + org.opencontainers.image.source=https://github.com/${{ github.repository }} - name: Build and push - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: push: true - tags: ${{ env.TAG }} + tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.VERSION }} labels: ${{ steps.meta.outputs.labels }} + annotations: ${{ steps.meta.outputs.annotations }} context: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }} file: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}/${{ env.DOCKERFILE }} @@ -58,14 +66,14 @@ jobs: no-fail: true - name: Run Trivy container image vulnerability scanner - uses: aquasecurity/trivy-action@0.18.0 + uses: aquasecurity/trivy-action@0.24.0 with: - image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.BRANCH }} + image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.VERSION }} format: sarif output: trivy-results.sarif - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v2 + uses: github/codeql-action/upload-sarif@v3 if: always() with: sarif_file: trivy-results.sarif \ No newline at end of file diff --git a/.github/workflows/docker-master.yml b/.github/workflows/docker-master.yml index f798eaf..b0eab55 100644 --- a/.github/workflows/docker-master.yml +++ b/.github/workflows/docker-master.yml @@ -9,6 +9,8 @@ env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} CONTEXT: . + BRANCH: master + VERSION: master DOCKERFILE_PATH: /ckan DOCKERFILE: Dockerfile @@ -29,10 +31,6 @@ jobs: with: fetch-depth: 0 - - name: Get highest ckan branch excluding -dev - id: getbranch - run: echo "VERSION=$(git branch -r | grep -o 'ckan-[0-9]*\.[0-9]*\.[0-9]*[^-dev]$' | sort -V | tail -n 1)" >> $GITHUB_ENV - - name: Login to registry uses: docker/login-action@v3 with: @@ -40,21 +38,29 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Get highest ckan branch excluding -dev + id: getbranch + run: echo "VERSION=$(git branch -r | grep -o 'ckan-[0-9]*\.[0-9]*\.[0-9]*[^-dev]$' | sort -V | tail -n 1)" >> $GITHUB_ENV + - name: Extract Docker metadata id: meta uses: docker/metadata-action@v5 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} labels: | - org.opencontainers.image.documentation=https://github.com/${{ github.repository }}/blob/master/README.md + org.opencontainers.image.documentation=https://github.com/${{ github.repository }}/blob/${{ env.BRANCH }}/README.md org.opencontainers.image.version=${{ env.VERSION }} + annotations: | + org.opencontainers.image.description=This image contains CKAN based on a Docker Compose deployment. The container includes CKAN along with its dependencies and configurations for spatial data support. + org.opencontainers.image.source=https://github.com/${{ github.repository }} - name: Build and push - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: push: true tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.VERSION }} labels: ${{ steps.meta.outputs.labels }} + annotations: ${{ steps.meta.outputs.annotations }} context: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }} file: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}/${{ env.DOCKERFILE }} @@ -65,14 +71,14 @@ jobs: no-fail: true - name: Run Trivy container image vulnerability scanner - uses: aquasecurity/trivy-action@0.18.0 + uses: aquasecurity/trivy-action@0.24.0 with: image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.VERSION }} format: sarif output: trivy-results.sarif - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v2 + uses: github/codeql-action/upload-sarif@v3 if: always() with: sarif_file: trivy-results.sarif \ No newline at end of file diff --git a/.github/workflows/docker-pr.yml b/.github/workflows/docker-pr.yml index fea16e4..ccd46a7 100644 --- a/.github/workflows/docker-pr.yml +++ b/.github/workflows/docker-pr.yml @@ -30,11 +30,11 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Checkout + - name: Check out code uses: actions/checkout@v4 - name: NGINX build - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: ./nginx file: ./nginx/Dockerfile @@ -42,7 +42,7 @@ jobs: tags: mjanez/ckan-docker-nginx:test-build-only - name: Apache HTTP Server build - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: ./apache file: ./apache/Dockerfile @@ -50,7 +50,7 @@ jobs: tags: mjanez/ckan-docker-apache:test-build-only - name: PostgreSQL build - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: ./postgresql file: ./postgresql/Dockerfile @@ -58,7 +58,7 @@ jobs: tags: mjanez/ckan-docker-postgresql:test-build-only - name: Solr build - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: ./solr file: ./solr/Dockerfile @@ -66,7 +66,7 @@ jobs: tags: mjanez/ckan-docker-solr:test-build-only - name: ckan-pycsw build - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v6 with: context: ./ckan-pycsw file: ./ckan-pycsw/Dockerfile diff --git a/.gitignore b/.gitignore index aad3592..94e4188 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,11 @@ src/* # environment .env .env.* -!.env.example pycsw.conf.template.* index.html.* + +# environment allowlist +!.env.example +!.env.es.example +!.env.localhost +samples/api/* diff --git a/README.md b/README.md index db2a77d..ac853f2 100644 --- a/README.md +++ b/README.md @@ -40,13 +40,11 @@ Contains Docker images for the different components of CKAN Cloud and a Docker c Available components: * CKAN custom multi-stage build with spatial capabilities from [ckan-docker-spatial](https://github.com/mjanez/ckan-docker-spatial)[^1], an image used as a base and built from the official CKAN repo. The following versions of CKAN are available: -| CKAN Version | Type | Docker tag | Notes | -| --- | --- | --- | --- | -| 2.9.8 | custom image | `ghcr.io/mjanez/ckan-spatial:ckan-2.9.8` | Stable version with CKAN 2.9.8 | -| 2.9.9 | custom image | `ghcr.io/mjanez/ckan-docker:ckan-2.9.9` | Stable version with CKAN 2.9.9 | -| 2.9.10 | custom image | `ghcr.io/mjanez/ckan-docker:ckan-2.9.10` | Stable version with CKAN 2.9.10 | -| 2.9.11 | custom image | `ghcr.io/mjanez/ckan-docker:ckan-2.9.11` | Stable version with CKAN 2.9.11 | -| 2.9.11 | latest custom image | `ghcr.io/mjanez/ckan-docker:master` | Latest `ckan-docker` image. | +| CKAN Version | Type | Base image | Docker tag | Notes | +| --- | --- | --- | --- | --- | +| 2.9.x | custom spatial image | `alpine:3.15` | `ghcr.io/mjanez/ckan-spatial:ckan-2.9.8`, `ghcr.io/mjanez/ckan-spatial:ckan-2.9.8`, `ghcr.io/mjanez/ckan-docker:ckan-2.9.9`, `ghcr.io/mjanez/ckan-docker:ckan-2.9.10`, `ghcr.io/mjanez/ckan-docker:ckan-2.9.11`, `ghcr.io/mjanez/ckan-docker:2.9.12` | Stable official versions of CKAN `2.9.8`, `2.9.10` and `2.9.11`, including a security backport: `2.9.12`. As of `2.9.12`, repo images are aligned with the [`ckan-docker-spatial`](https://github.com/mjanez/ckan-docker-spatial#pre-configured-ckan-docker-images) and [`ckan-docker-base](https://github.com/ckan/ckan-docker-base#pre-configured-ckan-docker-images)` tags. | +| 2.10.x | custom spatial image | `python:3.10-slim-bookworm` | `ghcr.io/mjanez/ckan-docker:2.10.5` | From `2.10` images only [Debian-based official Python images](https://hub.docker.com/_/python) rather than Alpine-based images will be provided. | +| 2.11.x | custom spatial image | `python:3.10-slim-bookworm` | `ghcr.io/mjanez/ckan-docker:2.11.0` | CKAN's latest official version. Only [Debian-based official Python images](https://hub.docker.com/_/python). | The non-CKAN images are as follows: * PostgreSQL: [Custom image](/postgresql/Dockerfile) based on official PostgreSQL image. Database files are stored in a named volume. @@ -271,7 +269,7 @@ The Docker image config files used to build your CKAN project are located in the * `Dockerfile`: this is based on `mjanez/ckan-base-spatial:`, a base image located in the [Github Package Registry](https://github.com/mjanez/ckan-docker/pkgs/container/ckan-base-spatial), that has CKAN installed along with all its dependencies, properly configured and running on [uWSGI](https://uwsgi-docs.readthedocs.io/en/latest/) (production setup) * `Dockerfile.dev`: this is based on `mjanez/ckan-base-spatial:-dev` also located located in the Github Package Registry, and extends `mjanez/ckan-base-spatial:` to include: - * Any extension cloned on the `./src` folder will be installed in the CKAN container when booting up Docker Compose (`docker compose up`). This includes installing any requirements listed in a `requirements.txt` (or `pip-requirements.txt`) file and running `python setup.py develop`. + * Any extension cloned on the `./src` folder will be installed in the CKAN container when booting up Docker Compose (`docker compose -f docker-compose.dev.yml up`). This includes installing any requirements listed in a `requirements.txt` (or `pip-requirements.txt`) file and running `python setup.py develop`. You can clone all the extensions explained in [`src/README`](/src/README.md) with their tag, reqs and autocrlf using a bash script. * CKAN is started running this: `/usr/bin/ckan -c /srv/app/ckan.ini run -H 0.0.0.0`. * Make sure to add the local plugins to the `CKAN__PLUGINS` env var in the `.env` file. @@ -435,9 +433,6 @@ You can now set breakpoints and remote debug your CKAN development instance usin Add these lines to the `ckan-dev` service in the docker compose.dev.yml file ```yaml -ports: - - "0.0.0.0:${CKAN_PORT}:5000" - stdin_open: true tty: true ``` diff --git a/ckan/Dockerfile b/ckan/Dockerfile index 893fa5a..9b96915 100644 --- a/ckan/Dockerfile +++ b/ckan/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/mjanez/ckan-base-spatial:ckan-2.9.11 +FROM ghcr.io/mjanez/ckan-spatial-base:2.9.12 LABEL maintainer="mnl.janez@gmail.com" # Set up environment variables @@ -16,13 +16,13 @@ COPY req_fixes req_fixes ## Harvest - v1.5.6 (Worker with supervisor) ## ## Geoview - v0.1.0 ## ## Spatial - v2.1.1 ## -## DCAT - v1.8.0-alpha (mjanez/GeoDCAT-AP/NTI-RISP extended version) ## +## DCAT - v1.8.0 (Latest stable version of ckanext-dcat with minor fixes) ## ## Scheming - release-3.0.0 ## ## Resource dictionary - v1.0.1 (mjanez/Fixed version) ## ## Pages - v0.5.2 ## ## PDFView - 0.0.8 ## ## Fluent - v1.0.1 (mjanez/Forked stable version) ## -## Scheming DCAT - v3.2.1 (mjanez/GeoDCAT-AP/NTI-RISP extended version) ## +## Scheming DCAT - v3.2.2 (mjanez/GeoDCAT-AP/NTI-RISP extended version) ## RUN echo ${TZ} > /etc/timezone && \ if ! [ /usr/share/zoneinfo/${TZ} -ef /etc/localtime ]; then cp /usr/share/zoneinfo/${TZ} /etc/localtime; fi && \ # Remove apk cache @@ -41,7 +41,7 @@ RUN echo ${TZ} > /etc/timezone && \ pip3 install --no-cache-dir -e git+https://github.com/ckan/ckanext-spatial.git@v2.1.1#egg=ckanext-spatial && \ pip3 install --no-cache-dir -r ${APP_DIR}/req_fixes/ckanext-spatial/requirements.txt && \ echo "mjanez/ckanext-dcat (GeoDCAT-AP extended version)" && \ - pip3 install --no-cache-dir -e git+https://github.com/mjanez/ckanext-dcat.git@v1.8.0-alpha#egg=ckanext-dcat && \ + pip3 install --no-cache-dir -e git+https://github.com/mjanez/ckanext-dcat.git@v1.8.0#egg=ckanext-dcat && \ pip3 install --no-cache-dir -r ${APP_DIR}/src/ckanext-dcat/requirements.txt && \ echo "ckan/ckanext-scheming" && \ pip3 install --no-cache-dir -e git+https://github.com/ckan/ckanext-scheming.git@release-3.0.0#egg=ckanext-scheming && \ @@ -54,7 +54,7 @@ RUN echo ${TZ} > /etc/timezone && \ echo "mjanez/ckanext-fluent" && \ pip3 install --no-cache-dir -e git+https://github.com/mjanez/ckanext-fluent.git@v1.0.1#egg=ckanext-fluent && \ echo "mjanez/ckanext-schemingdcat" && \ - pip3 install --no-cache-dir -e git+https://github.com/mjanez/ckanext-schemingdcat.git@v3.2.1#egg=ckanext_schemingdcat && \ + pip3 install --no-cache-dir -e git+https://github.com/mjanez/ckanext-schemingdcat.git@v3.2.2#egg=ckanext_schemingdcat && \ pip3 install --no-cache-dir -r ${APP_DIR}/src/ckanext-schemingdcat/requirements.txt # Used to configure the container environment by setting environment variables, creating users, running initialization scripts, .etc diff --git a/ckan/Dockerfile.dev b/ckan/Dockerfile.dev index 21f3a8a..228902e 100644 --- a/ckan/Dockerfile.dev +++ b/ckan/Dockerfile.dev @@ -1,4 +1,4 @@ -FROM ghcr.io/mjanez/ckan-base-spatial:ckan-2.9.11-dev +FROM ghcr.io/mjanez/ckan-spatial-dev:2.9.12 LABEL maintainer="mnl.janez@gmail.com" # Set up environment variables diff --git a/ckan/Dockerfile.ghcr b/ckan/Dockerfile.ghcr index 03dc78a..4e1e5a0 100644 --- a/ckan/Dockerfile.ghcr +++ b/ckan/Dockerfile.ghcr @@ -1,4 +1,4 @@ -FROM ghcr.io/mjanez/ckan-docker:ckan-2.9.11 +FROM ghcr.io/mjanez/ckan-docker:2.9.12 LABEL maintainer="mnl.janez@gmail.com" # Set up environment variables diff --git a/ckan/docker-entrypoint.d/01_setup_ckanext_config.sh b/ckan/docker-entrypoint.d/01_setup_ckanext_config.sh new file mode 100644 index 0000000..8ba4079 --- /dev/null +++ b/ckan/docker-entrypoint.d/01_setup_ckanext_config.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +# ckanext-scheming/ckanext-schemingdcat: Update settings defined in the envvars (https://github.com/ckan/ckanext-pages#configuration) +echo "[docker-entrypoint.01_setup_ckanext_config] Clear index" +ckan -c $CKAN_INI search-index clear + +echo "[docker-entrypoint.01_setup_ckanext_config] Loading ckanext-scheming and ckanext-schemingdcat settings into ckan.ini" +ckan config-tool $CKAN_INI \ + "scheming.dataset_schemas=$CKANEXT__SCHEMINGDCAT_DATASET_SCHEMA" \ + "scheming.group_schemas=$CKANEXT__SCHEMINGDCAT_GROUP_SCHEMAS" \ + "scheming.organization_schemas=$CKANEXT__SCHEMINGDCAT_ORGANIZATION_SCHEMAS" \ + "scheming.presets=$CKANEXT__SCHEMINGDCAT_PRESETS" \ + "schemingdcat.facet_list=$CKANEXT__SCHEMINGDCAT_FACET_LIST" \ + "schemingdcat.organization_custom_facets=$CKANEXT__SCHEMINGDCAT_ORGANIZATION_CUSTOM_FACETS" \ + "schemingdcat.group_custom_facets=$CKANEXT__SCHEMINGDCAT_GROUP_CUSTOM_FACETS" \ + "schemingdcat.geometadata_base_uri=$CKANEXT__SCHEMINGDCAT_GEOMETADATA_BASE_URI" \ + "schemingdcat.default_package_item_icon=$CKANEXT__SCHEMINGDCAT_DEFAULT_PACKAGE_ITEM_ICON" \ + "schemingdcat.default_package_item_show_spatial=$CKANEXT__SCHEMINGDCAT_DEFAULT_PACKAGE_ITEM_SHOW_SPATIAL" \ + "schemingdcat.show_metadata_templates_toolbar=$CKANEXT__SCHEMINGDCAT_SHOW_METADATA_TEMPLATES_TOOLBAR" \ + "schemingdcat.metadata_templates_search_identifier=$CKANEXT__METADATA_TEMPLATES_SEARCH_IDENTIFIER" \ + "schemingdcat.endpoints_yaml=$CKANEXT__SCHEMINGDCAT_ENDPOINTS_YAML" \ + +# ckanext-dcat: Add settings to the CKAN config file +echo "[docker-entrypoint.01_setup_ckanext_config] Loading ckanext-dcat settings in the CKAN config file" +ckan config-tool $CKAN_INI \ + "ckanext.dcat.base_uri = $CKANEXT__DCAT__BASE_URI" \ + "ckanext.dcat.catalog_endpoint = $CKANEXT__DCAT__DEFAULT_CATALOG_ENDPOINT" \ + "ckanext.dcat.rdf.profiles = $CKANEXT__DCAT__RDF_PROFILES" + +# ckan previews: Add CKAN Resource views to the CKAN config file +echo "[docker-entrypoint.01_setup_ckanext_config] Loading resource views in the CKAN config file" +ckan config-tool $CKAN_INI \ + "ckan.views.default_views = $CKAN__VIEWS__DEFAULT_VIEWS" \ + "ckan.preview.json_formats = $CKAN__PREVIEW__JSON_FORMATS" \ + "ckan.preview.xml_formats = $CKAN__PREVIEW__XML_FORMATS" \ + "ckan.preview.text_formats = $CKAN__PREVIEW__TEXT_FORMATS" \ + "ckan.preview.loadable = $CKAN__PREVIEW__LOADABLE" + +# ckanext-geoview: Add geoviews CKAN config file +echo "[docker-entrypoint.01_setup_ckanext_config] Loading geoviews in the CKAN config file" +ckan config-tool $CKAN_INI \ + "ckanext.geoview.ol_viewer.formats = $CKANEXT__GEOVIEW__OL_VIEWER__FORMATS" \ + "ckanext.geoview.shp_viewer.srid = $CKANEXT__GEOVIEW__SHP_VIEWER__SRID" \ + "ckanext.geoview.shp_viewer.encoding = $CKANEXT__GEOVIEW__SHP_VIEWER__ENCODING" \ + "ckanext.geoview.geojson.max_file_size = $CKANEXT__GEOVIEW__GEOJSON__MAX_FILE_SIZE" + +# ckanext-pages: Add pages CKAN config file +echo "[docker-entrypoint.01_setup_ckanext_config] Loading pages config in the CKAN config file" +ckan config-tool $CKAN_INI \ + "ckan.pages.allow_html = $CKANEXT__PAGES__ALOW_HTML" \ + "ckanext.pages.organization = $CKANEXT__PAGES__ORGANIZATION" \ + "ckanext.pages.group = $CKANEXT__PAGES__GROUP" \ + "ckanext.pages.about_menu = $CKANEXT__PAGES__ABOUT_MENU" \ + "ckanext.pages.group_menu = $CKANEXT__PAGES__GROUP_MENU" \ + "ckanext.pages.organization_menu = $CKANEXT__PAGES__ORGANIZATION_MENU" + +# Rebuild index +echo "[docker-entrypoint.01_setup_ckanext_config] Rebuild index" +ckan -c $CKAN_INI search-index rebuild \ No newline at end of file diff --git a/ckan/docker-entrypoint.d/02_setup_scheming.sh b/ckan/docker-entrypoint.d/02_setup_scheming.sh deleted file mode 100644 index 83fc384..0000000 --- a/ckan/docker-entrypoint.d/02_setup_scheming.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -# Update ckanext-scheming and ckanext-schemingdcat settings defined in the env var -echo "[docker-entrypoint.02_setup_scheming] Clear index" -ckan -c $CKAN_INI search-index clear - -echo "[docker-entrypoint.02_setup_scheming] Loading ckanext-scheming and ckanext-schemingdcat settings into ckan.ini" -ckan config-tool $CKAN_INI \ - "scheming.dataset_schemas=$CKANEXT__SCHEMINGDCAT_DATASET_SCHEMA" \ - "scheming.group_schemas=$CKANEXT__SCHEMINGDCAT_GROUP_SCHEMAS" \ - "scheming.organization_schemas=$CKANEXT__SCHEMINGDCAT_ORGANIZATION_SCHEMAS" \ - "scheming.presets=$CKANEXT__SCHEMINGDCAT_PRESETS" \ - "schemingdcat.facet_list=$CKANEXT__SCHEMINGDCAT_FACET_LIST" \ - "schemingdcat.organization_custom_facets=$CKANEXT__SCHEMINGDCAT_ORGANIZATION_CUSTOM_FACETS" \ - "schemingdcat.group_custom_facets=$CKANEXT__SCHEMINGDCAT_GROUP_CUSTOM_FACETS" \ - "schemingdcat.geometadata_base_uri=$CKANEXT__SCHEMINGDCAT_GEOMETADATA_BASE_URI" \ - "schemingdcat.default_package_item_icon=$CKANEXT__SCHEMINGDCAT_DEFAULT_PACKAGE_ITEM_ICON" \ - "schemingdcat.default_package_item_show_spatial=$CKANEXT__SCHEMINGDCAT_DEFAULT_PACKAGE_ITEM_SHOW_SPATIAL" \ - "schemingdcat.show_metadata_templates_toolbar=$CKANEXT__SCHEMINGDCAT_SHOW_METADATA_TEMPLATES_TOOLBAR" \ - "schemingdcat.metadata_templates_search_identifier=$CKANEXT__METADATA_TEMPLATES_SEARCH_IDENTIFIER" - -echo "[docker-entrypoint.02_setup_scheming] Rebuild index" -ckan -c $CKAN_INI search-index rebuild \ No newline at end of file diff --git a/ckan/docker-entrypoint.d/01_setup_xloader.sh b/ckan/docker-entrypoint.d/02_setup_xloader.sh similarity index 100% rename from ckan/docker-entrypoint.d/01_setup_xloader.sh rename to ckan/docker-entrypoint.d/02_setup_xloader.sh diff --git a/ckan/docker-entrypoint.d/03_setup_dcat.sh b/ckan/docker-entrypoint.d/03_setup_dcat.sh deleted file mode 100644 index 65d095d..0000000 --- a/ckan/docker-entrypoint.d/03_setup_dcat.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -# Add ckanext-dcat settings to the CKAN config file -echo "[docker-entrypoint.03_setup_dcat] Loading ckanext-dcat settings in the CKAN config file" -ckan config-tool $CKAN_INI \ - "ckanext.dcat.base_uri = $CKANEXT__DCAT__BASE_URI" \ - "ckanext.dcat.catalog_endpoint = $CKANEXT__DCAT__DEFAULT_CATALOG_ENDPOINT" \ - "ckanext.dcat.rdf.profiles = $CKANEXT__DCAT__RDF_PROFILES" \ No newline at end of file diff --git a/ckan/docker-entrypoint.d/04_setup_preview.sh b/ckan/docker-entrypoint.d/04_setup_preview.sh deleted file mode 100644 index 295f096..0000000 --- a/ckan/docker-entrypoint.d/04_setup_preview.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -#TODO: Correct views. - -# Add CKAN Resource views to the CKAN config file -echo "[docker-entrypoint.04_setup_preview] Loading resource views in the CKAN config file" -ckan config-tool $CKAN_INI \ - "ckan.views.default_views = $CKAN__VIEWS__DEFAULT_VIEWS" \ - "ckan.preview.json_formats = $CKAN__PREVIEW__JSON_FORMATS" \ - "ckan.preview.xml_formats = $CKAN__PREVIEW__XML_FORMATS" \ - "ckan.preview.text_formats = $CKAN__PREVIEW__TEXT_FORMATS" \ - "ckan.preview.loadable = $CKAN__PREVIEW__LOADABLE" - -# Add CKAN Resource geoviews to the CKAN config file -echo "[docker-entrypoint.04_setup_preview] Loading geoviews in the CKAN config file" -ckan config-tool $CKAN_INI \ - "ckanext.geoview.ol_viewer.formats = $CKANEXT__GEOVIEW__OL_VIEWER__FORMATS" \ - "ckanext.geoview.shp_viewer.srid = $CKANEXT__GEOVIEW__SHP_VIEWER__SRID" \ - "ckanext.geoview.shp_viewer.encoding = $CKANEXT__GEOVIEW__SHP_VIEWER__ENCODING" \ - "ckanext.geoview.geojson.max_file_size = $CKANEXT__GEOVIEW__GEOJSON__MAX_FILE_SIZE" \ No newline at end of file diff --git a/ckan/docker-entrypoint.d/05_setup_pages.sh b/ckan/docker-entrypoint.d/05_setup_pages.sh deleted file mode 100644 index 4f64e16..0000000 --- a/ckan/docker-entrypoint.d/05_setup_pages.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -# Add pages CKAN config file (https://github.com/ckan/ckanext-pages#configuration) -echo "[docker-entrypoint.05_setup_pages] Loading pages config in the CKAN config file" -ckan config-tool $CKAN_INI \ - "ckan.pages.allow_html = $CKANEXT__PAGES__ALOW_HTML" \ - "ckanext.pages.organization = $CKANEXT__PAGES__ORGANIZATION" \ - "ckanext.pages.group = $CKANEXT__PAGES__GROUP" \ - "ckanext.pages.about_menu = $CKANEXT__PAGES__ABOUT_MENU" \ - "ckanext.pages.group_menu = $CKANEXT__PAGES__GROUP_MENU" \ - "ckanext.pages.organization_menu = $CKANEXT__PAGES__ORGANIZATION_MENU" diff --git a/ckan/setup/start_ckan.sh.override b/ckan/setup/start_ckan.sh.override index fe1c5c4..7d52585 100644 --- a/ckan/setup/start_ckan.sh.override +++ b/ckan/setup/start_ckan.sh.override @@ -9,6 +9,11 @@ then JWT_SECRET=$(python3 -c 'import secrets; print("string:" + secrets.token_urlsafe())') ckan config-tool $CKAN_INI "api_token.jwt.encode.secret=${JWT_SECRET}" ckan config-tool $CKAN_INI "api_token.jwt.decode.secret=${JWT_SECRET}" + + echo "Update licenses in CKAN" + if [ -n "$CKAN___LICENSES_GROUP_URL" ]; then + ckan config-tool $CKAN_INI "licenses_group_url=${CKAN___LICENSES_GROUP_URL}" + fi fi # Run the prerun script to init CKAN and create the default admin user diff --git a/ckan/setup/start_ckan_development.sh.override b/ckan/setup/start_ckan_development.sh.override index b481c80..4669dac 100644 --- a/ckan/setup/start_ckan_development.sh.override +++ b/ckan/setup/start_ckan_development.sh.override @@ -54,6 +54,11 @@ then JWT_SECRET=$(python3 -c 'import secrets; print("string:" + secrets.token_urlsafe())') ckan config-tool $CKAN_INI "api_token.jwt.encode.secret=${JWT_SECRET}" ckan config-tool $CKAN_INI "api_token.jwt.decode.secret=${JWT_SECRET}" + + echo "Update licenses in CKAN" + if [ -n "$CKAN___LICENSES_GROUP_URL" ]; then + ckan config-tool $CKAN_INI "licenses_group_url=${CKAN___LICENSES_GROUP_URL}" + fi fi # Update the plugins setting in the ini file with the values defined in the env var diff --git a/doc/info_envfile.md b/doc/info_envfile.md index 1732872..9d379b1 100644 --- a/doc/info_envfile.md +++ b/doc/info_envfile.md @@ -101,7 +101,6 @@ This section contains configurations related to the core CKAN application. It in - `CKAN_SITE_ID`: The site ID for CKAN. - `CKAN_SITE_URL`: The URL of the CKAN site. - `CKAN__ROOT_PATH`: The root path for CKAN. -- `CKAN_PORT`: The port on which CKAN is running. - `CKAN__FAVICON`: The path to the favicon for CKAN. - `CKAN__SITE_LOGO`: The path to the logo for CKAN. - `CKAN___BEAKER__SESSION__SECRET`: The secret for the Beaker session. diff --git a/docker-compose.apache.yml b/docker-compose.apache.yml index 4fda7f5..c518706 100644 --- a/docker-compose.apache.yml +++ b/docker-compose.apache.yml @@ -57,7 +57,7 @@ services: - site_packages:/usr/lib/python3.9/site-packages restart: unless-stopped healthcheck: - test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:${CKAN_PORT}"] + test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"] pycsw: build: diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 7635a30..1e6bf87 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -17,7 +17,7 @@ services: - .env environment: - CKAN_COMPOSE_SERVICE=${CKAN_DEV_COMPOSE_SERVICE} - - PROXY_CKAN_PROXY_PASS=http://${CKAN_DEV_COMPOSE_SERVICE}:${CKAN_PORT} + - PROXY_CKAN_PROXY_PASS=http://${CKAN_DEV_COMPOSE_SERVICE}:5000 depends_on: - ${CKAN_DEV_COMPOSE_SERVICE} ports: @@ -42,7 +42,7 @@ services: - solr - redis ports: - - "0.0.0.0:${CKAN_PORT_HOST}:${CKAN_PORT}" + - "0.0.0.0:${CKAN_PORT_HOST}:5000" volumes: - ckan_storage:/var/lib/ckan - ckan_logs:/var/log @@ -52,7 +52,7 @@ services: - vscode_server:/root/.vscode-server restart: unless-stopped healthcheck: - test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:${CKAN_PORT}"] + test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"] pycsw: build: diff --git a/docker-compose.ghcr.apache.yml b/docker-compose.ghcr.apache.yml index 761d70f..22f5a72 100644 --- a/docker-compose.ghcr.apache.yml +++ b/docker-compose.ghcr.apache.yml @@ -57,7 +57,7 @@ services: - site_packages:/usr/lib/python3.9/site-packages restart: unless-stopped healthcheck: - test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:${CKAN_PORT}"] + test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"] pycsw: build: diff --git a/docker-compose.ghcr.yml b/docker-compose.ghcr.yml index 94b7219..e96b1f6 100644 --- a/docker-compose.ghcr.yml +++ b/docker-compose.ghcr.yml @@ -58,7 +58,7 @@ services: - site_packages:/usr/lib/python3.9/site-packages restart: unless-stopped healthcheck: - test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:${CKAN_PORT}"] + test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"] pycsw: build: diff --git a/docker-compose.yml b/docker-compose.yml index c087ac8..f4e1b2a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -58,7 +58,7 @@ services: - site_packages:/usr/lib/python3.9/site-packages restart: unless-stopped healthcheck: - test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:${CKAN_PORT}"] + test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"] pycsw: build: diff --git a/postgresql/docker-entrypoint-initdb.d/30_setup_test_databases.sh b/postgresql/docker-entrypoint-initdb.d/30_setup_test_databases.sh index da55af3..67013dd 100755 --- a/postgresql/docker-entrypoint-initdb.d/30_setup_test_databases.sh +++ b/postgresql/docker-entrypoint-initdb.d/30_setup_test_databases.sh @@ -2,6 +2,6 @@ set -e psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL - CREATE DATABASE ckan_test OWNER "$CKAN_DB_USER" ENCODING 'utf-8'; - CREATE DATABASE datastore_test OWNER "$CKAN_DB_USER" ENCODING 'utf-8'; + CREATE DATABASE $CKAN_TEST_DB OWNER "$CKAN_DB_USER" ENCODING 'utf-8'; + CREATE DATABASE $DATASTORE_TEST_DB OWNER "$CKAN_DB_USER" ENCODING 'utf-8'; EOSQL diff --git a/samples/.env.apache.example b/samples/.env.apache.example deleted file mode 100644 index 5caefdc..0000000 --- a/samples/.env.apache.example +++ /dev/null @@ -1,166 +0,0 @@ -# Base -APP_DIR=/srv/app - -# Container names -REDIS_CONTAINER_NAME=redis -POSTGRESQL_CONTAINER_NAME=db -SOLR_CONTAINER_NAME=solr -CKAN_CONTAINER_NAME=ckan -WORKER_CONTAINER_NAME=ckan-worker -APACHE_CONTAINER_NAME=apache -PYCSW_CONTAINER_NAME=pycsw - -# Host Ports -CKAN_PORT_HOST=5000 -APACHE_PORT_HOST=81 -PYCSW_PORT_HOST=8000 - -# Solr -SOLR_IMAGE_VERSION=2.9-solr9-spatial -SOLR_PORT=8983 -CKAN_SOLR_URL=http://solr:${SOLR_PORT}/solr/ckan -TEST_CKAN_SOLR_URL=http://solr:${SOLR_PORT}/solr/ckan - -# Redis -REDIS_VERSION=7-alpine -CKAN_REDIS_URL=redis://redis:6379/1 -TEST_CKAN_REDIS_URL=redis://redis:6379/1 - -# Apache HTTP Server -APACHE_VERSION=2.4-alpine -APACHE_PORT=80 -APACHE_LOG_DIR=/var/log/apache - -#NIGNX/APACHE -## Check CKAN__ROOT_PATH and CKANEXT__DCAT__BASE_URI. If you don't need to use domain locations, it is better to use the nginx configuration. Leave blank or use the root `/`. -PROXY_SERVER_NAME=localhost -PROXY_CKAN_LOCATION=/catalog -PROXY_PYCSW_LOCATION=/csw - -# pycsw -PYCSW_PORT=8000 -CKAN_URL=http://localhost:81/catalog -PYCSW_URL=http://localhost:81/csw -# SCHEMAS: ckan2pycsw/model/dataset.py - Dataset type -PYCSW_CKAN_SCHEMA=iso19139_geodcatap -PYCSW_OUPUT_SCHEMA=iso19139_inspire -# ckan-pycsw schedule -## ckan2pycsw days between each scheduler job -PYCSW_CRON_DAYS_INTERVAL=2 -# ckan2pycsw hour of start of the scheduler job (0-23) -PYCSW_CRON_HOUR_START=4 -## Timezone -TZ=UTC - -# CKAN databases -POSTGRES_USER=postgres -POSTGRES_PASSWORD=postgres -POSTGRES_DB=postgres -POSTGRES_HOST=db -CKAN_DB_USER=ckandbuser -CKAN_DB_PASSWORD=ckandbpassword -CKAN_DB=ckandb -DATASTORE_READONLY_USER=datastore_ro -DATASTORE_READONLY_PASSWORD=datastore -DATASTORE_DB=datastore -CKAN_SQLALCHEMY_URL=postgresql://ckandbuser:ckandbpassword@db/ckandb -CKAN_DATASTORE_WRITE_URL=postgresql://ckandbuser:ckandbpassword@db/datastore -CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore - -# Test database connections -TEST_CKAN_SQLALCHEMY_URL=postgres://ckan:ckan@db/ckan_test -TEST_CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore_test -TEST_CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore_test - -# CKAN core -## If use docker-compose.ghcr.yml only "*.*.*" versions available in: https://github.com/mjanez/ckan-docker/pkgs/container/ckan-docker -CKAN_VERSION=2.9.11 -CKAN_SITE_ID=default -# CKAN_SITE_URL = http:/ or https:/ + PROXY_SERVER_NAME. Optionally the APACHE_HOST_PORT if different from 80 -CKAN_SITE_URL=http://localhost:81 -CKAN__ROOT_PATH=/catalog/{{LANG}} -CKAN_PORT=5000 -CKAN__FAVICON=/catalog/base/images/ckan.ico -CKAN___BEAKER__SESSION__SECRET=CHANGE_ME -# See https://docs.ckan.org/en/latest/maintaining/configuration.html#api-token-settings -CKAN___API_TOKEN__JWT__ENCODE__SECRET=string:CHANGE_ME -CKAN___API_TOKEN__JWT__DECODE__SECRET=string:CHANGE_ME -CKAN_SYSADMIN_NAME=ckan_admin -CKAN_SYSADMIN_PASSWORD=test1234 -CKAN_SYSADMIN_EMAIL=your_email@example.com -CKAN_STORAGE_PATH=/var/lib/ckan -CKAN_LOGS_PATH=/var/log -CKAN_SMTP_SERVER=smtp.corporateict.domain:25 -CKAN_SMTP_STARTTLS=True -CKAN_SMTP_USER=user -CKAN_SMTP_PASSWORD=pass -CKAN_SMTP_MAIL_FROM=ckan@localhost -## Customize which text formats the text_view plugin will show -CKAN__PREVIEW__JSON_FORMATS="json jsonld" -# html htm rdf+xml owl+xml xml n3 n-triples turtle plain atom csv tsv rss txt json -CKAN__PREVIEW__XML_FORMATS="xml rdf rdf+xml owl+xml atom rss turtle ttl n3 n-triples" -CKAN__PREVIEW__TEXT_FORMATS="text plain text/plain text/turtle csv tsv rss txt json" -CKAN__PREVIEW__LOADABLE="html htm rdf+xml owl+xml xml n3 n-triples turtle plain atom csv tsv rss txt json arcgis_rest" - -## Resource Proxy settings -### Preview size limit, default: 1MB -CKAN__RESOURCE_PROXY__MAX_FILE_SIZE=50048576 -## Size of chunks to read/write__ -CKAN__RESOURCE_PROXY__CHUNK_SIZE=4096 -## Default timeout for fetching proxied items -CKAN__RESOURCE_PROXY__TIMEOUT=10 -CKAN__VIEWS__DEFAULT_VIEWS="image_view webpage_view text_view recline_view wmts_view geojson_view geo_view shp_view pdf_view" - -# Localization -CKAN__LOCALE_DEFAULT="en" -CKAN__LOCALE_ORDER="en es pt_BR ja it cs_CZ ca fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv" - -# Extensions -CKAN__PLUGINS="envvars stats text_view image_view webpage_view recline_view resourcedictionary datastore xloader harvest ckan_harvester spatial_metadata spatial_query spatial_harvest_metadata_api csw_harvester waf_harvester doc_harvester resource_proxy geo_view geojson_view wmts_view shp_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface schemingdcat_datasets schemingdcat_groups schemingdcat_organizations schemingdcat pdf_view pages" - -# ckanext-harvest -CKAN__HARVEST__MQ__TYPE=redis -CKAN__HARVEST__MQ__HOSTNAME=redis -CKAN__HARVEST__MQ__PORT=6379 -CKAN__HARVEST__MQ__REDIS_DB=1 - -# ckanext-xloader -CKANEXT__XLOADER__API_TOKEN=api_token -CKANEXT__XLOADER__JOBS__DB_URI=postgresql://ckan:ckan@db/ckan - -# ckanext-dcat -CKANEXT__DCAT__BASE_URI=http://localhost:81/catalog -# Default profile(s). Instead of this envvar, it's possible to specify all the profile(s) availables to be used for serialization using the profiles parameter: http://localhost:5000/catalog.xml?profiles=euro_dcat_ap,spain_dcat -CKANEXT__DCAT__RDF_PROFILES='euro_dcat_ap_2' -# The custom endpoint **must** start with a forward slash (`/`) and contain the `{_format}` placeholder. The endpoint is added to the CKAN_SITE_URL and CKAN__ROOT_PATH, example: http://localhost:5000/catalog/catalog.rdf -CKANEXT__DCAT__DEFAULT_CATALOG_ENDPOINT='/catalog.{_format}' - -# ckanext-spatial (Solr Backend - solr8-spatial) -CKANEXT__SPATIAL__SEARCH_BACKEND=solr-bbox -CKAN__SPATIAL__SRID=3857 -CKANEXT__SPATIAL__COMMON_MAP__TYPE=custom -CKANEXT__SPATIAL__COMMON_MAP__CUSTOM__URL=https://stamen-tiles-{s}.a.ssl.fastly.net/terrain/{z}/{x}/{y}.png -CKANEXT__SPATIAL__COMMON_MAP__ATTRIBUTION='Map tiles by Stamen Design (CC BY 3.0). Data by OpenStreetMap (CC BY SA)' - -# ckanext-geoview -CKANEXT__GEOVIEW__GEOJSON__MAX_FILE_SIZE=100000000 -CKANEXT__GEOVIEW__OL_VIEWER__FORMATS="wms wfs geojson gml kml" -CKANEXT__GEOVIEW__SHP_VIEWER__SRID=3857 -CKANEXT__GEOVIEW__SHP_VIEWER__ENCODING=UTF-8 - -# ckanext-schemingdcat -## CSW Endpoint for spatial metadata -CKANEXT__SCHEMINGDCAT_GEOMETADATA_BASE_URI=${PYCSW_URL} -## Scheming: setup_scheming.sh -CKANEXT__SCHEMINGDCAT_DATASET_SCHEMA="ckanext.schemingdcat:schemas/geodcatap/geodcatap_dataset.yaml" -CKANEXT__SCHEMINGDCAT_GROUP_SCHEMAS="ckanext.schemingdcat:schemas/geodcatap/geodcatap_group.json" -CKANEXT__SCHEMINGDCAT_ORGANIZATION_SCHEMAS="ckanext.schemingdcat:schemas/geodcatap/geodcatap_org.json" -CKANEXT__SCHEMINGDCAT_PRESETS="ckanext.schemingdcat:schemas/geodcatap/geodcatap_presets.json" -## Facets: setup_scheming.sh -CKANEXT__SCHEMINGDCAT_FACET_LIST="theme groups theme_es dcat_type groups publisher_name publisher_type spatial_uri owner_org res_format frequency tags tag_uri conforms_to" -CKANEXT__SCHEMINGDCAT_ORGANIZATION_CUSTOM_FACETS=True -CKANEXT__SCHEMINGDCAT_GROUP_CUSTOM_FACETS=True - -# WIP: ckanext-sparql_interface -CKANEXT__SPARQL__ENDPOINT_URL=https://dbpedia.org/sparql -CKANEXT__SPARQL__HIDE_ENDPOINT_URL=False \ No newline at end of file diff --git a/samples/.env.es.example b/samples/.env.es.example new file mode 100644 index 0000000..5555403 --- /dev/null +++ b/samples/.env.es.example @@ -0,0 +1,198 @@ +# Base +APP_DIR=/srv/app +# Sets Docker Compose project name to avoid resource conflicts between projects. Defaults to the folder name "ckan-docker" if not set. +#COMPOSE_PROJECT_NAME=ckan-docker-mytheme + +# Host Ports +CKAN_PORT_HOST=5000 +# Common proxy server for Apache or NGINX. +# Change all the PROXY_SERVER_PORT_HOST to PROXY_SERVER_HTTPS_PORT_HOST if you can use HTTPS instead of HTTP (Only for NGINX and not development compose) +PROXY_SERVER_PORT_HOST=81 +PROXY_SERVER_HTTPS_PORT_HOST=8443 +NGINX_PORT_HOST=${PROXY_SERVER_PORT_HOST} +NGINX_SSLPORT_HOST=${PROXY_SERVER_HTTPS_PORT_HOST} +APACHE_PORT_HOST=${PROXY_SERVER_PORT_HOST} +PYCSW_PORT_HOST=8000 + +# Solr +SOLR_IMAGE_VERSION=2.9-solr9-spatial +SOLR_PORT=8983 +SOLR_CKAN_DATABASE=ckan +CKAN_SOLR_URL=http://solr:${SOLR_PORT}/solr/${SOLR_CKAN_DATABASE} +TEST_CKAN_SOLR_URL=http://solr:${SOLR_PORT}/solr/${SOLR_CKAN_DATABASE} + +# Redis +REDIS_VERSION=7-alpine +REDIS_PORT=6379 +REDIS_CKAN_DATABASE=1 +CKAN_REDIS_URL=redis://redis:${REDIS_PORT}/${REDIS_CKAN_DATABASE} +TEST_CKAN_REDIS_URL=redis://redis:${REDIS_PORT}/${REDIS_CKAN_DATABASE} + +# NGINX +NGINX_PORT=80 +NGINX_SSLPORT=443 +NGINX_LOG_DIR=/var/log/nginx + +# Apache HTTP Server +APACHE_VERSION=2.4-alpine +APACHE_PORT=80 +APACHE_LOG_DIR=/var/log/apache + +#NGINX/APACHE +## Check CKAN__ROOT_PATH and CKANEXT__DCAT__BASE_URI and CKANEXT__SCHEMINGDCAT_GEOMETADATA_BASE_URI. If you don't need to use domain locations, it is better to use the nginx configuration. Leave blank or use the root `/`. If you dont need PROXY_SERVER_PORT_HOST (e.g. is 80), edit the PROXY_SERVER_URL and remove :${PROXY_SERVER_PORT_HOST} +PROXY_SERVER_NAME=localhost +PROXY_SERVER_URL=http://${PROXY_SERVER_NAME}:${PROXY_SERVER_PORT_HOST} +PROXY_CKAN_LOCATION=/catalog +PROXY_PYCSW_LOCATION=/csw + +# pycsw +PYCSW_PORT=8000 +CKAN_URL=${PROXY_SERVER_URL}${PROXY_CKAN_LOCATION} +PYCSW_URL=${PROXY_SERVER_URL}${PROXY_PYCSW_LOCATION} +# SCHEMAS: ckan2pycsw/model/dataset.py - Dataset type +PYCSW_CKAN_SCHEMA=iso19139_geodcatap +PYCSW_OUPUT_SCHEMA=iso19139_inspire +# ckan-pycsw schedule +## ckan2pycsw days between each scheduler job +PYCSW_CRON_DAYS_INTERVAL=2 +# ckan2pycsw hour of start of the scheduler job (0-23) +PYCSW_CRON_HOUR_START=4 +## Timezone +TZ=Europe/Madrid + +# CKAN databases +POSTGRES_USER=postgres +POSTGRES_PASSWORD=postgres +POSTGRES_DB=postgres +POSTGRES_HOST=db +CKAN_DB_USER=ckandbuser +CKAN_DB_PASSWORD=ckandbpassword +CKAN_DB=ckandb +DATASTORE_READONLY_USER=datastore_ro +DATASTORE_READONLY_PASSWORD=datastore +DATASTORE_DB=datastore +CKAN_SQLALCHEMY_URL=postgresql://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@${POSTGRES_HOST}/${CKAN_DB} +CKAN_DATASTORE_WRITE_URL=postgresql://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@${POSTGRES_HOST}/${DATASTORE_DB} +CKAN_DATASTORE_READ_URL=postgresql://${DATASTORE_READONLY_USER}:${DATASTORE_READONLY_PASSWORD}@${POSTGRES_HOST}/${DATASTORE_DB} + +# Test database connections +CKAN_TEST_DB=ckan_test +DATASTORE_TEST_DB=datastore_test +TEST_CKAN_SQLALCHEMY_URL=postgres://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@${POSTGRES_HOST}/${CKAN_TEST_DB} +TEST_CKAN_DATASTORE_WRITE_URL=postgresql://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@${POSTGRES_HOST}/${DATASTORE_TEST_DB} +TEST_CKAN_DATASTORE_READ_URL=postgresql://${DATASTORE_READONLY_USER}:${DATASTORE_READONLY_PASSWORD}@${POSTGRES_HOST}/${DATASTORE_TEST_DB} + +# Dev settings +USE_HTTPS_FOR_DEV=false +CKAN_DEV_COMPOSE_SERVICE=ckan-dev + +# CKAN core +## If use docker-compose.ghcr.yml only "*.*.*" versions available in: https://github.com/mjanez/ckan-docker/pkgs/container/ckan-docker +CKAN_VERSION=2.9.11 +CKAN_SITE_ID=default +# CKAN_SITE_URL = http:/ or https:/ + PROXY_SERVER_NAME. Optionally the APACHE_HOST_PORT if different from 80 +CKAN_SITE_URL=${PROXY_SERVER_URL} +CKAN__ROOT_PATH=${PROXY_CKAN_LOCATION}/{{LANG}} +CKAN__FAVICON=${PROXY_CKAN_LOCATION}/base/images/ckan.ico +CKAN__SITE_LOGO=/images/default/ckan-logo.png +# Custom licenses that are aligned with DCAT-AP +CKAN___LICENSES_GROUP_URL=https://raw.githubusercontent.com/mjanez/ckanext-schemingdcat/main/ckanext/schemingdcat/public/static/licenses.json +CKAN___BEAKER__SESSION__SECRET=CHANGE_ME +# See https://docs.ckan.org/en/latest/maintaining/configuration.html#api-token-settings +CKAN___API_TOKEN__JWT__ENCODE__SECRET=string:CHANGE_ME +CKAN___API_TOKEN__JWT__DECODE__SECRET=string:CHANGE_ME +CKAN_SYSADMIN_NAME=ckan_admin +CKAN_SYSADMIN_PASSWORD=test1234 +CKAN_SYSADMIN_EMAIL=your_email@example.com +CKAN_STORAGE_PATH=/var/lib/ckan +CKAN_LOGS_PATH=/var/log +CKAN_SMTP_SERVER=smtp.corporateict.domain:25 +CKAN_SMTP_STARTTLS=True +CKAN_SMTP_USER=user +CKAN_SMTP_PASSWORD=pass +CKAN_SMTP_MAIL_FROM=ckan@${PROXY_SERVER_NAME} +## Customize which text formats the text_view plugin will show +CKAN__PREVIEW__JSON_FORMATS="json jsonld" +# html htm rdf+xml owl+xml xml n3 n-triples turtle plain atom csv tsv rss txt json +CKAN__PREVIEW__XML_FORMATS="xml rdf rdf+xml owl+xml atom rss turtle ttl n3 n-triples" +CKAN__PREVIEW__TEXT_FORMATS="text plain text/plain text/turtle csv tsv rss txt json" +CKAN__PREVIEW__LOADABLE="html htm rdf+xml owl+xml xml n3 n-triples turtle plain atom csv tsv rss txt json arcgis_rest" +# ckanext-spatial: Allow Solr local params: https://github.com/ckan/ckanext-spatial/issues/328 +CKAN__SEARCH__SOLR_ALLOWED_QUERY_PARSERS=field +# CORS Settings. If True, all origins will be allowed (the response header Access-Control-Allow-Origin is set to ‘*’). If False, only origins from the ckan.cors.origin_whitelist setting will be allowed. +CKAN__CORS__ORIGIN_ALLOW_ALL=False +CKAN__CORS__ORIGIN_WHITELIST="" + +## Resource Proxy settings +### Preview size limit, default: 1MB +CKAN__RESOURCE_PROXY__MAX_FILE_SIZE=50048576 +## Size of chunks to read/write__ +CKAN__RESOURCE_PROXY__CHUNK_SIZE=4096 +## Default timeout for fetching proxied items +CKAN__RESOURCE_PROXY__TIMEOUT=10 +CKAN__VIEWS__DEFAULT_VIEWS="image_view webpage_view text_view recline_view wmts_view geojson_view geo_view shp_view pdf_view" + +# Localization +CKAN__LOCALE_DEFAULT="es" +CKAN__LOCALE_ORDER="es en pt_BR ja it cs_CZ ca fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv" + +# Extensions +CKAN__PLUGINS="stats image_view text_view recline_view webpage_view resourcedictionary datastore xloader spatial_metadata spatial_query spatial_harvest_metadata_api csw_harvester waf_harvester doc_harvester resource_proxy geo_view geojson_view wmts_view shp_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface schemingdcat_datasets schemingdcat_groups schemingdcat_organizations schemingdcat schemingdcat_ckan_harvester schemingdcat_xls_harvester harvest pdf_view pages fluent envvars" + +# ckanext-harvest +CKAN__HARVEST__MQ__TYPE=redis +CKAN__HARVEST__MQ__HOSTNAME=redis +CKAN__HARVEST__MQ__PORT=${REDIS_PORT} +CKAN__HARVEST__MQ__REDIS_DB=${REDIS_CKAN_DATABASE} +# Clean-up mechanism for the harvest log table. The default is 30 days. +CKAN__HARVEST__LOG_TIMEFRAME=40 + +# ckanext-xloader +CKANEXT__XLOADER__API_TOKEN=api_token +CKANEXT__XLOADER__JOBS__DB_URI=postgresql://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@${POSTGRES_HOST}/${CKAN_DB} + +# ckanext-dcat +CKANEXT__DCAT__BASE_URI=${CKAN_URL} +# Default profile(s). Instead of this envvar, it's possible to specify all the profile(s) availables to be used for serialization using the profiles parameter: http://localhost:5000/catalog.xml?profiles=eu_dcat_ap_2,es_dcat +CKANEXT__DCAT__RDF_PROFILES='eu_dcat_ap_2 eu_dcat_ap_scheming' +# The custom endpoint **must** start with a forward slash (`/`) and contain the `{_format}` placeholder. The endpoint is added to the CKAN_SITE_URL and CKAN__ROOT_PATH, example: http://localhost:5000/catalog/catalog.rdf +CKANEXT__DCAT__DEFAULT_CATALOG_ENDPOINT='/catalog.{_format}' + +# ckanext-spatial (Solr Backend - solr9-spatial: https://docs.ckan.org/projects/ckanext-spatial/en/latest/spatial-search.html#choosing-a-backend-for-the-spatial-search) +CKANEXT__SPATIAL__SEARCH_BACKEND=solr-spatial-field +CKAN__SPATIAL__SRID=3857 +CKANEXT__SPATIAL__COMMON_MAP__TYPE=custom +CKANEXT__SPATIAL__COMMON_MAP__CUSTOM__URL=https://rts.larioja.org/mapa-base/rioja/{z}/{x}/{y}.png +CKANEXT__SPATIAL__COMMON_MAP__ATTRIBUTION='SCNE, bajo CC BY 4.0' + +# ckanext-geoview +CKANEXT__GEOVIEW__GEOJSON__MAX_FILE_SIZE=100000000 +CKANEXT__GEOVIEW__OL_VIEWER__FORMATS="wms wfs geojson gml kml" +CKANEXT__GEOVIEW__SHP_VIEWER__SRID=3857 +CKANEXT__GEOVIEW__SHP_VIEWER__ENCODING=UTF-8 + +# ckanext-schemingdcat +## CSW Endpoint for spatial metadata +CKANEXT__SCHEMINGDCAT_GEOMETADATA_BASE_URI=${PYCSW_URL} +## Scheming: setup_scheming.sh +CKANEXT__SCHEMINGDCAT_DATASET_SCHEMA="ckanext.schemingdcat:schemas/geodcat_ap/es_geodcat_ap_2.yaml ckanext.schemingdcat:schemas/resources/dcat_3_document.yaml" +CKANEXT__SCHEMINGDCAT_GROUP_SCHEMAS="ckanext.schemingdcat:schemas/geodcat_ap/es_geodcat_ap_group.json" +CKANEXT__SCHEMINGDCAT_ORGANIZATION_SCHEMAS="ckanext.schemingdcat:schemas/geodcat_ap/es_geodcat_ap_org.json" +CKANEXT__SCHEMINGDCAT_PRESETS="ckanext.schemingdcat:schemas/default_presets.json ckanext.fluent:presets.json" +## Facets: setup_scheming.sh +CKANEXT__SCHEMINGDCAT_FACET_LIST="dataset_scope theme groups theme_es dcat_type groups publisher_name publisher_type spatial_uri owner_org res_format frequency tags tag_uri conforms_to" +CKANEXT__SCHEMINGDCAT_ORGANIZATION_CUSTOM_FACETS=True +CKANEXT__SCHEMINGDCAT_GROUP_CUSTOM_FACETS=True +CKANEXT__SCHEMINGDCAT_DEFAULT_PACKAGE_ITEM_ICON="theme" +CKANEXT__SCHEMINGDCAT_DEFAULT_PACKAGE_ITEM_SHOW_SPATIAL=True +CKANEXT__SCHEMINGDCAT_SHOW_METADATA_TEMPLATES_TOOLBAR=False +CKANEXT__METADATA_TEMPLATES_SEARCH_IDENTIFIER="schemingdcat_xls-template" +CKANEXT__SCHEMINGDCAT_ENDPOINTS_YAML="endpoints.yaml" + +# ckanext-pages +CKANEXT__PAGES__ALOW_HTML=False +CKANEXT__PAGES__ORGANIZATION=True +CKANEXT__PAGES__GROUP=True +CKANEXT__PAGES__ABOUT_MENU=False +CKANEXT__PAGES__GROUP_MENU=True +CKANEXT__PAGES__ORGANIZATION_MENU=True \ No newline at end of file diff --git a/samples/.env.localhost b/samples/.env.localhost index 232be42..11dd700 100644 --- a/samples/.env.localhost +++ b/samples/.env.localhost @@ -1,33 +1,32 @@ # Base APP_DIR=/srv/app - -# Container names -NGINX_CONTAINER_NAME=nginx -REDIS_CONTAINER_NAME=redis -POSTGRESQL_CONTAINER_NAME=db -SOLR_CONTAINER_NAME=solr -CKAN_CONTAINER_NAME=ckan -WORKER_CONTAINER_NAME=ckan-worker -APACHE_CONTAINER_NAME=apache -PYCSW_CONTAINER_NAME=pycsw +# Sets Docker Compose project name to avoid resource conflicts between projects. Defaults to the folder name "ckan-docker" if not set. +#COMPOSE_PROJECT_NAME=ckan-docker-mytheme # Host Ports CKAN_PORT_HOST=5000 -NGINX_PORT_HOST=81 -NGINX_SSLPORT_HOST=8443 -APACHE_PORT_HOST=81 +# Common proxy server for Apache or NGINX. +# Change all the PROXY_SERVER_PORT_HOST to PROXY_SERVER_HTTPS_PORT_HOST if you can use HTTPS instead of HTTP (Only for NGINX and not development compose) +PROXY_SERVER_PORT_HOST=81 +PROXY_SERVER_HTTPS_PORT_HOST=8443 +NGINX_PORT_HOST=${PROXY_SERVER_PORT_HOST} +NGINX_SSLPORT_HOST=${PROXY_SERVER_HTTPS_PORT_HOST} +APACHE_PORT_HOST=${PROXY_SERVER_PORT_HOST} PYCSW_PORT_HOST=8000 # Solr SOLR_IMAGE_VERSION=2.9-solr9-spatial SOLR_PORT=8983 -CKAN_SOLR_URL=http://solr:${SOLR_PORT}/solr/ckan -TEST_CKAN_SOLR_URL=http://solr:${SOLR_PORT}/solr/ckan +SOLR_CKAN_DATABASE=ckan +CKAN_SOLR_URL=http://solr:${SOLR_PORT}/solr/${SOLR_CKAN_DATABASE} +TEST_CKAN_SOLR_URL=http://solr:${SOLR_PORT}/solr/${SOLR_CKAN_DATABASE} # Redis REDIS_VERSION=7-alpine -CKAN_REDIS_URL=redis://redis:6379/1 -TEST_CKAN_REDIS_URL=redis://redis:6379/1 +REDIS_PORT=6379 +REDIS_CKAN_DATABASE=1 +CKAN_REDIS_URL=redis://redis:${REDIS_PORT}/${REDIS_CKAN_DATABASE} +TEST_CKAN_REDIS_URL=redis://redis:${REDIS_PORT}/${REDIS_CKAN_DATABASE} # NGINX NGINX_PORT=80 @@ -39,16 +38,17 @@ APACHE_VERSION=2.4-alpine APACHE_PORT=80 APACHE_LOG_DIR=/var/log/apache -#NIGNX/APACHE -## Check CKAN__ROOT_PATH and CKANEXT__DCAT__BASE_URI. If you don't need to use domain locations, it is better to use the nginx configuration. Leave blank or use the root `/`. +#NGINX/APACHE +## Check CKAN__ROOT_PATH and CKANEXT__DCAT__BASE_URI and CKANEXT__SCHEMINGDCAT_GEOMETADATA_BASE_URI. If you don't need to use domain locations, it is better to use the nginx configuration. Leave blank or use the root `/`. If you dont need PROXY_SERVER_PORT_HOST (e.g. is 80), edit the PROXY_SERVER_URL and remove :${PROXY_SERVER_PORT_HOST} PROXY_SERVER_NAME=localhost +PROXY_SERVER_URL=http://${PROXY_SERVER_NAME}:${PROXY_SERVER_PORT_HOST} PROXY_CKAN_LOCATION=/catalog PROXY_PYCSW_LOCATION=/csw # pycsw PYCSW_PORT=8000 -CKAN_URL=http://localhost:81/catalog -PYCSW_URL=http://localhost:81/csw +CKAN_URL=${PROXY_SERVER_URL}${PROXY_CKAN_LOCATION} +PYCSW_URL=${PROXY_SERVER_URL}${PROXY_PYCSW_LOCATION} # SCHEMAS: ckan2pycsw/model/dataset.py - Dataset type PYCSW_CKAN_SCHEMA=iso19139_geodcatap PYCSW_OUPUT_SCHEMA=iso19139_inspire @@ -71,24 +71,32 @@ CKAN_DB=ckandb DATASTORE_READONLY_USER=datastore_ro DATASTORE_READONLY_PASSWORD=datastore DATASTORE_DB=datastore -CKAN_SQLALCHEMY_URL=postgresql://ckandbuser:ckandbpassword@db/ckandb -CKAN_DATASTORE_WRITE_URL=postgresql://ckandbuser:ckandbpassword@db/datastore -CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore +CKAN_SQLALCHEMY_URL=postgresql://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@${POSTGRES_HOST}/${CKAN_DB} +CKAN_DATASTORE_WRITE_URL=postgresql://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@${POSTGRES_HOST}/${DATASTORE_DB} +CKAN_DATASTORE_READ_URL=postgresql://${DATASTORE_READONLY_USER}:${DATASTORE_READONLY_PASSWORD}@${POSTGRES_HOST}/${DATASTORE_DB} # Test database connections -TEST_CKAN_SQLALCHEMY_URL=postgres://ckan:ckan@db/ckan_test -TEST_CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore_test -TEST_CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore_test +CKAN_TEST_DB=ckan_test +DATASTORE_TEST_DB=datastore_test +TEST_CKAN_SQLALCHEMY_URL=postgres://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@${POSTGRES_HOST}/${CKAN_TEST_DB} +TEST_CKAN_DATASTORE_WRITE_URL=postgresql://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@${POSTGRES_HOST}/${DATASTORE_TEST_DB} +TEST_CKAN_DATASTORE_READ_URL=postgresql://${DATASTORE_READONLY_USER}:${DATASTORE_READONLY_PASSWORD}@${POSTGRES_HOST}/${DATASTORE_TEST_DB} + +# Dev settings +USE_HTTPS_FOR_DEV=false +CKAN_DEV_COMPOSE_SERVICE=ckan-dev # CKAN core ## If use docker-compose.ghcr.yml only "*.*.*" versions available in: https://github.com/mjanez/ckan-docker/pkgs/container/ckan-docker CKAN_VERSION=2.9.11 CKAN_SITE_ID=default # CKAN_SITE_URL = http:/ or https:/ + PROXY_SERVER_NAME. Optionally the APACHE_HOST_PORT if different from 80 -CKAN_SITE_URL=http://localhost:81 -CKAN__ROOT_PATH=/catalog/{{LANG}} -CKAN_PORT=5000 -CKAN__FAVICON=/catalog/base/images/ckan.ico +CKAN_SITE_URL=${PROXY_SERVER_URL} +CKAN__ROOT_PATH=${PROXY_CKAN_LOCATION}/{{LANG}} +CKAN__FAVICON=${PROXY_CKAN_LOCATION}/base/images/ckan.ico +CKAN__SITE_LOGO=/images/default/ckan-logo.png +# Custom licenses that are aligned with DCAT-AP +CKAN___LICENSES_GROUP_URL=https://raw.githubusercontent.com/mjanez/ckanext-schemingdcat/main/ckanext/schemingdcat/public/static/licenses.json CKAN___BEAKER__SESSION__SECRET=CHANGE_ME # See https://docs.ckan.org/en/latest/maintaining/configuration.html#api-token-settings CKAN___API_TOKEN__JWT__ENCODE__SECRET=string:CHANGE_ME @@ -102,13 +110,18 @@ CKAN_SMTP_SERVER=smtp.corporateict.domain:25 CKAN_SMTP_STARTTLS=True CKAN_SMTP_USER=user CKAN_SMTP_PASSWORD=pass -CKAN_SMTP_MAIL_FROM=ckan@localhost +CKAN_SMTP_MAIL_FROM=ckan@${PROXY_SERVER_NAME} ## Customize which text formats the text_view plugin will show CKAN__PREVIEW__JSON_FORMATS="json jsonld" # html htm rdf+xml owl+xml xml n3 n-triples turtle plain atom csv tsv rss txt json CKAN__PREVIEW__XML_FORMATS="xml rdf rdf+xml owl+xml atom rss turtle ttl n3 n-triples" CKAN__PREVIEW__TEXT_FORMATS="text plain text/plain text/turtle csv tsv rss txt json" CKAN__PREVIEW__LOADABLE="html htm rdf+xml owl+xml xml n3 n-triples turtle plain atom csv tsv rss txt json arcgis_rest" +# ckanext-spatial: Allow Solr local params: https://github.com/ckan/ckanext-spatial/issues/328 +CKAN__SEARCH__SOLR_ALLOWED_QUERY_PARSERS=field +# CORS Settings. If True, all origins will be allowed (the response header Access-Control-Allow-Origin is set to ‘*’). If False, only origins from the ckan.cors.origin_whitelist setting will be allowed. +CKAN__CORS__ORIGIN_ALLOW_ALL=False +CKAN__CORS__ORIGIN_WHITELIST="" ## Resource Proxy settings ### Preview size limit, default: 1MB @@ -122,33 +135,36 @@ CKAN__VIEWS__DEFAULT_VIEWS="image_view webpage_view text_view recline_view wmts_ # Localization CKAN__LOCALE_DEFAULT="en" CKAN__LOCALE_ORDER="en es pt_BR ja it cs_CZ ca fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv" +CKAN__LOCALES_OFFERED="en es pt_BR ja it cs_CZ ca fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv" # Extensions -CKAN__PLUGINS="envvars stats text_view image_view webpage_view recline_view resourcedictionary datastore xloader harvest ckan_harvester spatial_metadata spatial_query spatial_harvest_metadata_api csw_harvester waf_harvester doc_harvester resource_proxy geo_view geojson_view wmts_view shp_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface schemingdcat_datasets schemingdcat_groups schemingdcat_organizations schemingdcat pdf_view pages" +CKAN__PLUGINS="stats image_view text_view recline_view webpage_view resourcedictionary datastore xloader spatial_metadata spatial_query spatial_harvest_metadata_api csw_harvester waf_harvester doc_harvester resource_proxy geo_view geojson_view wmts_view shp_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface schemingdcat_datasets schemingdcat_groups schemingdcat_organizations schemingdcat schemingdcat_ckan_harvester schemingdcat_xls_harvester harvest pdf_view pages fluent envvars" # ckanext-harvest CKAN__HARVEST__MQ__TYPE=redis CKAN__HARVEST__MQ__HOSTNAME=redis -CKAN__HARVEST__MQ__PORT=6379 -CKAN__HARVEST__MQ__REDIS_DB=1 +CKAN__HARVEST__MQ__PORT=${REDIS_PORT} +CKAN__HARVEST__MQ__REDIS_DB=${REDIS_CKAN_DATABASE} +# Clean-up mechanism for the harvest log table. The default is 30 days. +CKAN__HARVEST__LOG_TIMEFRAME=40 # ckanext-xloader CKANEXT__XLOADER__API_TOKEN=api_token -CKANEXT__XLOADER__JOBS__DB_URI=postgresql://ckan:ckan@db/ckan +CKANEXT__XLOADER__JOBS__DB_URI=postgresql://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@${POSTGRES_HOST}/${CKAN_DB} # ckanext-dcat -CKANEXT__DCAT__BASE_URI=http://localhost:81/catalog -# Default profile(s). Instead of this envvar, it's possible to specify all the profile(s) availables to be used for serialization using the profiles parameter: http://localhost:5000/catalog.xml?profiles=euro_dcat_ap,spain_dcat -CKANEXT__DCAT__RDF_PROFILES='euro_dcat_ap_2' +CKANEXT__DCAT__BASE_URI=${CKAN_URL} +# Default profile(s). Instead of this envvar, it's possible to specify all the profile(s) availables to be used for serialization using the profiles parameter: http://localhost:5000/catalog.xml?profiles=eu_dcat_ap_2,es_dcat +CKANEXT__DCAT__RDF_PROFILES='eu_dcat_ap_2 eu_dcat_ap_scheming' # The custom endpoint **must** start with a forward slash (`/`) and contain the `{_format}` placeholder. The endpoint is added to the CKAN_SITE_URL and CKAN__ROOT_PATH, example: http://localhost:5000/catalog/catalog.rdf CKANEXT__DCAT__DEFAULT_CATALOG_ENDPOINT='/catalog.{_format}' -# ckanext-spatial (Solr Backend - solr8-spatial) -CKANEXT__SPATIAL__SEARCH_BACKEND=solr-bbox +# ckanext-spatial (Solr Backend - solr9-spatial: https://docs.ckan.org/projects/ckanext-spatial/en/latest/spatial-search.html#choosing-a-backend-for-the-spatial-search) +CKANEXT__SPATIAL__SEARCH_BACKEND=solr-spatial-field CKAN__SPATIAL__SRID=3857 CKANEXT__SPATIAL__COMMON_MAP__TYPE=custom -CKANEXT__SPATIAL__COMMON_MAP__CUSTOM__URL=https://stamen-tiles-{s}.a.ssl.fastly.net/terrain/{z}/{x}/{y}.png -CKANEXT__SPATIAL__COMMON_MAP__ATTRIBUTION='Map tiles by Stamen Design (CC BY 3.0). Data by OpenStreetMap (CC BY SA)' +CKANEXT__SPATIAL__COMMON_MAP__CUSTOM__URL=http://a.tile.openstreetmap.org/{z}/{x}/{y}.png +CKANEXT__SPATIAL__COMMON_MAP__ATTRIBUTION='Map tiles by OpenStreetMap (CC BY SA)' # ckanext-geoview CKANEXT__GEOVIEW__GEOJSON__MAX_FILE_SIZE=100000000 @@ -160,15 +176,24 @@ CKANEXT__GEOVIEW__SHP_VIEWER__ENCODING=UTF-8 ## CSW Endpoint for spatial metadata CKANEXT__SCHEMINGDCAT_GEOMETADATA_BASE_URI=${PYCSW_URL} ## Scheming: setup_scheming.sh -CKANEXT__SCHEMINGDCAT_DATASET_SCHEMA="ckanext.schemingdcat:schemas/geodcatap/geodcatap_dataset.yaml" -CKANEXT__SCHEMINGDCAT_GROUP_SCHEMAS="ckanext.schemingdcat:schemas/geodcatap/geodcatap_group.json" -CKANEXT__SCHEMINGDCAT_ORGANIZATION_SCHEMAS="ckanext.schemingdcat:schemas/geodcatap/geodcatap_org.json" -CKANEXT__SCHEMINGDCAT_PRESETS="ckanext.schemingdcat:schemas/geodcatap/geodcatap_presets.json" +CKANEXT__SCHEMINGDCAT_DATASET_SCHEMA="ckanext.schemingdcat:schemas/geodcat_ap/eu_geodcat_ap_2.yaml ckanext.schemingdcat:schemas/resources/dcat_3_document.yaml" +CKANEXT__SCHEMINGDCAT_GROUP_SCHEMAS="ckanext.schemingdcat:schemas/geodcat_ap/eu_geodcat_ap_group.json" +CKANEXT__SCHEMINGDCAT_ORGANIZATION_SCHEMAS="ckanext.schemingdcat:schemas/geodcat_ap/eu_geodcat_ap_org.json" +CKANEXT__SCHEMINGDCAT_PRESETS="ckanext.schemingdcat:schemas/default_presets.json ckanext.fluent:presets.json" ## Facets: setup_scheming.sh -CKANEXT__SCHEMINGDCAT_FACET_LIST="theme groups theme_es dcat_type groups publisher_name publisher_type spatial_uri owner_org res_format frequency tags tag_uri conforms_to" +CKANEXT__SCHEMINGDCAT_FACET_LIST="dataset_scope theme groups theme_eu dcat_type groups publisher_name publisher_type spatial_uri owner_org res_format frequency tags tag_uri conforms_to" CKANEXT__SCHEMINGDCAT_ORGANIZATION_CUSTOM_FACETS=True CKANEXT__SCHEMINGDCAT_GROUP_CUSTOM_FACETS=True +CKANEXT__SCHEMINGDCAT_DEFAULT_PACKAGE_ITEM_ICON="theme" +CKANEXT__SCHEMINGDCAT_DEFAULT_PACKAGE_ITEM_SHOW_SPATIAL=True +CKANEXT__SCHEMINGDCAT_SHOW_METADATA_TEMPLATES_TOOLBAR=False +CKANEXT__METADATA_TEMPLATES_SEARCH_IDENTIFIER="schemingdcat_xls-template" +CKANEXT__SCHEMINGDCAT_ENDPOINTS_YAML="endpoints.yaml" -# WIP: ckanext-sparql_interface -CKANEXT__SPARQL__ENDPOINT_URL=https://dbpedia.org/sparql -CKANEXT__SPARQL__HIDE_ENDPOINT_URL=False \ No newline at end of file +# ckanext-pages +CKANEXT__PAGES__ALOW_HTML=False +CKANEXT__PAGES__ORGANIZATION=True +CKANEXT__PAGES__GROUP=True +CKANEXT__PAGES__ABOUT_MENU=False +CKANEXT__PAGES__GROUP_MENU=True +CKANEXT__PAGES__ORGANIZATION_MENU=True \ No newline at end of file diff --git a/samples/.env.nginx.example b/samples/.env.nginx.example deleted file mode 100644 index 4c2fed2..0000000 --- a/samples/.env.nginx.example +++ /dev/null @@ -1,166 +0,0 @@ -# Base -APP_DIR=/srv/app - -# Container names -NGINX_CONTAINER_NAME=nginx -REDIS_CONTAINER_NAME=redis -POSTGRESQL_CONTAINER_NAME=db -SOLR_CONTAINER_NAME=solr -CKAN_CONTAINER_NAME=ckan -WORKER_CONTAINER_NAME=ckan-worker -PYCSW_CONTAINER_NAME=pycsw - -# Host Ports -CKAN_PORT_HOST=5000 -NGINX_PORT_HOST=81 -NGINX_SSLPORT_HOST=8443 - -# Solr -SOLR_IMAGE_VERSION=2.9-solr9-spatial -SOLR_PORT=8983 -CKAN_SOLR_URL=http://solr:${SOLR_PORT}/solr/ckan -TEST_CKAN_SOLR_URL=http://solr:${SOLR_PORT}/solr/ckan - -# Redis -REDIS_VERSION=7-alpine -CKAN_REDIS_URL=redis://redis:6379/1 -TEST_CKAN_REDIS_URL=redis://redis:6379/1 - -# NGINX -NGINX_PORT=80 -NGINX_SSLPORT=443 -NGINX_LOG_DIR=/var/log/nginx - -#NIGNX/APACHE -## Check CKAN__ROOT_PATH and CKANEXT__DCAT__BASE_URI. If you don't need to use domain locations, it is better to use the nginx configuration. Leave blank or use the root `/`. -PROXY_SERVER_NAME=localhost -PROXY_CKAN_LOCATION=/catalog -PROXY_PYCSW_LOCATION=/csw - -# pycsw -PYCSW_PORT=8000 -CKAN_URL=http://localhost:81/catalog -PYCSW_URL=http://localhost:81/csw -# SCHEMAS: ckan2pycsw/model/dataset.py - Dataset type -PYCSW_CKAN_SCHEMA=iso19139_geodcatap -PYCSW_OUPUT_SCHEMA=iso19139_inspire -# ckan-pycsw schedule -## ckan2pycsw days between each scheduler job -PYCSW_CRON_DAYS_INTERVAL=2 -# ckan2pycsw hour of start of the scheduler job (0-23) -PYCSW_CRON_HOUR_START=4 -## Timezone -TZ=UTC - -# CKAN databases -POSTGRES_USER=postgres -POSTGRES_PASSWORD=postgres -POSTGRES_DB=postgres -POSTGRES_HOST=db -CKAN_DB_USER=ckandbuser -CKAN_DB_PASSWORD=ckandbpassword -CKAN_DB=ckandb -DATASTORE_READONLY_USER=datastore_ro -DATASTORE_READONLY_PASSWORD=datastore -DATASTORE_DB=datastore -CKAN_SQLALCHEMY_URL=postgresql://ckandbuser:ckandbpassword@db/ckandb -CKAN_DATASTORE_WRITE_URL=postgresql://ckandbuser:ckandbpassword@db/datastore -CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore - -# Test database connections -TEST_CKAN_SQLALCHEMY_URL=postgres://ckan:ckan@db/ckan_test -TEST_CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore_test -TEST_CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore_test - -# CKAN core -## If use docker-compose.ghcr.yml only "*.*.*" versions available in: https://github.com/mjanez/ckan-docker/pkgs/container/ckan-docker -CKAN_VERSION=2.9.11 -CKAN_SITE_ID=default -# CKAN_SITE_URL = http:/ or https:/ + PROXY_SERVER_NAME. Optionally the APACHE_HOST_PORT if different from 80 -CKAN_SITE_URL=http://localhost:81 -CKAN__ROOT_PATH=/catalog/{{LANG}} -CKAN_PORT=5000 -CKAN__FAVICON=/catalog/base/images/ckan.ico -CKAN___BEAKER__SESSION__SECRET=CHANGE_ME -# See https://docs.ckan.org/en/latest/maintaining/configuration.html#api-token-settings -CKAN___API_TOKEN__JWT__ENCODE__SECRET=string:CHANGE_ME -CKAN___API_TOKEN__JWT__DECODE__SECRET=string:CHANGE_ME -CKAN_SYSADMIN_NAME=ckan_admin -CKAN_SYSADMIN_PASSWORD=test1234 -CKAN_SYSADMIN_EMAIL=your_email@example.com -CKAN_STORAGE_PATH=/var/lib/ckan -CKAN_LOGS_PATH=/var/log -CKAN_SMTP_SERVER=smtp.corporateict.domain:25 -CKAN_SMTP_STARTTLS=True -CKAN_SMTP_USER=user -CKAN_SMTP_PASSWORD=pass -CKAN_SMTP_MAIL_FROM=ckan@localhost -## Customize which text formats the text_view plugin will show -CKAN__PREVIEW__JSON_FORMATS="json jsonld" -# html htm rdf+xml owl+xml xml n3 n-triples turtle plain atom csv tsv rss txt json -CKAN__PREVIEW__XML_FORMATS="xml rdf rdf+xml owl+xml atom rss turtle ttl n3 n-triples" -CKAN__PREVIEW__TEXT_FORMATS="text plain text/plain text/turtle csv tsv rss txt json" -CKAN__PREVIEW__LOADABLE="html htm rdf+xml owl+xml xml n3 n-triples turtle plain atom csv tsv rss txt json arcgis_rest" - -## Resource Proxy settings -### Preview size limit, default: 1MB -CKAN__RESOURCE_PROXY__MAX_FILE_SIZE=50048576 -## Size of chunks to read/write__ -CKAN__RESOURCE_PROXY__CHUNK_SIZE=4096 -## Default timeout for fetching proxied items -CKAN__RESOURCE_PROXY__TIMEOUT=10 -CKAN__VIEWS__DEFAULT_VIEWS="image_view webpage_view text_view recline_view wmts_view geojson_view geo_view shp_view pdf_view" - -# Localization -CKAN__LOCALE_DEFAULT="en" -CKAN__LOCALE_ORDER="en es pt_BR ja it cs_CZ ca fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv" - -# Extensions -CKAN__PLUGINS="envvars stats text_view image_view webpage_view recline_view resourcedictionary datastore xloader harvest ckan_harvester spatial_metadata spatial_query spatial_harvest_metadata_api csw_harvester waf_harvester doc_harvester resource_proxy geo_view geojson_view wmts_view shp_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface schemingdcat_datasets schemingdcat_groups schemingdcat_organizations schemingdcat pdf_view pages" - -# ckanext-harvest -CKAN__HARVEST__MQ__TYPE=redis -CKAN__HARVEST__MQ__HOSTNAME=redis -CKAN__HARVEST__MQ__PORT=6379 -CKAN__HARVEST__MQ__REDIS_DB=1 - -# ckanext-xloader -CKANEXT__XLOADER__API_TOKEN=api_token -CKANEXT__XLOADER__JOBS__DB_URI=postgresql://ckan:ckan@db/ckan - -# ckanext-dcat -CKANEXT__DCAT__BASE_URI=http://localhost:81/catalog -# Default profile(s). Instead of this envvar, it's possible to specify all the profile(s) availables to be used for serialization using the profiles parameter: http://localhost:5000/catalog.xml?profiles=euro_dcat_ap,spain_dcat -CKANEXT__DCAT__RDF_PROFILES='euro_dcat_ap_2' -# The custom endpoint **must** start with a forward slash (`/`) and contain the `{_format}` placeholder. The endpoint is added to the CKAN_SITE_URL and CKAN__ROOT_PATH, example: http://localhost:5000/catalog/catalog.rdf -CKANEXT__DCAT__DEFAULT_CATALOG_ENDPOINT='/catalog.{_format}' - -# ckanext-spatial (Solr Backend - solr8-spatial) -CKANEXT__SPATIAL__SEARCH_BACKEND=solr-bbox -CKAN__SPATIAL__SRID=3857 -CKANEXT__SPATIAL__COMMON_MAP__TYPE=custom -CKANEXT__SPATIAL__COMMON_MAP__CUSTOM__URL=https://stamen-tiles-{s}.a.ssl.fastly.net/terrain/{z}/{x}/{y}.png -CKANEXT__SPATIAL__COMMON_MAP__ATTRIBUTION='Map tiles by Stamen Design (CC BY 3.0). Data by OpenStreetMap (CC BY SA)' - -# ckanext-geoview -CKANEXT__GEOVIEW__GEOJSON__MAX_FILE_SIZE=100000000 -CKANEXT__GEOVIEW__OL_VIEWER__FORMATS="wms wfs geojson gml kml" -CKANEXT__GEOVIEW__SHP_VIEWER__SRID=3857 -CKANEXT__GEOVIEW__SHP_VIEWER__ENCODING=UTF-8 - -# ckanext-schemingdcat -## CSW Endpoint for spatial metadata -CKANEXT__SCHEMINGDCAT_GEOMETADATA_BASE_URI=${PYCSW_URL} -## Scheming: setup_scheming.sh -CKANEXT__SCHEMINGDCAT_DATASET_SCHEMA="ckanext.schemingdcat:schemas/geodcatap/geodcatap_dataset.yaml" -CKANEXT__SCHEMINGDCAT_GROUP_SCHEMAS="ckanext.schemingdcat:schemas/geodcatap/geodcatap_group.json" -CKANEXT__SCHEMINGDCAT_ORGANIZATION_SCHEMAS="ckanext.schemingdcat:schemas/geodcatap/geodcatap_org.json" -CKANEXT__SCHEMINGDCAT_PRESETS="ckanext.schemingdcat:schemas/geodcatap/geodcatap_presets.json" -## Facets: setup_scheming.sh -CKANEXT__SCHEMINGDCAT_FACET_LIST="theme groups theme_es dcat_type groups publisher_name publisher_type spatial_uri owner_org res_format frequency tags tag_uri conforms_to" -CKANEXT__SCHEMINGDCAT_ORGANIZATION_CUSTOM_FACETS=True -CKANEXT__SCHEMINGDCAT_GROUP_CUSTOM_FACETS=True - -# WIP: ckanext-sparql_interface -CKANEXT__SPARQL__ENDPOINT_URL=https://dbpedia.org/sparql -CKANEXT__SPARQL__HIDE_ENDPOINT_URL=False \ No newline at end of file diff --git a/samples/ckan.ini.example b/samples/ckan.ini.example index 7ec88b5..8b7ceb9 100644 --- a/samples/ckan.ini.example +++ b/samples/ckan.ini.example @@ -133,7 +133,7 @@ scheming.presets = ckanext.scheming:presets.json ## ckanext-dcat ckanext.dcat.base_uri = http://localhost:5000 -ckanext.dcat.rdf.profiles = euro_dcat_ap_2 euro_dcat_ap +ckanext.dcat.rdf.profiles = eu_dcat_ap_2 eu_dcat_ap ## ckanext-spatial ckanext.spatial.search_backend = solr-bbox diff --git a/samples/custom/.env.es.example b/samples/custom/.env.es.example deleted file mode 100644 index 645bc2f..0000000 --- a/samples/custom/.env.es.example +++ /dev/null @@ -1,182 +0,0 @@ -# Base -APP_DIR=/srv/app - -# Container names -NGINX_CONTAINER_NAME=nginx -REDIS_CONTAINER_NAME=redis -POSTGRESQL_CONTAINER_NAME=db -SOLR_CONTAINER_NAME=solr -CKAN_CONTAINER_NAME=ckan -WORKER_CONTAINER_NAME=ckan-worker -APACHE_CONTAINER_NAME=apache -PYCSW_CONTAINER_NAME=pycsw - -# Host Ports -CKAN_PORT_HOST=5000 -NGINX_PORT_HOST=81 -NGINX_SSLPORT_HOST=8443 -APACHE_PORT_HOST=81 -PYCSW_PORT_HOST=8000 - -# Solr -SOLR_IMAGE_VERSION=2.9-solr9-spatial -SOLR_PORT=8983 -CKAN_SOLR_URL=http://solr:${SOLR_PORT}/solr/ckan -TEST_CKAN_SOLR_URL=http://solr:${SOLR_PORT}/solr/ckan - -# Redis -REDIS_VERSION=7-alpine -CKAN_REDIS_URL=redis://redis:6379/1 -TEST_CKAN_REDIS_URL=redis://redis:6379/1 - -# NGINX -NGINX_PORT=80 -NGINX_SSLPORT=443 -NGINX_LOG_DIR=/var/log/nginx - -# Apache HTTP Server -APACHE_VERSION=2.4-alpine -APACHE_PORT=80 -APACHE_LOG_DIR=/var/log/apache - -#NGINX/APACHE -## Check CKAN__ROOT_PATH and CKANEXT__DCAT__BASE_URI. If you don't need to use domain locations, it is better to use the nginx configuration. Leave blank or use the root `/`. -PROXY_SERVER_NAME=localhost -PROXY_CKAN_LOCATION=/catalog -PROXY_PYCSW_LOCATION=/csw - -# pycsw -PYCSW_PORT=8000 -CKAN_URL=http://localhost:81/catalog -PYCSW_URL=http://localhost:81/csw -# SCHEMAS: ckan2pycsw/model/dataset.py - Dataset type -PYCSW_CKAN_SCHEMA=iso19139_geodcatap -PYCSW_OUPUT_SCHEMA=iso19139_inspire -# ckan-pycsw schedule -## ckan2pycsw days between each scheduler job -PYCSW_CRON_DAYS_INTERVAL=2 -# ckan2pycsw hour of start of the scheduler job (0-23) -PYCSW_CRON_HOUR_START=4 -## Timezone -TZ=Europe/Madrid - -# CKAN databases -POSTGRES_USER=postgres -POSTGRES_PASSWORD=postgres -POSTGRES_DB=postgres -POSTGRES_HOST=db -CKAN_DB_USER=ckandbuser -CKAN_DB_PASSWORD=ckandbpassword -CKAN_DB=ckandb -DATASTORE_READONLY_USER=datastore_ro -DATASTORE_READONLY_PASSWORD=datastore -DATASTORE_DB=datastore -CKAN_SQLALCHEMY_URL=postgresql://ckandbuser:ckandbpassword@db/ckandb -CKAN_DATASTORE_WRITE_URL=postgresql://ckandbuser:ckandbpassword@db/datastore -CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore - -# Test database connections -TEST_CKAN_SQLALCHEMY_URL=postgres://ckan:ckan@db/ckan_test -TEST_CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore_test -TEST_CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore_test - -# CKAN core -## If use docker-compose.ghcr.yml only "*.*.*" versions available in: https://github.com/mjanez/ckan-docker/pkgs/container/ckan-docker -CKAN_VERSION=2.9.11 -CKAN_SITE_ID=default -# CKAN_SITE_URL = http:/ or https:/ + PROXY_SERVER_NAME. Optionally the APACHE_HOST_PORT if different from 80 -CKAN_SITE_URL=http://localhost:81 -CKAN__ROOT_PATH=/catalog/{{LANG}} -CKAN_PORT=5000 -CKAN__FAVICON=/catalog/base/images/ckan.ico -CKAN___BEAKER__SESSION__SECRET=CHANGE_ME -# See https://docs.ckan.org/en/latest/maintaining/configuration.html#api-token-settings -CKAN___API_TOKEN__JWT__ENCODE__SECRET=string:CHANGE_ME -CKAN___API_TOKEN__JWT__DECODE__SECRET=string:CHANGE_ME -CKAN_SYSADMIN_NAME=ckan_admin -CKAN_SYSADMIN_PASSWORD=test1234 -CKAN_SYSADMIN_EMAIL=your_email@example.com -CKAN_STORAGE_PATH=/var/lib/ckan -CKAN_LOGS_PATH=/var/log -CKAN_SMTP_SERVER=smtp.corporateict.domain:25 -CKAN_SMTP_STARTTLS=True -CKAN_SMTP_USER=user -CKAN_SMTP_PASSWORD=pass -CKAN_SMTP_MAIL_FROM=ckan@localhost -## Customize which text formats the text_view plugin will show -CKAN__PREVIEW__JSON_FORMATS="json jsonld" -# html htm rdf+xml owl+xml xml n3 n-triples turtle plain atom csv tsv rss txt json -CKAN__PREVIEW__XML_FORMATS="xml rdf rdf+xml owl+xml atom rss turtle ttl n3 n-triples" -CKAN__PREVIEW__TEXT_FORMATS="text plain text/plain text/turtle csv tsv rss txt json" -CKAN__PREVIEW__LOADABLE="html htm rdf+xml owl+xml xml n3 n-triples turtle plain atom csv tsv rss txt json arcgis_rest" - -## Resource Proxy settings -### Preview size limit, default: 1MB -CKAN__RESOURCE_PROXY__MAX_FILE_SIZE=50048576 -## Size of chunks to read/write__ -CKAN__RESOURCE_PROXY__CHUNK_SIZE=4096 -## Default timeout for fetching proxied items -CKAN__RESOURCE_PROXY__TIMEOUT=10 -CKAN__VIEWS__DEFAULT_VIEWS="image_view webpage_view text_view recline_view wmts_view geojson_view geo_view shp_view pdf_view" - -# Localization -CKAN__LOCALE_DEFAULT="es" -CKAN__LOCALE_ORDER="es en pt_BR ja it cs_CZ ca fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv" - -# Extensions -CKAN__PLUGINS="envvars stats text_view image_view webpage_view recline_view resourcedictionary datastore xloader harvest ckan_harvester spatial_metadata spatial_query spatial_harvest_metadata_api csw_harvester waf_harvester doc_harvester resource_proxy geo_view geojson_view wmts_view shp_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface schemingdcat_datasets schemingdcat_groups schemingdcat_organizations schemingdcat pdf_view pages" - -# ckanext-harvest -CKAN__HARVEST__MQ__TYPE=redis -CKAN__HARVEST__MQ__HOSTNAME=redis -CKAN__HARVEST__MQ__PORT=6379 -CKAN__HARVEST__MQ__REDIS_DB=1 - -# ckanext-xloader -CKANEXT__XLOADER__API_TOKEN=api_token -CKANEXT__XLOADER__JOBS__DB_URI=postgresql://ckan:ckan@db/ckan - -# ckanext-dcat -CKANEXT__DCAT__BASE_URI=http://localhost:81/catalog -# Default profile(s). Instead of this envvar, it's possible to specify all the profile(s) availables to be used for serialization using the profiles parameter: http://localhost:5000/catalog.xml?profiles=euro_dcat_ap,spain_dcat -CKANEXT__DCAT__RDF_PROFILES='euro_dcat_ap_2' -# The custom endpoint **must** start with a forward slash (`/`) and contain the `{_format}` placeholder. The endpoint is added to the CKAN_SITE_URL and CKAN__ROOT_PATH, example: http://localhost:5000/catalog/catalog.rdf -CKANEXT__DCAT__DEFAULT_CATALOG_ENDPOINT='/catalog.{_format}' - -# ckanext-spatial (Solr Backend - solr8-spatial) -CKANEXT__SPATIAL__SEARCH_BACKEND=solr-bbox -CKAN__SPATIAL__SRID=3857 -CKANEXT__SPATIAL__COMMON_MAP__TYPE=custom -CKANEXT__SPATIAL__COMMON_MAP__CUSTOM__URL=https://rts.larioja.org/mapa-base/rioja/{z}/{x}/{y}.png -CKANEXT__SPATIAL__COMMON_MAP__ATTRIBUTION='SCNE, bajo CC BY 4.0' - -# ckanext-geoview -CKANEXT__GEOVIEW__GEOJSON__MAX_FILE_SIZE=100000000 -CKANEXT__GEOVIEW__OL_VIEWER__FORMATS="wms wfs geojson gml kml" -CKANEXT__GEOVIEW__SHP_VIEWER__SRID=3857 -CKANEXT__GEOVIEW__SHP_VIEWER__ENCODING=UTF-8 - -# ckanext-schemingdcat -## CSW Endpoint for spatial metadata -CKANEXT__SCHEMINGDCAT_GEOMETADATA_BASE_URI=${PYCSW_URL} -## Scheming: setup_scheming.sh -CKANEXT__SCHEMINGDCAT_DATASET_SCHEMA="ckanext.schemingdcat:schemas/geodcatap_es/geodcatap_es_dataset.yaml" -CKANEXT__SCHEMINGDCAT_GROUP_SCHEMAS="ckanext.schemingdcat:schemas/geodcatap_es/geodcatap_es_group.json" -CKANEXT__SCHEMINGDCAT_ORGANIZATION_SCHEMAS="ckanext.schemingdcat:schemas/geodcatap_es/geodcatap_es_org.json" -CKANEXT__SCHEMINGDCAT_PRESETS="ckanext.schemingdcat:schemas/geodcatap_es/geodcatap_es_presets.json" -## Facets: setup_scheming.sh -CKANEXT__SCHEMINGDCAT_FACET_LIST="theme groups theme_es dcat_type groups publisher_name publisher_type spatial_uri owner_org res_format frequency tags tag_uri conforms_to" -CKANEXT__SCHEMINGDCAT_ORGANIZATION_CUSTOM_FACETS=True -CKANEXT__SCHEMINGDCAT_GROUP_CUSTOM_FACETS=True - -# ckanext-pages -CKANEXT__PAGES__ALOW_HTML=False -CKANEXT__PAGES__ORGANIZATION=True -CKANEXT__PAGES__GROUP=True -CKANEXT__PAGES__ABOUT_MENU=False -CKANEXT__PAGES__GROUP_MENU=False -CKANEXT__PAGES__ORGANIZATION_MENU=False - -# WIP: ckanext-sparql_interface -CKANEXT__SPARQL__ENDPOINT_URL=https://dbpedia.org/sparql -CKANEXT__SPARQL__HIDE_ENDPOINT_URL=False \ No newline at end of file diff --git a/solr/Dockerfile b/solr/Dockerfile index 1d2f171..c81a81c 100644 --- a/solr/Dockerfile +++ b/solr/Dockerfile @@ -6,13 +6,15 @@ EXPOSE ${SOLR_PORT} ARG CKAN_BRANCH="dev-v2.10" -ENV SOLR_CONFIG_DIR="/opt/solr/server/solr/configsets" +ENV SOLR_INSTALL="/opt/solr" +ENV SOLR_CONFIG_DIR="$SOLR_INSTALL/server/solr/configsets" ENV SOLR_SCHEMA_FILE="$SOLR_CONFIG_DIR/ckan/conf/managed-schema" +ENV SOLR_CKAN_DATABASE=ckan USER root # Create a CKAN configset by copying the default one -RUN cp -R $SOLR_CONFIG_DIR/_default $SOLR_CONFIG_DIR/ckan +RUN cp -R $SOLR_CONFIG_DIR/_default $SOLR_CONFIG_DIR/$SOLR_CKAN_DATABASE # Update the schema ADD https://raw.githubusercontent.com/ckan/ckan/$CKAN_BRANCH/ckan/config/solr/schema.xml $SOLR_SCHEMA_FILE @@ -20,4 +22,4 @@ RUN chmod 644 $SOLR_SCHEMA_FILE USER solr -CMD ["sh", "-c", "solr-precreate ckan $SOLR_CONFIG_DIR/ckan"] \ No newline at end of file +CMD ["sh", "-c", "solr-precreate $SOLR_CKAN_DATABASE $SOLR_CONFIG_DIR/$SOLR_CKAN_DATABASE"] \ No newline at end of file diff --git a/solr/Dockerfile.spatial b/solr/Dockerfile.spatial index 56f9348..cb63e1c 100644 --- a/solr/Dockerfile.spatial +++ b/solr/Dockerfile.spatial @@ -9,6 +9,7 @@ ARG CKAN_BRANCH="dev-v2.10" ENV SOLR_INSTALL="/opt/solr" ENV SOLR_CONFIG_DIR="$SOLR_INSTALL/server/solr/configsets" ENV SOLR_SCHEMA_FILE="$SOLR_CONFIG_DIR/ckan/conf/managed-schema" +ENV SOLR_CKAN_DATABASE=ckan ARG JTS_VERSION="1.19.0" ARG JTS_JAR_FILE="$SOLR_INSTALL/server/solr-webapp/webapp/WEB-INF/lib/jts-core-$JTS_VERSION.jar" @@ -16,7 +17,7 @@ ARG JTS_JAR_FILE="$SOLR_INSTALL/server/solr-webapp/webapp/WEB-INF/lib/jts-core-$ USER root # Create a CKAN configset by copying the default one -RUN cp -R $SOLR_CONFIG_DIR/_default $SOLR_CONFIG_DIR/ckan +RUN cp -R $SOLR_CONFIG_DIR/_default $SOLR_CONFIG_DIR/$SOLR_CKAN_DATABASE # Update the schema ADD https://raw.githubusercontent.com/ckan/ckan/$CKAN_BRANCH/ckan/config/solr/schema.xml $SOLR_SCHEMA_FILE @@ -50,13 +51,16 @@ ENV SOLR_BBOX_FIELDS ' \ \ \ + \ \ \ \ + \ \ \ \ \ + \ ' \ '' \ '' @@ -68,4 +72,4 @@ RUN chmod 644 $SOLR_SCHEMA_FILE USER solr -CMD ["sh", "-c", "solr-precreate ckan $SOLR_CONFIG_DIR/ckan"] \ No newline at end of file +CMD ["sh", "-c", "solr-precreate $SOLR_CKAN_DATABASE $SOLR_CONFIG_DIR/$SOLR_CKAN_DATABASE"] \ No newline at end of file diff --git a/src/README.md b/src/README.md index 0a600b1..9979d74 100644 --- a/src/README.md +++ b/src/README.md @@ -2,4 +2,67 @@ This folder is used to clone CKAN extensions in development mode. >**Warning** -> Save the extension code in this folder before updating/deleting the repo, all contents (`src/*)` are in `.gitignore` file. \ No newline at end of file +> Save the extension code in this folder before updating/deleting the repo, all contents (`src/*)` are in `.gitignore` file. + +## Clone repos with bash +Update `repos` with your requirements: +```bash +#!/bin/bash + +# Check if the script is running from the src directory +current_dir=$(basename "$(pwd)") +if [ "$current_dir" != "src" ]; then + echo "This script must be run from the src directory. Please move the script to the src directory and try again." + exit 1 +fi + +# Define repositories and tags +declare -A repos=( + ["ckan/ckanext-xloader"]="1.0.1" + ["ckan/ckanext-harvest"]="v1.5.6" + ["ckan/ckanext-geoview"]="v0.1.0" + ["ckan/ckanext-spatial"]="v2.1.1" + ["mjanez/ckanext-dcat"]="v1.8.0" + ["ckan/ckanext-scheming"]="release-3.0.0" + ["mjanez/ckanext-resourcedictionary"]="v1.0.1" + ["ckan/ckanext-pages"]="v0.5.2" + ["ckan/ckanext-pdfview"]="0.0.8" + ["mjanez/ckanext-fluent"]="v1.0.1" + ["mjanez/ckanext-schemingdcat"]="develop" + ["mjanez/ckanext-mqa"]="develop" +) + +# Detect if the script is running on Windows +if [[ "$OSTYPE" == "msys" || "$OSTYPE" == "cygwin" || "$OSTYPE" == "win32" ]]; then + echo "It seems you are on Windows. Do you want to configure 'core.autocrlf=input' for the cloned repositories? (y/n)" + read -r response + if [[ "$response" =~ ^[Yy]$ ]]; then + config_autocrlf="--config core.autocrlf=input" + else + config_autocrlf="" + fi +else + config_autocrlf="" +fi + +# The script assumes it is running from the src directory, so clone_path is set to "." +clone_path="." + +# Clone each repository at its specific tag +for repo in "${!repos[@]}"; do + tag="${repos[$repo]}" + repo_name=$(basename "$repo") + echo "Cloning $repo at tag $tag into $clone_path..." + if git clone $config_autocrlf --branch "$tag" "https://github.com/$repo.git" "$clone_path/$repo_name"; then + # Check for replacement .txt files in ../ckan/req_fixes/ + req_fix_path="../ckan/req_fixes/${repo_name}" + if [ -d "$req_fix_path" ]; then + echo "Replacing .txt files for $repo_name..." + # Find and replace all .txt files from req_fix_path to the cloned repo directory + find "$req_fix_path" -name "*.txt" -exec bash -c 'f="{}"; cp "$f" "'"$clone_path/${repo_name}"'/${f##*/}"' \; + fi + else + echo "Error cloning $repo." + fi +done +``` \ No newline at end of file