From 6642a75ae94fbdd5b53d6e007f2b9953ac0af42d Mon Sep 17 00:00:00 2001 From: mnjnz <96422458+mjanez@users.noreply.github.com> Date: Sat, 27 Apr 2024 17:56:25 +0200 Subject: [PATCH] Update docker-compose files and remove unnecessary container names - Add volumes for installed files for faster startup - Improve README - Remove recline and add datatables_view --- .env.example | 13 +-- README.md | 46 +++++++--- docker-compose.apache.yml | 9 +- docker-compose.dev.yml | 10 +-- docker-compose.ghcr.yml | 9 +- docker-compose.ghcr_apache.yml | 155 --------------------------------- docker-compose.yml | 23 +---- 7 files changed, 44 insertions(+), 221 deletions(-) delete mode 100644 docker-compose.ghcr_apache.yml diff --git a/.env.example b/.env.example index b1159f9..3119039 100644 --- a/.env.example +++ b/.env.example @@ -1,16 +1,5 @@ -# Base APP_DIR=/srv/app -# Container names -NGINX_CONTAINER_NAME=nginx -REDIS_CONTAINER_NAME=redis -POSTGRESQL_CONTAINER_NAME=db -SOLR_CONTAINER_NAME=solr -CKAN_CONTAINER_NAME=ckan -WORKER_CONTAINER_NAME=ckan-worker -APACHE_CONTAINER_NAME=apache -PYCSW_CONTAINER_NAME=pycsw - # Host Ports CKAN_PORT_HOST=5000 NGINX_PORT_HOST=81 @@ -134,7 +123,7 @@ CKAN__LOCALE_ORDER="en es pt_BR ja it cs_CZ ca fr el sv sr sr@latin no sk fi ru CKAN__LOCALES_OFFERED="en es pt_BR ja it cs_CZ ca fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv" # Extensions -CKAN__PLUGINS="envvars stats text_view image_view webpage_view recline_view resourcedictionary datastore xloader harvest spatial_metadata spatial_query spatial_harvest_metadata_api csw_harvester waf_harvester doc_harvester resource_proxy geo_view geojson_view wmts_view shp_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface schemingdcat_datasets schemingdcat_groups schemingdcat_organizations schemingdcat schemingdcat_ckan_harvester schemingdcat_xls_harvester pdf_view pages fluent" +CKAN__PLUGINS="envvars stats image_view text_view datatables_view webpage_view resourcedictionary datastore xloader harvest spatial_metadata spatial_query spatial_harvest_metadata_api csw_harvester waf_harvester doc_harvester resource_proxy geo_view geojson_view wmts_view shp_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface schemingdcat_datasets schemingdcat_groups schemingdcat_organizations schemingdcat schemingdcat_ckan_harvester schemingdcat_xls_harvester pdf_view pages fluent" # ckanext-harvest CKAN__HARVEST__MQ__TYPE=redis diff --git a/README.md b/README.md index 3a0e3d0..41a3ec5 100644 --- a/README.md +++ b/README.md @@ -203,7 +203,7 @@ To start the containers: docker compose -f docker-compose.dev.yml up -See [CKAN Images](#ckan-images) for more details of what happens when using development mode. +See [CKAN images](#5-ckan-images) for more details of what happens when using development mode. #### Create an extension You can use the ckan [extension](https://docs.ckan.org/en/latest/extensions/tutorial.html#creating-a-new-extension) instructions to create a CKAN extension, only executing the command inside the CKAN container and setting the mounted `src/` folder as output: @@ -215,15 +215,19 @@ You can use the ckan [extension](https://docs.ckan.org/en/latest/extensions/tuto The new extension files and directories are created in the `/srv/app/src_extensions/` folder in the running container. They will also exist in the local src/ directory as local `/src` directory is mounted as `/srv/app/src_extensions/` on the ckan container. You might need to change the owner of its folder to have the appropiate permissions. -##### Running HTTPS on development mode +#### Running HTTPS on development mode Sometimes is useful to run your local development instance under HTTPS, for instance if you are using authentication extensions like [ckanext-saml2auth](https://github.com/keitaroinc/ckanext-saml2auth). To enable it, set the following in your `.env` file: +``` USE_HTTPS_FOR_DEV=true +``` and update the site URL setting: +``` CKAN_SITE_URL=https://localhost:5000 +``` After recreating the `ckan-dev` container, you should be able to access CKAN at https://localhost:5000 @@ -330,30 +334,52 @@ ckan ## ckan-docker addons ### Debugging -#### VSCode dev containers +#### Debugging CKAN Development Instance with VSCode Dev Containers and debugpy + The [Visual Studio Code Dev Containers](https://code.visualstudio.com/docs/devcontainers/containers) extension is a powerful tool that enables developers to use a container as a complete development environment. With this extension, developers can open any folder inside a container and take advantage of the full range of features provided by Visual Studio Code. To do this, developers create a `devcontainer.json `file in their project that specifies how to access or create a development container with a predefined tool and runtime stack. This allows developers to work in an isolated environment, ensuring that the development environment is consistent across team members and that project dependencies are easy to manage. ![Developing inside a Container](https://code.visualstudio.com/assets/docs/devcontainers/containers/architecture-containers.png) +To set this up: + 1. Install [VSCode](https://code.visualstudio.com/). -1. Install the[ Remote Development extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.vscode-remote-extensionpack) for VSCode. +2. Install the [Remote Development extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.vscode-remote-extensionpack) for VSCode. -3. In your project directory, create a file named `devcontainer.json.` This file will contain the configuration for your `dev container`. +3. In your project directory, create a `devcontainer.json` file. This file will contain the configuration for your development container. -4. In the `devcontainer.json` file, specify the Docker image that you want to use for your `dev container`. +4. In the `devcontainer.json` file, specify the Docker image for your development container and any additional configuration settings, such as environment variables, ports to expose, and startup commands. -5. Specify any additional configuration settings for your `dev container`, such as environment variables, ports to expose, and startup commands. +5. Enable `debugpy` for your development instance in your `.env` file: -6. Open your project in a `dev container` by using the Remote Development extension in VSCode. You can do this by clicking the `Open Folder in Container` button in the command palette or by opening the folder using the `Remote-Containers: Open Folder in Container` command. Also you can attach to an active container `Attach to Running Container`. + ```ini + USE_DEBUGPY_FOR_DEV=true + ``` -7. VSCode will start a new container based on the configuration settings in your `devcontainer.json` file. Once the container is started, you can work on your project just like you would on your local machine. +6. Start the containers in [development mode](#development-mode) and launch VS Code. +7. Install the "Dev Container" extension: press `CTRL+SHIFT+X`, type "dev container", click "install". + +8. Click the `Open a Remote Window` button in the bottom-left of the VS Code window. + +9. Click `Attach to Running Container...` and select your ckan-dev container, e.g. `ckan-docker-ckan-dev-1`. + +10. Click the `Run and Debug` icon on the left panel then `create a launch.json`, select `Python Debugger`, `Remote Attach`, host `localhost` and port `5678`. + +11. Press `F5` or click the `Run` menu and `Start Debugging`. + +You can now set breakpoints and remote debug your CKAN development instance using VSCode Dev Containers and debugpy. #### pdb Add these lines to the `ckan-dev` service in the docker compose.dev.yml file -![pdb](https://user-images.githubusercontent.com/54408245/179964232-9e98a451-5fe9-4842-ba9b-751bcc627730.png) +```yaml +ports: + - "0.0.0.0:${CKAN_PORT}:5000" + +stdin_open: true +tty: true +``` Debug with pdb (example) - Interact with `docker attach $(docker container ls -qf name=ckan)` diff --git a/docker-compose.apache.yml b/docker-compose.apache.yml index 24996ea..e76f817 100644 --- a/docker-compose.apache.yml +++ b/docker-compose.apache.yml @@ -1,5 +1,3 @@ -version: "3" - volumes: ckan_storage: ckan_logs: @@ -8,7 +6,6 @@ volumes: services: apache: - container_name: ${APACHE_CONTAINER_NAME} build: context: apache/ dockerfile: Dockerfile @@ -29,7 +26,6 @@ services: restart: on-failure:3 ckan: - container_name: ${CKAN_CONTAINER_NAME} build: context: ckan/ dockerfile: Dockerfile @@ -56,12 +52,12 @@ services: volumes: - ckan_storage:/var/lib/ckan - ckan_logs:/var/log + - /usr/lib/python3.9/site-packages restart: unless-stopped healthcheck: test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:${CKAN_PORT}"] pycsw: - container_name: ${PYCSW_CONTAINER_NAME} build: context: ckan-pycsw/ dockerfile: Dockerfile @@ -87,7 +83,6 @@ services: test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:${PYCSW_PORT}"] db: - container_name: ${POSTGRESQL_CONTAINER_NAME} build: context: postgresql/ networks: @@ -114,7 +109,6 @@ services: test: ["CMD", "pg_isready", "-U", "${POSTGRES_USER}", "-d", "${POSTGRES_DB}"] solr: - container_name: ${SOLR_CONTAINER_NAME} build: context: solr/ dockerfile: Dockerfile.spatial @@ -134,7 +128,6 @@ services: test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:${SOLR_PORT}/solr/"] redis: - container_name: ${REDIS_CONTAINER_NAME} image: redis:${REDIS_VERSION} logging: driver: "json-file" diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 9bd222d..6d8681a 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -1,5 +1,3 @@ -version: "3" - volumes: ckan_storage: ckan_logs: @@ -8,7 +6,6 @@ volumes: services: apache: - container_name: ${APACHE_CONTAINER_NAME} build: context: apache/ dockerfile: Dockerfile @@ -21,7 +18,6 @@ services: restart: on-failure:3 ckan-dev: - container_name: ${CKAN_CONTAINER_NAME} build: context: ckan/ dockerfile: Dockerfile.dev @@ -44,12 +40,13 @@ services: - ckan_storage:/var/lib/ckan - ckan_logs:/var/log - ./src:/srv/app/src_extensions + - /usr/lib/python3.9/site-packages + - /root/.vscode-server restart: unless-stopped healthcheck: test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:${CKAN_PORT}"] pycsw: - container_name: ${PYCSW_CONTAINER_NAME} build: context: ckan-pycsw/ dockerfile: Dockerfile @@ -67,7 +64,6 @@ services: test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:${PYCSW_PORT}"] db: - container_name: ${POSTGRESQL_CONTAINER_NAME} build: context: postgresql/ environment: @@ -87,7 +83,6 @@ services: test: ["CMD", "pg_isready", "-U", "${POSTGRES_USER}", "-d", "${POSTGRES_DB}"] solr: - container_name: ${SOLR_CONTAINER_NAME} build: context: solr/ dockerfile: Dockerfile.spatial @@ -105,7 +100,6 @@ services: test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:${SOLR_PORT}/solr/"] redis: - container_name: ${REDIS_CONTAINER_NAME} image: redis:${REDIS_VERSION} logging: driver: "json-file" diff --git a/docker-compose.ghcr.yml b/docker-compose.ghcr.yml index be60d7e..d81a4ab 100644 --- a/docker-compose.ghcr.yml +++ b/docker-compose.ghcr.yml @@ -1,5 +1,3 @@ -version: "3" - volumes: ckan_storage: ckan_logs: @@ -8,7 +6,6 @@ volumes: services: nginx: - container_name: ${NGINX_CONTAINER_NAME} build: context: nginx/ dockerfile: Dockerfile @@ -29,7 +26,6 @@ services: - "0.0.0.0:${NGINX_SSLPORT_HOST}:${NGINX_SSLPORT}" ckan: - container_name: ${CKAN_CONTAINER_NAME} build: context: ckan/ dockerfile: Dockerfile.ghcr @@ -54,12 +50,12 @@ services: volumes: - ckan_storage:/var/lib/ckan - ckan_logs:/var/log + - /usr/lib/python3.9/site-packages restart: unless-stopped healthcheck: test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:${CKAN_PORT}"] pycsw: - container_name: ${PYCSW_CONTAINER_NAME} build: context: ckan-pycsw/ dockerfile: Dockerfile @@ -85,7 +81,6 @@ services: test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:${PYCSW_PORT}"] db: - container_name: ${POSTGRESQL_CONTAINER_NAME} build: context: postgresql/ networks: @@ -112,7 +107,6 @@ services: test: ["CMD", "pg_isready", "-U", "${POSTGRES_USER}", "-d", "${POSTGRES_DB}"] solr: - container_name: ${SOLR_CONTAINER_NAME} build: context: solr/ dockerfile: Dockerfile.spatial @@ -132,7 +126,6 @@ services: test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:${SOLR_PORT}/solr/"] redis: - container_name: ${REDIS_CONTAINER_NAME} image: redis:${REDIS_VERSION} logging: driver: "json-file" diff --git a/docker-compose.ghcr_apache.yml b/docker-compose.ghcr_apache.yml deleted file mode 100644 index 6df6cf7..0000000 --- a/docker-compose.ghcr_apache.yml +++ /dev/null @@ -1,155 +0,0 @@ -version: "3" - -volumes: - ckan_storage: - ckan_logs: - pg_data: - solr_data: - -services: - apache: - container_name: ${APACHE_CONTAINER_NAME} - build: - context: apache/ - dockerfile: Dockerfile - env_file: - - .env - logging: - driver: "json-file" - options: - max-size: "100m" - max-file: "10" - networks: - - webnet - - ckannet - depends_on: - - ckan - ports: - - "0.0.0.0:${APACHE_PORT_HOST}:${APACHE_PORT}" - - ckan: - container_name: ${CKAN_CONTAINER_NAME} - build: - context: ckan/ - dockerfile: Dockerfile.ghcr - args: - - TZ=${TZ} - networks: - - ckannet - - dbnet - - solrnet - - redisnet - env_file: - - .env - logging: - driver: "json-file" - options: - max-size: "100m" - max-file: "10" - depends_on: - - db - - solr - - redis - volumes: - - ckan_storage:/var/lib/ckan - - ckan_logs:/var/log - restart: unless-stopped - healthcheck: - test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:${CKAN_PORT}"] - - pycsw: - container_name: ${PYCSW_CONTAINER_NAME} - build: - context: ckan-pycsw/ - dockerfile: Dockerfile - env_file: - - .env - logging: - driver: "json-file" - options: - max-size: "100m" - max-file: "10" - networks: - - webnet - - ckannet - depends_on: - - ckan - ports: - - "0.0.0.0:${PYCSW_PORT_HOST}:${PYCSW_PORT}" - volumes: - - ./log:${APP_DIR}/log - - ./metadata:${APP_DIR}/metadata - restart: on-failure:3 - healthcheck: - test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:${PYCSW_PORT}"] - - db: - container_name: ${POSTGRESQL_CONTAINER_NAME} - build: - context: postgresql/ - networks: - - dbnet - environment: - - POSTGRES_USER - - POSTGRES_PASSWORD - - POSTGRES_DB - - CKAN_DB_USER - - CKAN_DB_PASSWORD - - CKAN_DB - - DATASTORE_READONLY_USER - - DATASTORE_READONLY_PASSWORD - - DATASTORE_DB - volumes: - - pg_data:/var/lib/postgresql/data - logging: - driver: "json-file" - options: - max-size: "100m" - max-file: "10" - restart: unless-stopped - healthcheck: - test: ["CMD", "pg_isready", "-U", "${POSTGRES_USER}", "-d", "${POSTGRES_DB}"] - - solr: - container_name: ${SOLR_CONTAINER_NAME} - build: - context: solr/ - dockerfile: Dockerfile.spatial - networks: - - solrnet - env_file: - - .env - logging: - driver: "json-file" - options: - max-size: "100m" - max-file: "10" - volumes: - - solr_data:/var/solr - restart: unless-stopped - healthcheck: - test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:${SOLR_PORT}/solr/"] - - redis: - container_name: ${REDIS_CONTAINER_NAME} - image: redis:${REDIS_VERSION} - logging: - driver: "json-file" - options: - max-size: "100m" - max-file: "10" - networks: - - redisnet - restart: unless-stopped - healthcheck: - test: ["CMD", "redis-cli", "-e", "QUIT"] - -networks: - webnet: - ckannet: - solrnet: - internal: true - dbnet: - internal: true - redisnet: - internal: true diff --git a/docker-compose.yml b/docker-compose.yml index 19d41d7..df6affb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: "3" - volumes: ckan_storage: ckan_logs: @@ -8,7 +6,6 @@ volumes: services: nginx: - container_name: ${NGINX_CONTAINER_NAME} build: context: nginx/ dockerfile: Dockerfile @@ -29,7 +26,6 @@ services: - "0.0.0.0:${NGINX_SSLPORT_HOST}:${NGINX_SSLPORT}" ckan: - container_name: ${CKAN_CONTAINER_NAME} build: context: ckan/ dockerfile: Dockerfile @@ -54,12 +50,12 @@ services: volumes: - ckan_storage:/var/lib/ckan - ckan_logs:/var/log + - /usr/lib/python3.9/site-packages restart: unless-stopped healthcheck: test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:${CKAN_PORT}"] pycsw: - container_name: ${PYCSW_CONTAINER_NAME} build: context: ckan-pycsw/ dockerfile: Dockerfile @@ -70,22 +66,11 @@ services: options: max-size: "100m" max-file: "10" - networks: - - webnet - - ckannet - depends_on: - - ckan - ports: - - "0.0.0.0:${PYCSW_PORT_HOST}:${PYCSW_PORT}" - volumes: - - ./log:${APP_DIR}/log - - ./metadata:${APP_DIR}/metadata - restart: on-failure:3 + restart: unless-stopped healthcheck: - test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:${PYCSW_PORT}"] + test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"] db: - container_name: ${POSTGRESQL_CONTAINER_NAME} build: context: postgresql/ networks: @@ -112,7 +97,6 @@ services: test: ["CMD", "pg_isready", "-U", "${POSTGRES_USER}", "-d", "${POSTGRES_DB}"] solr: - container_name: ${SOLR_CONTAINER_NAME} build: context: solr/ dockerfile: Dockerfile.spatial @@ -132,7 +116,6 @@ services: test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:${SOLR_PORT}/solr/"] redis: - container_name: ${REDIS_CONTAINER_NAME} image: redis:${REDIS_VERSION} logging: driver: "json-file"