docker-ckan/examples/harvest/docker-compose.yml

# docker-compose build && docker-compose up -d
version: "3"

volumes:
  ckan_data:
  pg_data:
  solr_data:

services:
  ckan:
    container_name: ckan
    build:
      context: .
    networks:
      - frontend
      - backend
    depends_on:
      - db
    ports:
      - "0.0.0.0:${CKAN_PORT}:5000"
    env_file:
      - ./.ckan-env
    environment:
      - CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan
      - CKAN_DATASTORE_WRITE_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/datastore
      - CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore
      - CKAN_SOLR_URL=http://solr:8983/solr/ckan
      - CKAN_REDIS_URL=redis://redis:6379/1
      - CKAN_SITE_URL=${CKAN_SITE_URL}
      - CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
      - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
    volumes:
      - ckan_data:/srv/app/data

  ckan-harvest-gather:
    container_name: ckan-harvest-gather
    build:
      context: .
    command: ckan -c /srv/app/production.ini harvester gather_consumer
    restart: on-failure
    networks:
      - frontend
      - backend
    depends_on:
      - db
      - ckan
    env_file:
      - ./.ckan-env
    environment:
      - CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan
      - CKAN_DATASTORE_WRITE_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/datastore
      - CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore
      - CKAN_SOLR_URL=http://solr:8983/solr/ckan
      - CKAN_REDIS_URL=redis://redis:6379/1
      - CKAN_SITE_URL=${CKAN_SITE_URL}
      - CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
      - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}

  ckan-harvest-fetch:
    container_name: ckan-harvest-fetch
    build:
      context: .
    command: ckan -c /srv/app/production.ini harvester fetch_consumer
    restart: on-failure
    networks:
      - frontend
      - backend
    depends_on:
      - db
      - ckan
    env_file:
      - ./.ckan-env
    environment:
      - CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan
      - CKAN_DATASTORE_WRITE_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/datastore
      - CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore
      - CKAN_SOLR_URL=http://solr:8983/solr/ckan
      - CKAN_REDIS_URL=redis://redis:6379/1
      - CKAN_SITE_URL=${CKAN_SITE_URL}
      - CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
      - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}

  ckan-harvest-run:
    container_name: ckan-harvest-run
    build:
      context: .
    command: /bin/sh -c "while true; do sleep 900; ckan -c /srv/app/production.ini harvester run; done"
    networks:
      - frontend
      - backend
    depends_on:
      - db
      - ckan
    env_file:
      - ./.ckan-env
    environment:
      - CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan
      - CKAN_DATASTORE_WRITE_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/datastore
      - CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore
      - CKAN_SOLR_URL=http://solr:8983/solr/ckan
      - CKAN_REDIS_URL=redis://redis:6379/1
      - CKAN_SITE_URL=${CKAN_SITE_URL}
      - CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
      - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}

  ckan-harvest-cleanlog:
    container_name: ckan-harvest-cleanlog
    build:
      context: .
    command: /bin/sh -c "while true; do sleep 86400; ckan -c /srv/app/production.ini harvester clean_harvest_log; done"
    networks:
      - frontend
      - backend
    depends_on:
      - db
      - ckan
    env_file:
      - ./.ckan-env
    environment:
      - CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan
      - CKAN_DATASTORE_WRITE_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/datastore
      - CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore
      - CKAN_SOLR_URL=http://solr:8983/solr/ckan
      - CKAN_REDIS_URL=redis://redis:6379/1
      - CKAN_SITE_URL=${CKAN_SITE_URL}
      - CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
      - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}

  datapusher:
    container_name: datapusher
    image: ghcr.io/keitaroinc/datapusher:${DATAPUSHER_VERSION}
    networks:
      - frontend
      - backend
    ports:
      - "8000:8000"
    environment:
      - DATAPUSHER_MAX_CONTENT_LENGTH=${DATAPUSHER_MAX_CONTENT_LENGTH}
      - DATAPUSHER_CHUNK_SIZE=${DATAPUSHER_CHUNK_SIZE}
      - DATAPUSHER_CHUNK_INSERT_ROWS=${DATAPUSHER_CHUNK_INSERT_ROWS}
      - DATAPUSHER_DOWNLOAD_TIMEOUT=${DATAPUSHER_DOWNLOAD_TIMEOUT}
      - DATAPUSHER_SSL_VERIFY=${DATAPUSHER_SSL_VERIFY}
      - DATAPUSHER_REWRITE_RESOURCES=${DATAPUSHER_REWRITE_RESOURCES}
      - DATAPUSHER_REWRITE_URL=${DATAPUSHER_REWRITE_URL}

  db:
    container_name: db
    build:
      context: ../../compose
      dockerfile: postgresql/Dockerfile
      args:
        - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
        - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
    networks:
      - backend
    environment:
      - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
    volumes:
      - pg_data:/var/lib/postgresql/data
    healthcheck:
      test: ["CMD", "pg_isready", "-U", "ckan"]

  solr:
    container_name: solr
    build:
      context: ../../compose
      dockerfile: solr/Dockerfile
      args:
        - CKAN_VERSION=${CKAN_VERSION}
    networks:
      - backend
    volumes:
      - solr_data:/opt/solr/server/solr/ckan/data

  redis:
    container_name: redis
    image: redis:${REDIS_VERSION}
    networks:
      - backend

networks:
  frontend:
  backend: