Support CKAN 2.9 with Python 3

.circleci/config.yml

@@ -10,17 +10,19 @@ jobs:
             pip install pip==20.3.3
             pip install flake8
             flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-  
+
   test_local_docker:
     working_directory: ~/ckanext-datajson
     machine: true
+    environment:
+      COMPOSE_FILE: docker-compose.legacy.yml
     steps:
       - checkout
       - run:
           name: Start CKAN
           command: |
-            docker-compose build
-            docker-compose up -d
+            make build
+            docker-compose -f $COMPOSE_FILE up -d
       - run:
           name: install dockerize
           command: wget https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz && sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz && rm dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz
@@ -32,10 +34,10 @@ jobs:
       - run:
           name: Test extension
           command: |
-            docker-compose logs db
-            docker-compose logs ckan
-            docker-compose exec ckan /bin/bash -c "nosetests --ckan --with-pylons=src/ckan/test-catalog-next.ini src_extensions/datajson/ckanext/datajson/tests"
-  
+            docker-compose -f $COMPOSE_FILE logs db
+            docker-compose -f $COMPOSE_FILE logs ckan
+            make test-legacy
+
   build_ckan_28:
     working_directory: ~/ckanext-datajson
     machine:
@@ -45,66 +47,6 @@ jobs:
       - run:
           name: install and test
           command: bin/travis-build-2.8.bash
-
-  build_ckan_23:
-    working_directory: ~/ckanext-datajson
-    machine:
-      image: circleci/classic:201708-01
-    environment:
-      CKANVERSION=2.3
-    steps:
-      - checkout
-      - run:
-          name: install
-          command: bin/travis-build.bash
-      - run:
-          name: test
-          command: bin/travis-run.sh
-
-  build_gsa_ckan_inventory:
-    working_directory: ~/ckanext-datajson
-    machine:
-      image: circleci/classic:201708-01
-    environment:
-      CKANVERSION=inventory
-    steps:
-      - checkout
-      - run:
-          name: install
-          command: bin/travis-build-inventory.bash
-      - run:
-          name: test
-          command: bin/travis-run-inventory.sh
-
-  build_gsa_ckan_inventory_next:
-    working_directory: ~/ckanext-datajson
-    machine:
-      image: circleci/classic:201708-01
-    environment:
-      CKANVERSION=inventory-next
-    steps:
-      - checkout
-      - run:
-          name: install
-          command: bin/travis-build-inventory.bash
-      - run:
-          name: test
-          command: bin/travis-run-inventory.sh
-
-  build_gsa_ckan_datagov:
-    working_directory: ~/ckanext-datajson
-    machine:
-      image: circleci/classic:201708-01
-    environment:
-      CKANVERSION=datagov
-    steps:
-      - checkout
-      - run:
-          name: install
-          command: bin/travis-build.bash
-      - run:
-          name: test
-          command: bin/travis-run.sh
 
 workflows:
   version: 2
@@ -120,12 +62,3 @@ workflows:
       - build_ckan_28:
           requires:
             - lint
-      - build_ckan_23:
-          requires:
-            - lint
-      - build_gsa_ckan_inventory:
-          requires:
-            - lint
-      - build_gsa_ckan_datagov:
-          requires:
-            - lint

.env

@@ -12,12 +12,10 @@ CKAN_SYSADMIN_EMAIL=your_email@example.com
 TZ=UTC
 
 # Database connections (TODO: avoid duplication)
-CKAN_SQLALCHEMY_URL=postgresql://ckan:ckan@db/ckan
 CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore
 CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore
 
 # Test database connections
-TEST_CKAN_SQLALCHEMY_URL=postgres://ckan:ckan@db/ckan_test
 TEST_CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore_test
 TEST_CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore_test
 
@@ -32,16 +30,14 @@ TEST_CKAN_REDIS_URL=redis://redis:6379/1
 
 # Core settings
 CKAN__STORAGE_PATH=/var/lib/ckan
+CKAN__PLUGINS: datajson harvest datajson_harvest
 
 CKAN_SMTP_SERVER=smtp.corporateict.domain:25
 CKAN_SMTP_STARTTLS=True
 CKAN_SMTP_USER=user
 CKAN_SMTP_PASSWORD=pass
 CKAN_SMTP_MAIL_FROM=ckan@localhost
 
-# Extensions
-CKAN__PLUGINS=envvars image_view text_view recline_view datastore datapusher datagov_harvest ckan_harvester geodatagov datajson datajson_harvest geodatagov_miscs z3950_harvester arcgis_harvester geodatagov_geoportal_harvester waf_harvester_collection geodatagov_csw_harvester geodatagov_doc_harvester geodatagov_waf_harvester spatial_metadata spatial_query report qa archiver datagovtheme datagovcatalog googleanalyticsbasic dcat dcat_json_interface structured_data
-
 # Harvest settings
 CKAN__HARVEST__MQ__TYPE=redis
 CKAN__HARVEST__MQ__HOSTNAME=redis
@@ -67,5 +63,5 @@ CKAN___BROKER_HOST=redis://redis/1
 CKAN___CELERY_RESULT_BACKEND=redis
 CKAN___REDIS_HOST=redis
 CKAN___REDIS_PORT=6379
-CKAN___REDIS_DB=0
+CKAN___REDIS_DB=1
 CKAN___REDIS_CONNECT_RETRY=True

.github/workflows/test.yml

@@ -0,0 +1,68 @@
+name: Tests
+on: [push, pull_request]
+env:
+  CODE_COVERAGE_THRESHOLD_REQUIRED: 70
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.6'
+      - name: Install requirements
+        run: pip install flake8 pycodestyle
+      - name: Check syntax
+        run: flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ckan
+      - name: Run flake8
+        run: flake8 . --count --show-source --statistics --exclude ckan
+
+  test:
+    strategy:
+      matrix:
+        ckan-version: [2.9, 2.8]
+      fail-fast: false
+
+    name: CKAN ${{ matrix.ckan-version }}
+    runs-on: ubuntu-latest
+    container:
+      image: openknowledge/ckan-dev:${{ matrix.ckan-version }}
+    services:
+      solr:
+        image: ckan/ckan-solr-dev:${{ matrix.ckan-version }}
+      postgres:
+        image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }}
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: postgres
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+      redis:
+          image: redis:3
+    env:
+      CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@postgres/ckan_test
+      CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:pass@postgres/datastore_test
+      CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgres/datastore_test
+      CKAN_SOLR_URL: http://solr:8983/solr/ckan
+      CKAN_REDIS_URL: redis://redis:6379/1
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install requirements
+      run: |
+        pip install -r requirements.txt -r dev-requirements.txt -e .
+    - name: Setup extension (CKAN >= 2.9)
+      if: ${{ matrix.ckan-version != '2.7' && matrix.ckan-version != '2.8' }}
+      run: |
+        ckan -c test.ini db init
+    - name: Setup extension (CKAN < 2.9)
+      if: ${{ matrix.ckan-version == '2.7' || matrix.ckan-version == '2.8' }}
+      run: |
+        paster --plugin=ckan db init -c test.ini
+    - name: Run tests
+      run: |
+        set -o pipefail
+        pytest --ckan-ini=test.ini --cov=ckanext.datajson --cov-fail-under=${CODE_COVERAGE_THRESHOLD_REQUIRED} --disable-warnings ckanext/datajson/tests > pytest-coverage.txt
+    - name: Report test coverage
+      if: ${{ always() }}
+      uses: coroo/pytest-coverage-commentator@v1.0.2

.gitignore

@@ -1,5 +1,6 @@
 **.pyc
 build
+.coverage
 ckanext_datajson.egg-info
 .DS_Store
 ckanext/datajson/export_map/*.map.json

Dockerfile

@@ -0,0 +1,8 @@
+ARG CKAN_VERSION=2.8
+FROM openknowledge/ckan-dev:${CKAN_VERSION}
+
+COPY . /app
+WORKDIR /app
+
+# python cryptography takes a while to build
+RUN pip install -r requirements.txt -r dev-requirements.txt -e .

Makefile

@@ -0,0 +1,33 @@
+CKAN_VERSION ?= 2.8
+COMPOSE_FILE ?= docker-compose.yml
+
+build: ## Build the docker containers
+	CKAN_VERSION=$(CKAN_VERSION) docker-compose -f $(COMPOSE_FILE) build
+
+lint: ## Lint the code
+	@# our linting only runs with python3
+	@# TODO use CKAN_VERSION make variable once 2.8 is deprecated
+	CKAN_VERSION=2.9 docker-compose -f docker-compose.new.yml run --rm app flake8 . --count --show-source --statistics --exclude ckan
+
+clean: ## Clean workspace and containers
+	find . -name *.pyc -delete
+	CKAN_VERSION=$(CKAN_VERSION) docker-compose -f $(COMPOSE_FILE) down -v --remove-orphan
+
+test: ## Run tests in a new container
+	CKAN_VERSION=$(CKAN_VERSION) docker-compose -f $(COMPOSE_FILE) run --rm app ./test.sh
+
+test-legacy: ## Run legacy nose tests in an existing container
+	@# TODO wait for CKAN to be up; use docker-compose run instead
+	docker-compose exec ckan /bin/bash -c "nosetests --ckan --with-pylons=src/ckan/test-catalog-next.ini src_extensions/datajson/ckanext/datajson/tests/nose"
+
+up: ## Start the containers
+	CKAN_VERSION=$(CKAN_VERSION) docker-compose -f $(COMPOSE_FILE) up
+
+
+.DEFAULT_GOAL := help
+.PHONY: build clean help lint test test-legacy up
+
+# Output documentation for top-level targets
+# Thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
+help: ## This help
+	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-10s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)