diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 69e090c..f31445a 100755 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,11 +16,11 @@ jobs: steps: - uses: actions/checkout@v2 - run: docker-compose up -d - - name: Wait for MySQL + - name: Wait for PostgreSQL run: | set +e for i in {0..60}; do - docker-compose exec -T chris_store_dev_db mysqladmin -uroot -prootp status 2> /dev/null + docker-compose exec -T chris_store_dev_db psql -U chris -d chris_store_dev -c "select 1" 2> /dev/null if [ "$?" = "0" ]; then dbup=y break @@ -29,11 +29,9 @@ jobs: echo . done if [ "$dbup" != "y" ]; then - echo "::error ::Timed out waiting for MySQL database." + echo "::error ::Timed out waiting for PostgreSQL database." exit 1 fi - - name: Prepare MySQL for tests - run: docker-compose exec -T chris_store_dev_db mysql -uroot -prootp -e 'GRANT ALL PRIVILEGES ON *.* TO "chris"@"%"' - name: Run all tests run: docker-compose exec -T chris_store_dev python manage.py test - run: docker-compose down -v diff --git a/Dockerfile b/Dockerfile index d4e756b..5099836 100755 --- a/Dockerfile +++ b/Dockerfile @@ -49,10 +49,10 @@ RUN apt-get update \ && export LC_ALL=en_US.UTF-8 \ && locale-gen en_US.UTF-8 \ && dpkg-reconfigure locales \ - && apt-get install -y libssl-dev libmysqlclient-dev \ + && apt-get install -y libssl-dev libpq-dev \ && apt-get install -y apache2 apache2-dev \ - && pip install --upgrade pip \ - && pip install -r ${REQPATH}/${ENVIRONMENT}.txt \ + && pip install --upgrade pip \ + && pip install -r ${REQPATH}/${ENVIRONMENT}.txt \ && useradd -l -u $UID -ms /bin/bash localuser # Start as user localuser diff --git a/README.md b/README.md index 7d28176..6ed2ebe 100755 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ ![License][license-badge] ![Last Commit][last-commit-badge] -Back end for the ChRIS store. This is a Django-MySQL project that houses descriptions of ChRIS plugin-apps and workflows for registering to a ChRIS CUBE instance. +Back end for the ChRIS store. This is a Django-PostgreSQL project that houses descriptions of ChRIS plugin-apps and workflows for registering to a ChRIS CUBE instance. ## ChRIS store development, testing and deployment diff --git a/docker-compose_dev.yml b/docker-compose_dev.yml index c7dd974..43b7b4c 100755 --- a/docker-compose_dev.yml +++ b/docker-compose_dev.yml @@ -35,17 +35,15 @@ services: role: "Development server" chris_store_dev_db: - image: mysql:8 - command: --collation-server=utf8mb4_0900_as_cs + image: postgres:13 volumes: - - chris_store_dev_db_data:/var/lib/mysql:z + - chris_store_dev_db_data:/var/lib/postgresql/data:z environment: - - MYSQL_ROOT_PASSWORD=rootp - - MYSQL_DATABASE=chris_store_dev - - MYSQL_USER=chris - - MYSQL_PASSWORD=Chris1234 + - POSTGRES_DB=chris_store_dev + - POSTGRES_USER=chris + - POSTGRES_PASSWORD=Chris1234 labels: - name: "ChRIS_store MySQL Database" + name: "ChRIS_store PostgreSQL Database" role: "Development database" swift_service_dev: diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 94938fa..f3ec1af 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -2,9 +2,9 @@ if [[ "$DJANGO_DB_MIGRATE" == 'on' ]]; then if [[ "$DJANGO_SETTINGS_MODULE" == 'config.settings.local' ]]; then - python migratedb.py -u root -p rootp --host chris_store_dev_db --noinput + python migratedb.py -u chris -p Chris1234 -d chris_store_dev --host chris_store_dev_db --noinput elif [[ "$DJANGO_SETTINGS_MODULE" == 'config.settings.production' ]]; then - python migratedb.py -u root -p $MYSQL_ROOT_PASSWORD --host $DATABASE_HOST --noinput + python migratedb.py -u $POSTGRES_USER -p $POSTGRES_PASSWORD -d $POSTGRES_DB --host $DATABASE_HOST --noinput fi fi diff --git a/make.sh b/make.sh index c18657e..1eeabdd 100755 --- a/make.sh +++ b/make.sh @@ -42,8 +42,8 @@ if [[ "$1" == 'up' ]]; then title -d 1 "Pulling core containers where needed..." printf "${LightCyan}%40s${Green}%-40s${Yellow}\n" \ - "docker pull" " mysql:8" | ./boxes.sh - docker pull mysql:8 | ./boxes.sh + "docker pull" " postgres:13" | ./boxes.sh + docker pull postgres:13 | ./boxes.sh echo "" | ./boxes.sh printf "${LightCyan}%40s${Green}%-40s${Yellow}\n" \ "docker pull " "fnndsc/docker-swift-onlyone" | ./boxes.sh @@ -64,14 +64,14 @@ if [[ "$1" == 'up' ]]; then windowBottom title -d 1 "Waiting until ChRIS store database server is ready to accept connections..." - docker-compose -f docker-compose_dev.yml exec chris_store_dev_db sh -c 'while ! mysqladmin -uroot -prootp status 2> /dev/null; do sleep 5; done;' >& dc.out > /dev/null + echo "This might take a few seconds..." | ./boxes.sh ${Yellow} + windowBottom + docker-compose -f docker-compose_dev.yml exec chris_store_dev_db sh -c 'while ! psql -U chris -d chris_store_dev -c "select 1" 2> /dev/null; do sleep 5; done;' >& dc.out > /dev/null echo -en "\033[2A\033[2K" - sed -E 's/[[:alnum:]]+:/\n&/g' dc.out | ./boxes.sh - # Give all permissions to chris user on the test DB. This is required for the Django tests: - echo "Granting user all DB permissions...." | ./boxes.sh ${LightCyan} - echo "This is required for the Django tests." | ./boxes.sh ${LightCyan} - docker-compose -f docker-compose_dev.yml exec chris_store_dev_db mysql -uroot -prootp -e 'GRANT ALL PRIVILEGES ON test_chris_store_dev.* TO "chris"@"%"' >& dc.out > /dev/null - cat dc.out | ./boxes.sh + cat dc.out | ./boxes.sh ${LightGreen} + echo "" | ./boxes.sh + echo "ChRIS store database is ready to accept connections" | ./boxes.sh ${LightGreen} + echo "" | ./boxes.sh windowBottom title -d 1 "Running Django Unit tests..." diff --git a/requirements/base.txt b/requirements/base.txt index 80c3aae..58fbec7 100755 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -2,8 +2,7 @@ Django==2.2.24 django-filter==2.4.0 djangorestframework==3.12.4 django-cors-middleware==1.5.0 -mysqlclient==2.0.3 -mysql-connector-python==8.0.25 +psycopg2==2.8.6 python-swiftclient==3.12.0 django-storage-swift==1.2.19 mod-wsgi==4.8.0 diff --git a/store_backend/config/settings/common.py b/store_backend/config/settings/common.py index a488cad..ce1a8fd 100755 --- a/store_backend/config/settings/common.py +++ b/store_backend/config/settings/common.py @@ -98,7 +98,7 @@ DATABASES = { 'default': { - 'ENGINE': 'mysql.connector.django', + 'ENGINE': 'django.db.backends.postgresql', } } diff --git a/store_backend/config/settings/local.py b/store_backend/config/settings/local.py index 5822bb4..646194e 100755 --- a/store_backend/config/settings/local.py +++ b/store_backend/config/settings/local.py @@ -81,7 +81,7 @@ DATABASES['default']['PASSWORD'] = 'Chris1234' DATABASES['default']['TEST'] = {'NAME': 'test_chris_store_dev'} DATABASES['default']['HOST'] = 'chris_store_dev_db' -DATABASES['default']['PORT'] = '3306' +DATABASES['default']['PORT'] = '5432' # Mail settings # ------------------------------------------------------------------------------ diff --git a/store_backend/config/settings/production.py b/store_backend/config/settings/production.py index ab55688..21c624e 100755 --- a/store_backend/config/settings/production.py +++ b/store_backend/config/settings/production.py @@ -43,9 +43,9 @@ def get_secret(setting, secret_type=env): # DATABASE CONFIGURATION # ------------------------------------------------------------------------------ # Raises ImproperlyConfigured exception if DATABASE_URL not in os.environ -DATABASES['default']['NAME'] = get_secret('MYSQL_DATABASE') -DATABASES['default']['USER'] = get_secret('MYSQL_USER') -DATABASES['default']['PASSWORD'] = get_secret('MYSQL_PASSWORD') +DATABASES['default']['NAME'] = get_secret('POSTGRES_DB') +DATABASES['default']['USER'] = get_secret('POSTGRES_USER') +DATABASES['default']['PASSWORD'] = get_secret('POSTGRES_PASSWORD') DATABASES['default']['HOST'] = get_secret('DATABASE_HOST') DATABASES['default']['PORT'] = get_secret('DATABASE_PORT') diff --git a/store_backend/migratedb.py b/store_backend/migratedb.py index a24afd7..ae78039 100755 --- a/store_backend/migratedb.py +++ b/store_backend/migratedb.py @@ -2,7 +2,7 @@ import time import sys -import MySQLdb +import psycopg2 from argparse import ArgumentParser # django needs to be loaded @@ -15,6 +15,7 @@ parser = ArgumentParser(description="Check database service connection") parser.add_argument('-u', '--user', help="Database user name") parser.add_argument('-p', '--password', help="Database user password") +parser.add_argument('-d', '--database', help="Database name") parser.add_argument('--host', help="Database host") parser.add_argument('--max-attempts', type=int, dest='attempts', help="Maximum number of connection attempts") @@ -30,9 +31,8 @@ db = None while max_tries > 0 and db is None: try: - db = MySQLdb.connect(user=args.user, - passwd=args.password, - host=host) + db = psycopg2.connect(host=host, user=args.user, password=args.password, + dbname=args.database) except Exception: time.sleep(5) max_tries -= 1 diff --git a/swarm/prod/docker-compose.yml b/swarm/prod/docker-compose.yml index 0874d71..364518c 100755 --- a/swarm/prod/docker-compose.yml +++ b/swarm/prod/docker-compose.yml @@ -15,7 +15,7 @@ services: image: fnndsc/chris_store environment: - DATABASE_HOST=chris_store_db - - DATABASE_PORT=3306 + - DATABASE_PORT=5432 - SWIFT_AUTH_URL=http://swift_service:8080/auth/v1.0 env_file: - ./secrets/.chris_store.env @@ -34,18 +34,17 @@ services: role: "Production server using Apache's mod_wsgi" chris_store_db: - image: mysql:8 - command: --collation-server=utf8mb4_0900_as_cs + image: postgres:13 volumes: - - chris_store_db_data:/var/lib/mysql + - chris_store_db_data:/var/lib/postgresql/data env_file: - ./secrets/.chris_store_db.env deploy: restart_policy: condition: on-failure labels: - name: "ChRIS_store MySQL Database" - role: "Production MySQL database" + name: "ChRIS_store PostgreSQL Database" + role: "Production PostgreSQL database" swift_service: image: fnndsc/docker-swift-onlyone