From dbf55ac8529503e8552e4aed8a06379eb9184719 Mon Sep 17 00:00:00 2001 From: Leo Cai Date: Tue, 12 Dec 2023 15:39:28 +0800 Subject: [PATCH 1/3] compatibility test after release --- .github/workflows/release_build.yml | 20 ++++++++ tests/proton_ci/requirements.txt | 3 +- tests/proton_ci/run_compatibility_tests.py | 57 ++++++++++++++++++++++ 3 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 tests/proton_ci/run_compatibility_tests.py diff --git a/.github/workflows/release_build.yml b/.github/workflows/release_build.yml index 8dde0bf6c68..c08dfd9a531 100644 --- a/.github/workflows/release_build.yml +++ b/.github/workflows/release_build.yml @@ -177,6 +177,26 @@ jobs: GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + Trigger_Compatibility_Tests: + needs: [CommitTag] + runs-on: ubuntu-latest + steps: + - name: checkout repo + uses: actions/checkout@v4 + - name: setup python + uses: actions/setup-python@v4 + with: + python-version: '3.8' + - name: run command + run: | + ./release --version patch + export PROTON_VERSION=`grep "SET(VERSION_STRING" $GITHUB_WORKSPACE/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_STRING \(.*\)$/\1/' | sed 's/[) ].*//'` + echo "PROTON_VERSION=$PROTON_VERSION" + cd $GITHUB_WORKSPACE/tests/proton_ci + pip install -r requirements.txt + python run_compatibility_tests.py + env: + GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} Build_Darwin_X86_64: uses: timeplus-io/proton/.github/workflows/run_command.yml@develop with: diff --git a/tests/proton_ci/requirements.txt b/tests/proton_ci/requirements.txt index 7007e99cfb0..082dfabd710 100644 --- a/tests/proton_ci/requirements.txt +++ b/tests/proton_ci/requirements.txt @@ -2,4 +2,5 @@ PyGithub>=1.58.0 unidiff>=0.7.0 boto3==1.20.4 timeplus==1.1.2 -urllib3==1.26.7 \ No newline at end of file +urllib3==1.26.7 +requests>=2.26.0 diff --git a/tests/proton_ci/run_compatibility_tests.py b/tests/proton_ci/run_compatibility_tests.py new file mode 100644 index 00000000000..38ff7a611d4 --- /dev/null +++ b/tests/proton_ci/run_compatibility_tests.py @@ -0,0 +1,57 @@ +import json +import time +from typing import List, Any + +import requests +from env_helper import GH_PERSONAL_ACCESS_TOKEN, PROTON_VERSION +import re + +HEADERS = { + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {GH_PERSONAL_ACCESS_TOKEN}", + "X-GitHub-Api-Version": "2022-11-28" +} +MAX_TEST_VERSION_NUM = 5 + + +def valid_version_tag(tag_list: List[str]) -> Any: + for tag in tag_list: + if re.match(r"\d+\.\d+\.\d+", tag) is not None: + return tag + return None + + +if __name__ == "__main__": + proton_image_version_list = requests.get( + url="https://api.github.com/orgs/timeplus-io/packages/container/proton/versions?per_page=100", + headers=HEADERS + ).json() + valid_version_list = [] + current_version = "" + for version_info in proton_image_version_list: + created_at = time.mktime(time.strptime(version_info['created_at'], "%Y-%m-%dT%H:%M:%SZ")) + tags = version_info['metadata']['container']['tags'] + tag_name = valid_version_tag(tags) + if tag_name is None: + continue + if tag_name == PROTON_VERSION or tag_name == PROTON_VERSION + "-rc": + current_version = tag_name + continue + valid_version_list.append((created_at, tag_name)) + valid_version_list.sort(key=lambda version_tuple: -int(version_tuple[0])) + valid_version_list = valid_version_list[:min(len(valid_version_list), MAX_TEST_VERSION_NUM)] + if current_version == "": + print("Cannot find current proton version.") + for _, version in valid_version_list: + response = requests.post( + "https://api.github.com/repos/timeplus-io/proton/actions/workflows/compatibility_test.yml/dispatches", + headers=HEADERS, + data=json.dumps({ + "ref": "develop", + "inputs": { + "source": version, + "target": current_version + } + }) + ) + print(f'target="{current_version}", source="{version}", status_code="{response.status_code}"') From b41446458abaff8a2ea2c003bacf8488d6f53bc4 Mon Sep 17 00:00:00 2001 From: Leo Cai Date: Tue, 12 Dec 2023 17:03:08 +0800 Subject: [PATCH 2/3] compatibility test for arm and x64, fix code of trigger compatibility test after release build --- .github/workflows/compatibility_test.yml | 144 +++++++++++++++------ .github/workflows/release_build.yml | 36 +++--- tests/proton_ci/run_compatibility_tests.py | 38 +++--- 3 files changed, 143 insertions(+), 75 deletions(-) diff --git a/.github/workflows/compatibility_test.yml b/.github/workflows/compatibility_test.yml index 64ec3d3da1f..54a5dc08846 100644 --- a/.github/workflows/compatibility_test.yml +++ b/.github/workflows/compatibility_test.yml @@ -12,14 +12,10 @@ on: required: false default: '' jobs: - upload_data: - uses: timeplus-io/proton/.github/workflows/run_command.yml@develop - with: - ec2-instance-type: ${{ vars.X64_INSTANCE_TYPE }} - ec2-image-id: ${{ vars.X64_TEST_AMI }} - ec2-volume-size: '30' - submodules: false - timeout: 30 + prepare_upload_data: + if: ${{ github.event.inputs.source == '' || github.event.inputs.source == github.event.inputs.target }} + runs-on: ubuntu-latest + outputs: command: | export PROTON_VERSION=${{ github.event.inputs.target }} @@ -49,38 +45,25 @@ jobs: cd $GITHUB_WORKSPACE tar -zcvf $PROTON_VERSION.tar.gz data - aws s3 cp --no-progress $PROTON_VERSION.tar.gz s3://tp-internal/proton/compatibility/oss/ - secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: ${{ secrets.AWS_REGION }} - AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} - GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }} - TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }} - TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }} - - compatibility_test: - if: ${{ github.event.inputs.source!='' }} - uses: timeplus-io/proton/.github/workflows/run_command.yml@develop - with: - ec2-instance-type: ${{ vars.X64_INSTANCE_TYPE }} - ec2-image-id: ${{ vars.X64_TEST_AMI }} - ec2-volume-size: '30' - submodules: false - timeout: 30 + aws s3 cp --no-progress $PROTON_VERSION.tar.gz s3://tp-internal/proton/compatibility/oss/$ARCH/ + steps: + - name: display command + run: | + echo 'command: ${{ steps.set_command.outputs.command }}' + prepare_compatibility_test: + if: ${{ github.event.inputs.source != '' && github.event.inputs.source != github.event.inputs.target }} + runs-on: ubuntu-latest + outputs: command: | export TARGET_VERSION=${{ github.event.inputs.target }} export SOURCE_VERSION=${{ github.event.inputs.source }} export PROTON_VERSION=$TARGET_VERSION - + # prepare data cd $GITHUB_WORKSPACE - aws s3 cp --no-progress s3://tp-internal/proton/compatibility/oss/$SOURCE_VERSION.tar.gz . + aws s3 cp --no-progress s3://tp-internal/proton/compatibility/oss/$ARCH/$SOURCE_VERSION.tar.gz . tar -zxvf $SOURCE_VERSION.tar.gz - + cd $GITHUB_WORKSPACE/tests/stream # make virtualenv @@ -90,20 +73,82 @@ jobs: apt install python3-venv -y python -m venv env source env/bin/activate - + pip install --upgrade pip - + # FIXME: remove this line after pyyaml community fixed install bug pip install pyyaml==5.3.1 - + # FIXME(yokofly): docker 7.0.0 introduce a breaking change # https://github.com/docker/docker-py/issues/3194 pip install docker==6.1.3 - + pip install -r helpers/requirements.txt - + bash test_compatibility/basic_tests.sh bash test_compatibility/extra_tests.sh + steps: + - name: display command + run: | + echo 'command: ${{ steps.set_command.outputs.command }}' + upload_data_x64: + needs: [prepare_upload_data] + uses: timeplus-io/proton/.github/workflows/run_command.yml@develop + with: + ec2-instance-type: ${{ vars.X64_INSTANCE_TYPE }} + ec2-image-id: ${{ vars.X64_TEST_AMI }} + ec2-volume-size: '30' + submodules: false + timeout: 30 + arch: ${{ vars.X64_ARCH }} + command: | + ${{ needs.prepare_upload_data.outputs.command }} + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.AWS_REGION }} + AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} + GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }} + TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }} + TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }} + upload_data_arm: + needs: [prepare_upload_data] + uses: timeplus-io/proton/.github/workflows/run_command.yml@develop + with: + ec2-instance-type: ${{ vars.ARM_INSTANCE_TYPE }} + ec2-image-id: ${{ vars.ARM_TEST_AMI }} + ec2-volume-size: '30' + submodules: false + timeout: 30 + arch: ${{ vars.ARM_ARCH }} + command: | + ${{ needs.prepare_upload_data.outputs.command }} + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.AWS_REGION }} + AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} + GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }} + TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }} + TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }} + compatibility_test_x64: + needs: [prepare_compatibility_test] + uses: timeplus-io/proton/.github/workflows/run_command.yml@develop + with: + ec2-instance-type: ${{ vars.X64_INSTANCE_TYPE }} + ec2-image-id: ${{ vars.X64_TEST_AMI }} + ec2-volume-size: '30' + submodules: false + timeout: 30 + arch: ${{ vars.X64_ARCH }} + command: | + ${{ needs.prepare_compatibility_test.outputs.command }} secrets: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -115,3 +160,26 @@ jobs: TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }} TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }} TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }} + compatibility_test_arm: + needs: [prepare_compatibility_test] + uses: timeplus-io/proton/.github/workflows/run_command.yml@develop + with: + ec2-instance-type: ${{ vars.ARM_INSTANCE_TYPE }} + ec2-image-id: ${{ vars.ARM_TEST_AMI }} + ec2-volume-size: '30' + submodules: false + timeout: 30 + arch: ${{ vars.ARM_ARCH }} + command: | + ${{ needs.prepare_compatibility_test.outputs.command }} + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.AWS_REGION }} + AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} + GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }} + TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }} + TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }} \ No newline at end of file diff --git a/.github/workflows/release_build.yml b/.github/workflows/release_build.yml index c08dfd9a531..763d2755bb6 100644 --- a/.github/workflows/release_build.yml +++ b/.github/workflows/release_build.yml @@ -169,6 +169,22 @@ jobs: -H "X-GitHub-Api-Version: 2022-11-28" \ https://api.github.com/repos/timeplus-io/proton/actions/workflows/manual_trigger_test.yml/dispatches \ -d "{\"ref\":\"develop\",\"inputs\":{\"arch\": \"arm\", \"tag\":\"$PROTON_TAG-rc\"}}\"" + + # trigger compatibility test + export PROTON_VERSION=$PROTON_TAG + cd $GITHUB_WORKSPACE/tests/proton_ci + + # make virtualenv + ln -s /usr/bin/python3 /usr/bin/python + apt-get update + systemctl stop unattended-upgrades + apt install python3-venv -y + python -m venv env + source env/bin/activate + pip install --upgrade pip + + pip install -r requirements.txt + python run_compatibility_tests.py secrets: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -177,26 +193,6 @@ jobs: GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - Trigger_Compatibility_Tests: - needs: [CommitTag] - runs-on: ubuntu-latest - steps: - - name: checkout repo - uses: actions/checkout@v4 - - name: setup python - uses: actions/setup-python@v4 - with: - python-version: '3.8' - - name: run command - run: | - ./release --version patch - export PROTON_VERSION=`grep "SET(VERSION_STRING" $GITHUB_WORKSPACE/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_STRING \(.*\)$/\1/' | sed 's/[) ].*//'` - echo "PROTON_VERSION=$PROTON_VERSION" - cd $GITHUB_WORKSPACE/tests/proton_ci - pip install -r requirements.txt - python run_compatibility_tests.py - env: - GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} Build_Darwin_X86_64: uses: timeplus-io/proton/.github/workflows/run_command.yml@develop with: diff --git a/tests/proton_ci/run_compatibility_tests.py b/tests/proton_ci/run_compatibility_tests.py index 38ff7a611d4..e596d6ce6e0 100644 --- a/tests/proton_ci/run_compatibility_tests.py +++ b/tests/proton_ci/run_compatibility_tests.py @@ -1,4 +1,5 @@ import json +import logging import time from typing import List, Any @@ -26,8 +27,9 @@ def valid_version_tag(tag_list: List[str]) -> Any: url="https://api.github.com/orgs/timeplus-io/packages/container/proton/versions?per_page=100", headers=HEADERS ).json() + valid_version_list = [] - current_version = "" + current_version = "latest" for version_info in proton_image_version_list: created_at = time.mktime(time.strptime(version_info['created_at'], "%Y-%m-%dT%H:%M:%SZ")) tags = version_info['metadata']['container']['tags'] @@ -36,22 +38,24 @@ def valid_version_tag(tag_list: List[str]) -> Any: continue if tag_name == PROTON_VERSION or tag_name == PROTON_VERSION + "-rc": current_version = tag_name - continue valid_version_list.append((created_at, tag_name)) + valid_version_list.sort(key=lambda version_tuple: -int(version_tuple[0])) - valid_version_list = valid_version_list[:min(len(valid_version_list), MAX_TEST_VERSION_NUM)] - if current_version == "": - print("Cannot find current proton version.") + valid_version_list = valid_version_list[:min(len(valid_version_list), MAX_TEST_VERSION_NUM + 1)] + for _, version in valid_version_list: - response = requests.post( - "https://api.github.com/repos/timeplus-io/proton/actions/workflows/compatibility_test.yml/dispatches", - headers=HEADERS, - data=json.dumps({ - "ref": "develop", - "inputs": { - "source": version, - "target": current_version - } - }) - ) - print(f'target="{current_version}", source="{version}", status_code="{response.status_code}"') + try: + response = requests.post( + "https://api.github.com/repos/timeplus-io/proton/actions/workflows/compatibility_test.yml/dispatches", + headers=HEADERS, + data=json.dumps({ + "ref": "develop", + "inputs": { + "source": version, + "target": current_version + } + }) + ) + assert response.status_code == 204 + except Exception as e: + logging.error(e) From 287f9f5511a9dee45834aa1c126de5f757e409ad Mon Sep 17 00:00:00 2001 From: Leo Cai Date: Fri, 15 Dec 2023 14:51:25 +0800 Subject: [PATCH 3/3] add test cases --- .github/workflows/compatibility_test.yml | 29 ++-- .../stream/test_compatibility/basic_tests.sh | 13 +- .../basic_tests/example.yaml | 9 ++ .../basic_tests/materialized_view.yaml | 48 +++++++ .../{ => configs}/docker-compose.yaml | 10 +- .../stream/test_compatibility/extra_tests.sh | 15 ++- .../extra_tests/materialized_view.yaml | 81 ++++++++++++ .../stream/test_compatibility/prepare_data.sh | 16 ++- .../prepare_data/materialized_view.yaml | 70 ++++++++++ .../test_compatibility/requirements.txt | 4 + .../run_compatibility_tests.py | 124 ++++++++++++++++++ 11 files changed, 381 insertions(+), 38 deletions(-) create mode 100644 tests/stream/test_compatibility/basic_tests/example.yaml create mode 100644 tests/stream/test_compatibility/basic_tests/materialized_view.yaml rename tests/stream/test_compatibility/{ => configs}/docker-compose.yaml (90%) create mode 100644 tests/stream/test_compatibility/extra_tests/materialized_view.yaml create mode 100644 tests/stream/test_compatibility/prepare_data/materialized_view.yaml create mode 100644 tests/stream/test_compatibility/requirements.txt create mode 100644 tests/stream/test_compatibility/run_compatibility_tests.py diff --git a/.github/workflows/compatibility_test.yml b/.github/workflows/compatibility_test.yml index 54a5dc08846..156bf4b3d28 100644 --- a/.github/workflows/compatibility_test.yml +++ b/.github/workflows/compatibility_test.yml @@ -1,5 +1,5 @@ name: compatibility_test - +run-name: Compatibility Test [${{ (github.event.inputs.source == '' || github.event.inputs.source == github.event.inputs.target) && format('Upload {0}', github.event.inputs.target) || format('{0} -> {1}', github.event.inputs.source, github.event.inputs.target) }}] on: workflow_dispatch: inputs: @@ -31,21 +31,21 @@ jobs: pip install --upgrade pip - # FIXME: remove this line after pyyaml community fixed install bug - pip install pyyaml==5.3.1 - # FIXME(yokofly): docker 7.0.0 introduce a breaking change # https://github.com/docker/docker-py/issues/3194 pip install docker==6.1.3 - pip install -r helpers/requirements.txt + pip install -r test_compatibility/requirements.txt - bash test_compatibility/prepare_data.sh - bash test_compatibility/basic_tests.sh + timeout --foreground 10m bash test_compatibility/prepare_data.sh + timeout --foreground 10m bash test_compatibility/basic_tests.sh cd $GITHUB_WORKSPACE tar -zcvf $PROTON_VERSION.tar.gz data aws s3 cp --no-progress $PROTON_VERSION.tar.gz s3://tp-internal/proton/compatibility/oss/$ARCH/ + + cd $GITHUB_WORKSPACE/tests/stream + timeout --foreground 10m bash test_compatibility/extra_tests.sh steps: - name: display command run: | @@ -75,18 +75,15 @@ jobs: source env/bin/activate pip install --upgrade pip - - # FIXME: remove this line after pyyaml community fixed install bug - pip install pyyaml==5.3.1 - + # FIXME(yokofly): docker 7.0.0 introduce a breaking change # https://github.com/docker/docker-py/issues/3194 pip install docker==6.1.3 + + pip install -r test_compatibility/requirements.txt - pip install -r helpers/requirements.txt - - bash test_compatibility/basic_tests.sh - bash test_compatibility/extra_tests.sh + timeout --foreground 10m bash test_compatibility/basic_tests.sh + timeout --foreground 10m bash test_compatibility/extra_tests.sh steps: - name: display command run: | @@ -182,4 +179,4 @@ jobs: DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }} TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }} - TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }} \ No newline at end of file + TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }} diff --git a/tests/stream/test_compatibility/basic_tests.sh b/tests/stream/test_compatibility/basic_tests.sh index 3f123d371b5..fb8d97badb6 100644 --- a/tests/stream/test_compatibility/basic_tests.sh +++ b/tests/stream/test_compatibility/basic_tests.sh @@ -1,5 +1,10 @@ -docker-compose -f test_compatibility/docker-compose.yaml up -d +set -e +CUR_DIR="$GITHUB_WORKSPACE/tests/stream/test_compatibility" +docker-compose -f "$CUR_DIR/configs/docker-compose.yaml" up -d +docker ps sleep 5 -docker exec proton-server proton client -nm -q "select x from example where _tp_time > earliest_ts() limit 3;" -docker exec proton-server proton client -nm -q "select x from example_external limit 3 settings seek_to='earliest';" -docker-compose -f test_compatibility/docker-compose.yaml down +export TEST_DIR="$CUR_DIR/basic_tests" +python $CUR_DIR/run_compatibility_tests.py +docker exec proton-server pkill proton-server +sleep 10 +docker-compose -f "$CUR_DIR/configs/docker-compose.yaml" down -v diff --git a/tests/stream/test_compatibility/basic_tests/example.yaml b/tests/stream/test_compatibility/basic_tests/example.yaml new file mode 100644 index 00000000000..657355e2265 --- /dev/null +++ b/tests/stream/test_compatibility/basic_tests/example.yaml @@ -0,0 +1,9 @@ +example: + description: This is an example of test_suit + steps: + - cmd: select 1; + query_time: 3 + wait: 1 + query_id: '0000-0000' + expect: + - [1] diff --git a/tests/stream/test_compatibility/basic_tests/materialized_view.yaml b/tests/stream/test_compatibility/basic_tests/materialized_view.yaml new file mode 100644 index 00000000000..966f62df150 --- /dev/null +++ b/tests/stream/test_compatibility/basic_tests/materialized_view.yaml @@ -0,0 +1,48 @@ +alter_stream_add_column: + description: query from a materialized view on stream, and then alter the stream by adding column + steps: + - cmd: | + select (* except _tp_time) from table(test_1_mv) order by col1; + query_time: 3 + expect: + - [ 1, 1.1, 'a1' ] + - [ 2, 2.1, 'b2' ] + - [ 3, 3.1, 'c3' ] + - [ 4, 4.1, 'd4' ] + - [ 5, 5.1, 'e5' ] + - [ 6, 6.1, 'f6' ] + - cmd: | + select (* except _tp_time) from test_1_mv where _tp_time > earliest_ts() order by col1; + query_time: 3 + expect: + - [ 1, 1.1, 'a1' ] + - [ 2, 2.1, 'b2' ] + - [ 3, 3.1, 'c3' ] + - [ 4, 4.1, 'd4' ] + - [ 5, 5.1, 'e5' ] + - [ 6, 6.1, 'f6' ] + +global_aggregation: + description: a materialized view on stream global aggregation + steps: + - cmd: | + select (* except _tp_time) from table(test_2_mv); + query_time: 3 + expect: + - [ 1.1, 6.1, 8 ] + - cmd: | + select (* except _tp_time) from test_2_mv where _tp_time > earliest_ts() limit 1; + expect: + - [ 1.1, 6.1, 8 ] + +func_now: + description: test for function now() + steps: + - cmd: | + select (* except _tp_time) from table(test_3_mv) order by i; + query_time: 3 + expect: + - [ "any_value", 1 ] + - [ "any_value", 2 ] + - [ "any_value", 3 ] + - [ "any_value", 4 ] diff --git a/tests/stream/test_compatibility/docker-compose.yaml b/tests/stream/test_compatibility/configs/docker-compose.yaml similarity index 90% rename from tests/stream/test_compatibility/docker-compose.yaml rename to tests/stream/test_compatibility/configs/docker-compose.yaml index ce7432f27bb..c0242046dd9 100644 --- a/tests/stream/test_compatibility/docker-compose.yaml +++ b/tests/stream/test_compatibility/configs/docker-compose.yaml @@ -8,11 +8,11 @@ services: - $GITHUB_WORKSPACE/data/proton-redp/datas:/var/lib/proton - $GITHUB_WORKSPACE/data/proton-redp/log:/var/log/proton-server ports: - - "13218:3218" # HTTP Streaming - - "18123:8123" # HTTP Snapshot - - "18463:8463" # TCP Streaming - - "15432:5432" # Postgres Snapshot - - "17587:7587" # TCP Snapshot + - "3218:3218" # HTTP Streaming + - "8123:8123" # HTTP Snapshot + - "8463:8463" # TCP Streaming + - "5432:5432" # Postgres Snapshot + - "7587:7587" # TCP Snapshot deploy: replicas: 1 restart_policy: diff --git a/tests/stream/test_compatibility/extra_tests.sh b/tests/stream/test_compatibility/extra_tests.sh index f2611d071eb..88122165b72 100644 --- a/tests/stream/test_compatibility/extra_tests.sh +++ b/tests/stream/test_compatibility/extra_tests.sh @@ -1,7 +1,10 @@ -docker-compose -f test_compatibility/docker-compose.yaml up -d +set -e +CUR_DIR="$GITHUB_WORKSPACE/tests/stream/test_compatibility" +docker-compose -f "$CUR_DIR/configs/docker-compose.yaml" up -d +docker ps sleep 5 -docker exec proton-server proton client -nm -q "insert into example (x) values (4)(5)(6);" -docker exec proton-server proton client -nm -q "select x from example where _tp_time > earliest_ts() limit 6;" -docker exec proton-server proton client -nm -q "insert into example_external (x) values (8)(10)(12);" -docker exec proton-server proton client -nm -q "select x from example_external limit 6 settings seek_to='earliest';" -docker-compose -f test_compatibility/docker-compose.yaml down +export TEST_DIR="$CUR_DIR/extra_tests" +python $CUR_DIR/run_compatibility_tests.py +docker exec proton-server pkill proton-server +sleep 10 +docker-compose -f "$CUR_DIR/configs/docker-compose.yaml" down -v diff --git a/tests/stream/test_compatibility/extra_tests/materialized_view.yaml b/tests/stream/test_compatibility/extra_tests/materialized_view.yaml new file mode 100644 index 00000000000..20133966426 --- /dev/null +++ b/tests/stream/test_compatibility/extra_tests/materialized_view.yaml @@ -0,0 +1,81 @@ +alter_stream_add_column: + description: query from a materialized view on stream, and then alter the stream by adding column + steps: + - cmd: | + insert into test_1_s (* except _tp_time) + values + (7, 7.1, 'g7', 47), (8, 8.1, 'h8', 48), (9, 9.1, 'i9', 49); + wait: 2 + - cmd: | + select (* except _tp_time) from table(test_1_mv) order by col1; + query_time: 3 + expect: + - [ 1, 1.1, 'a1' ] + - [ 2, 2.1, 'b2' ] + - [ 3, 3.1, 'c3' ] + - [ 4, 4.1, 'd4' ] + - [ 5, 5.1, 'e5' ] + - [ 6, 6.1, 'f6' ] + - [ 7, 7.1, 'g7' ] + - [ 8, 8.1, 'h8' ] + - [ 9, 9.1, 'i9' ] + - cmd: | + select (* except _tp_time) from test_1_mv where _tp_time > earliest_ts() order by col1 limit 9; + query_time: 3 + expect: + - [ 1, 1.1, 'a1' ] + - [ 2, 2.1, 'b2' ] + - [ 3, 3.1, 'c3' ] + - [ 4, 4.1, 'd4' ] + - [ 5, 5.1, 'e5' ] + - [ 6, 6.1, 'f6' ] + - [ 7, 7.1, 'g7' ] + - [ 8, 8.1, 'h8' ] + - [ 9, 9.1, 'i9' ] + +global_aggregation: + description: a materialized view on stream global aggregation + steps: + - cmd: | + select (* except _tp_time) from table(test_2_mv); + query_time: 3 + expect: + - [ 1.1, 6.1, 8 ] + - cmd: | + select (* except _tp_time) from test_2_mv where _tp_time > earliest_ts(); + query_time: 3 + expect: + - [ 1.1, 6.1, 8 ] + - cmd: | + insert into test_2_s (* except _tp_time) + values + (7, 7.1), (8, 8.1), (9, 9.1); + wait: 3 + - cmd: | + select (* except _tp_time) from table(test_2_mv) order by cnt_col2; + query_time: 3 + expect: + - [ 1.1, 6.1, 8 ] + - [ 1.1, 9.1, 11 ] + - cmd: | + select (* except _tp_time) from test_2_mv where _tp_time > earliest_ts() order by cnt_col2; + query_time: 3 + expect: + - [ 1.1, 6.1, 8 ] + - [ 1.1, 9.1, 11 ] + +func_now: + description: test for function now() + steps: + - cmd: | + insert into test_3_s (* except _tp_time) values (5); + wait: 3 + - cmd: | + select (* except _tp_time) from table(test_3_mv) order by i; + query_time: 3 + expect: + - [ "any_value", 1 ] + - [ "any_value", 2 ] + - [ "any_value", 3 ] + - [ "any_value", 4 ] + - [ "any_value", 5 ] diff --git a/tests/stream/test_compatibility/prepare_data.sh b/tests/stream/test_compatibility/prepare_data.sh index 7916f8a6892..1b5fe186c2c 100644 --- a/tests/stream/test_compatibility/prepare_data.sh +++ b/tests/stream/test_compatibility/prepare_data.sh @@ -1,8 +1,10 @@ -docker-compose -f test_compatibility/docker-compose.yaml up -d +set -e +CUR_DIR="$GITHUB_WORKSPACE/tests/stream/test_compatibility" +docker-compose -f "$CUR_DIR/configs/docker-compose.yaml" up -d +docker ps sleep 5 -docker exec proton-server proton client -nm -q "create stream example(x int);" -docker exec proton-server proton client -nm -q "insert into example (x) values (1)(2)(3);" -docker exec redpanda rpk topic create example_topic -docker exec proton-server proton client -nm -q "create external stream example_external (x int) settings type='kafka', brokers='stream-store:9092',topic='example_topic',data_format = 'JSONEachRow';" -docker exec proton-server proton client -nm -q "insert into example_external (x) values (2)(4)(6);" -docker-compose -f test_compatibility/docker-compose.yaml down +export TEST_DIR="$CUR_DIR/prepare_data" +python $CUR_DIR/run_compatibility_tests.py +docker exec proton-server pkill proton-server +sleep 10 +docker-compose -f "$CUR_DIR/configs/docker-compose.yaml" down diff --git a/tests/stream/test_compatibility/prepare_data/materialized_view.yaml b/tests/stream/test_compatibility/prepare_data/materialized_view.yaml new file mode 100644 index 00000000000..52699bcf267 --- /dev/null +++ b/tests/stream/test_compatibility/prepare_data/materialized_view.yaml @@ -0,0 +1,70 @@ +alter_stream_add_column: + description: a materialized view on stream, and then alter the stream by adding column + steps: + - cmd: | + create stream test_1_s ( + col1 int, + col2 float, + col3 string + ); + - cmd: | + create materialized view test_1_mv as ( + select + col1, col2, col3 + from + test_1_s + ); + - cmd: | + insert into test_1_s (* except _tp_time) + values + (1, 1.1, 'a1'), (2, 2.1, 'b2'), (3, 3.1, 'c3'); + - cmd: | + alter stream test_1_s add column col4 uint64; + - cmd: | + insert into test_1_s (* except _tp_time) + values + (4, 4.1, 'd4', 44), (5, 5.1, 'e5', 45), (6, 6.1, 'f6', 46); + +global_aggregation: + description: a materialized view on stream global aggregation + steps: + - cmd: | + create stream test_2_s ( + col1 int, + col2 float64 + ); + - cmd: | + create materialized view test_2_mv as( + select + min(col2) as min_col2, + max(col2) as max_col2, + count(col2) as cnt_col2 + from test_2_s + ); + - cmd: | + insert into test_2_s (* except _tp_time) + values + (1, 1.1), (2, 2.1), (3, 3.1), (4, 4.1), (5, 5.1), (6, 6.1), (3, 3.1), (4, 4.1); + +func_now: + description: test for function now() + steps: + - cmd: | + create stream test_3_s ( + i int + ); + - cmd: | + create materialized view test_3_mv as ( + select + now(), i + from + test_3_s + ); + - cmd: | + insert into test_3_s (* except _tp_time) values (1); + - cmd: | + insert into test_3_s (* except _tp_time) values (2); + - cmd: | + insert into test_3_s (* except _tp_time) values (3); + - cmd: | + insert into test_3_s (* except _tp_time) values (4); diff --git a/tests/stream/test_compatibility/requirements.txt b/tests/stream/test_compatibility/requirements.txt new file mode 100644 index 00000000000..05fbb2ae684 --- /dev/null +++ b/tests/stream/test_compatibility/requirements.txt @@ -0,0 +1,4 @@ +docker-compose==1.29.2 +proton-driver>=0.2.9 +ddt>=1.7.0 +pyyaml==5.3.1 diff --git a/tests/stream/test_compatibility/run_compatibility_tests.py b/tests/stream/test_compatibility/run_compatibility_tests.py new file mode 100644 index 00000000000..b6533bb299c --- /dev/null +++ b/tests/stream/test_compatibility/run_compatibility_tests.py @@ -0,0 +1,124 @@ +import inspect +import logging +import math +import os +import sys +import threading +import time +import types +import unittest +import uuid + +from ddt import process_file_data +from proton_driver import Client + +EPS = 1e-6 +FOLDER_ATTR = "%folder_path" +CUR_DIR = os.path.dirname(os.path.abspath(__file__)) +logger = logging.getLogger(__name__) +formatter = logging.Formatter("%(asctime)s %(levelname)s %(funcName)s %(message)s") +console_handler = logging.StreamHandler(sys.stderr) +console_handler.setFormatter(formatter) +logger.addHandler(console_handler) +logger.setLevel(logging.INFO) + + +def folder_data(value): + def wrapper(func): + setattr(func, FOLDER_ATTR, value) + return func + + return wrapper + + +def copy_func(f, name=None): + return types.FunctionType(f.func_code, f.func_globals, name or f.func_name, f.func_defaults, f.func_closure) + + +def using_folder(arg=None, **kwargs): + def wrapper(cls): + for name, func in list(cls.__dict__.items()): + if hasattr(func, FOLDER_ATTR): + folder_name = getattr(func, FOLDER_ATTR) + folder = os.listdir(folder_name) + for filename in folder: + filename_without_ext, *_, ext = filename.split(".") + full_file_path = os.path.join(folder_name, filename) + func_name = f'{name}_{filename_without_ext}_' + process_file_data(cls, func_name, func, full_file_path) + delattr(cls, name) + return cls + + return wrapper(arg) if inspect.isclass(arg) else wrapper + + +def check_item_eq(item, expect_item, item_type): + check_map = [ + ("int", lambda x, y: int(x) == int(y)), + ("float", lambda x, y: (math.isnan(float(x)) and math.isnan(float(y))) or abs(float(x) - float(y)) < EPS) + ] + if expect_item == "any_value": + return True + for key_word, check_func in check_map: + if key_word in item_type: + return check_func(item, expect_item) + return str(item) == str(expect_item) + + +def check_list_eq(row, expect_row, type_row): + if len(row) != len(expect_row) or len(type_row) != len(expect_row): + return False + for val, expect_val, t in zip(row, expect_row, type_row): + if not check_item_eq(val, expect_val, t): + return False + return True + + +def kill_query(query_id): + client = Client( + user="default", + password="", + host="localhost", + port="8463" + ) + client.execute(f"kill query where query_id='{query_id}'") + client.disconnect() + + +@using_folder +class TestCompatibility(unittest.TestCase): + + def run_cmd(self, cmd, expect=None, query_id=None, wait=1, query_time=None, **kwargs): + logger.info(cmd) + client = Client( + user="default", + password="", + host="localhost", + port="8463" + ) + if query_id is None: + query_id = str(uuid.uuid4()) + killer = threading.Timer(query_time, kill_query, (query_id,)) + if query_time is not None: + killer.start() + if expect is None: + client.execute(cmd, query_id=query_id) + else: + rows = client.execute_iter(cmd, with_column_types=True, query_id=query_id) + type_row = [t[1] for t in next(rows)] + for i, expect_row in enumerate(expect): + row = next(rows) + self.assertTrue(check_list_eq(row, expect_row, type_row), f"{cmd}\n row [{i}] {row} != {expect_row}") + client.disconnect() + killer.cancel() + time.sleep(wait) + + @folder_data(os.environ.get("TEST_DIR", CUR_DIR)) + def test_compatibility(self, steps, description="", **kwargs): + logger.info(f"description: {description}") + for step in steps: + self.run_cmd(**step) + + +if __name__ == '__main__': + unittest.main(verbosity=2)