CI/issue 407 compatibility test after release

.github/workflows/compatibility_test.yml

@@ -1,5 +1,5 @@
 name: compatibility_test
-
+run-name: Compatibility Test [${{ (github.event.inputs.source == '' || github.event.inputs.source == github.event.inputs.target) && format('Upload {0}', github.event.inputs.target) || format('{0} -> {1}', github.event.inputs.source, github.event.inputs.target) }}]
 on:
   workflow_dispatch:
     inputs:
@@ -12,14 +12,10 @@ on:
         required: false
         default: ''
 jobs:
-  upload_data:
-    uses: timeplus-io/proton/.github/workflows/run_command.yml@develop
-    with:
-      ec2-instance-type: ${{ vars.X64_INSTANCE_TYPE }}
-      ec2-image-id: ${{ vars.X64_TEST_AMI }}
-      ec2-volume-size: '30'
-      submodules: false
-      timeout: 30
+  prepare_upload_data:
+    if: ${{ github.event.inputs.source == '' || github.event.inputs.source == github.event.inputs.target }}
+    runs-on: ubuntu-latest
+    outputs:
       command: |
         export PROTON_VERSION=${{ github.event.inputs.target }}
 
@@ -35,52 +31,39 @@ jobs:
 
         pip install --upgrade pip
 
-        # FIXME: remove this line after pyyaml community fixed install bug
-        pip install pyyaml==5.3.1
-
         # FIXME(yokofly): docker 7.0.0 introduce a breaking change
         # https://github.com/docker/docker-py/issues/3194
         pip install docker==6.1.3
 
-        pip install -r helpers/requirements.txt
+        pip install -r test_compatibility/requirements.txt
 
-        bash test_compatibility/prepare_data.sh
-        bash test_compatibility/basic_tests.sh
+        timeout --foreground 10m bash test_compatibility/prepare_data.sh
+        timeout --foreground 10m bash test_compatibility/basic_tests.sh
 
         cd $GITHUB_WORKSPACE
         tar -zcvf $PROTON_VERSION.tar.gz data
-        aws s3 cp --no-progress $PROTON_VERSION.tar.gz s3://tp-internal/proton/compatibility/oss/
-    secrets:
-      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      AWS_REGION: ${{ secrets.AWS_REGION }}
-      AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
-      GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
-      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
-      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
-      TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }}
-      TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }}
-      TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }}
-
-  compatibility_test:
-    if: ${{ github.event.inputs.source!='' }}
-    uses: timeplus-io/proton/.github/workflows/run_command.yml@develop
-    with:
-      ec2-instance-type: ${{ vars.X64_INSTANCE_TYPE }}
-      ec2-image-id: ${{ vars.X64_TEST_AMI }}
-      ec2-volume-size: '30'
-      submodules: false
-      timeout: 30
+        aws s3 cp --no-progress $PROTON_VERSION.tar.gz s3://tp-internal/proton/compatibility/oss/$ARCH/
+
+        cd $GITHUB_WORKSPACE/tests/stream
+        timeout --foreground 10m bash test_compatibility/extra_tests.sh
+    steps:
+      - name: display command
+        run: |
+          echo 'command: ${{ steps.set_command.outputs.command }}'
+  prepare_compatibility_test:
+    if: ${{ github.event.inputs.source != '' && github.event.inputs.source != github.event.inputs.target }}
+    runs-on: ubuntu-latest
+    outputs:
       command: |
         export TARGET_VERSION=${{ github.event.inputs.target }}
         export SOURCE_VERSION=${{ github.event.inputs.source }}
         export PROTON_VERSION=$TARGET_VERSION
-
+        
         # prepare data
         cd $GITHUB_WORKSPACE
-        aws s3 cp --no-progress s3://tp-internal/proton/compatibility/oss/$SOURCE_VERSION.tar.gz .
+        aws s3 cp --no-progress s3://tp-internal/proton/compatibility/oss/$ARCH/$SOURCE_VERSION.tar.gz .
         tar -zxvf $SOURCE_VERSION.tar.gz
-
+        
         cd $GITHUB_WORKSPACE/tests/stream
 
         # make virtualenv
@@ -90,20 +73,102 @@ jobs:
         apt install python3-venv -y
         python -m venv env
         source env/bin/activate
-
+        
         pip install --upgrade pip
 
-        # FIXME: remove this line after pyyaml community fixed install bug
-        pip install pyyaml==5.3.1
-
         # FIXME(yokofly): docker 7.0.0 introduce a breaking change
         # https://github.com/docker/docker-py/issues/3194
         pip install docker==6.1.3
 
-        pip install -r helpers/requirements.txt
-
-        bash test_compatibility/basic_tests.sh
-        bash test_compatibility/extra_tests.sh
+        pip install -r test_compatibility/requirements.txt
+
+        timeout --foreground 10m bash test_compatibility/basic_tests.sh
+        timeout --foreground 10m bash test_compatibility/extra_tests.sh
+    steps:
+      - name: display command
+        run: |
+          echo 'command: ${{ steps.set_command.outputs.command }}'
+  upload_data_x64:
+    needs: [prepare_upload_data]
+    uses: timeplus-io/proton/.github/workflows/run_command.yml@develop
+    with:
+      ec2-instance-type: ${{ vars.X64_INSTANCE_TYPE }}
+      ec2-image-id: ${{ vars.X64_TEST_AMI }}
+      ec2-volume-size: '30'
+      submodules: false
+      timeout: 30
+      arch: ${{ vars.X64_ARCH }}
+      command: |
+        ${{ needs.prepare_upload_data.outputs.command }}
+    secrets:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_REGION: ${{ secrets.AWS_REGION }}
+      AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
+      GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+      TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }}
+      TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }}
+      TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }}
+  upload_data_arm:
+    needs: [prepare_upload_data]
+    uses: timeplus-io/proton/.github/workflows/run_command.yml@develop
+    with:
+      ec2-instance-type: ${{ vars.ARM_INSTANCE_TYPE }}
+      ec2-image-id: ${{ vars.ARM_TEST_AMI }}
+      ec2-volume-size: '30'
+      submodules: false
+      timeout: 30
+      arch: ${{ vars.ARM_ARCH }}
+      command: |
+        ${{ needs.prepare_upload_data.outputs.command }}
+    secrets:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_REGION: ${{ secrets.AWS_REGION }}
+      AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
+      GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+      TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }}
+      TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }}
+      TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }}
+  compatibility_test_x64:
+    needs: [prepare_compatibility_test]
+    uses: timeplus-io/proton/.github/workflows/run_command.yml@develop
+    with:
+      ec2-instance-type: ${{ vars.X64_INSTANCE_TYPE }}
+      ec2-image-id: ${{ vars.X64_TEST_AMI }}
+      ec2-volume-size: '30'
+      submodules: false
+      timeout: 30
+      arch: ${{ vars.X64_ARCH }}
+      command: |
+        ${{ needs.prepare_compatibility_test.outputs.command }}
+    secrets:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_REGION: ${{ secrets.AWS_REGION }}
+      AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
+      GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+      TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }}
+      TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }}
+      TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }}
+  compatibility_test_arm:
+    needs: [prepare_compatibility_test]
+    uses: timeplus-io/proton/.github/workflows/run_command.yml@develop
+    with:
+      ec2-instance-type: ${{ vars.ARM_INSTANCE_TYPE }}
+      ec2-image-id: ${{ vars.ARM_TEST_AMI }}
+      ec2-volume-size: '30'
+      submodules: false
+      timeout: 30
+      arch: ${{ vars.ARM_ARCH }}
+      command: |
+        ${{ needs.prepare_compatibility_test.outputs.command }}
     secrets:
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

.github/workflows/release_build.yml

@@ -169,6 +169,22 @@ jobs:
           -H "X-GitHub-Api-Version: 2022-11-28" \
           https://api.github.com/repos/timeplus-io/proton/actions/workflows/manual_trigger_test.yml/dispatches \
           -d "{\"ref\":\"develop\",\"inputs\":{\"arch\": \"arm\", \"tag\":\"$PROTON_TAG-rc\"}}\""
+
+        # trigger compatibility test
+        export PROTON_VERSION=$PROTON_TAG
+        cd $GITHUB_WORKSPACE/tests/proton_ci
+
+        # make virtualenv
+        ln -s /usr/bin/python3 /usr/bin/python
+        apt-get update
+        systemctl stop unattended-upgrades
+        apt install python3-venv -y
+        python -m venv env
+        source env/bin/activate
+        pip install --upgrade pip
+
+        pip install -r requirements.txt
+        python run_compatibility_tests.py
     secrets:
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

tests/proton_ci/requirements.txt

@@ -2,4 +2,5 @@ PyGithub>=1.58.0
 unidiff>=0.7.0
 boto3==1.20.4
 timeplus==1.1.2
-urllib3==1.26.7
+urllib3==1.26.7
+requests>=2.26.0

tests/proton_ci/run_compatibility_tests.py

@@ -0,0 +1,61 @@
+import json
+import logging
+import time
+from typing import List, Any
+
+import requests
+from env_helper import GH_PERSONAL_ACCESS_TOKEN, PROTON_VERSION
+import re
+
+HEADERS = {
+    "Accept": "application/vnd.github+json",
+    "Authorization": f"Bearer {GH_PERSONAL_ACCESS_TOKEN}",
+    "X-GitHub-Api-Version": "2022-11-28"
+}
+MAX_TEST_VERSION_NUM = 5
+
+
+def valid_version_tag(tag_list: List[str]) -> Any:
+    for tag in tag_list:
+        if re.match(r"\d+\.\d+\.\d+", tag) is not None:
+            return tag
+    return None
+
+
+if __name__ == "__main__":
+    proton_image_version_list = requests.get(
+        url="https://api.github.com/orgs/timeplus-io/packages/container/proton/versions?per_page=100",
+        headers=HEADERS
+    ).json()
+
+    valid_version_list = []
+    current_version = "latest"
+    for version_info in proton_image_version_list:
+        created_at = time.mktime(time.strptime(version_info['created_at'], "%Y-%m-%dT%H:%M:%SZ"))
+        tags = version_info['metadata']['container']['tags']
+        tag_name = valid_version_tag(tags)
+        if tag_name is None:
+            continue
+        if tag_name == PROTON_VERSION or tag_name == PROTON_VERSION + "-rc":
+            current_version = tag_name
+        valid_version_list.append((created_at, tag_name))
+
+    valid_version_list.sort(key=lambda version_tuple: -int(version_tuple[0]))
+    valid_version_list = valid_version_list[:min(len(valid_version_list), MAX_TEST_VERSION_NUM + 1)]
+
+    for _, version in valid_version_list:
+        try:
+            response = requests.post(
+                "https://api.github.com/repos/timeplus-io/proton/actions/workflows/compatibility_test.yml/dispatches",
+                headers=HEADERS,
+                data=json.dumps({
+                    "ref": "develop",
+                    "inputs": {
+                        "source": version,
+                        "target": current_version
+                    }
+                })
+            )
+            assert response.status_code == 204
+        except Exception as e:
+            logging.error(e)

tests/stream/test_compatibility/basic_tests.sh

@@ -1,5 +1,10 @@
-docker-compose -f test_compatibility/docker-compose.yaml up -d
+set -e
+CUR_DIR="$GITHUB_WORKSPACE/tests/stream/test_compatibility"
+docker-compose -f "$CUR_DIR/configs/docker-compose.yaml" up -d
+docker ps
 sleep 5
-docker exec proton-server proton client -nm -q "select x from example where _tp_time > earliest_ts() limit 3;"
-docker exec proton-server proton client -nm -q "select x from example_external limit 3 settings seek_to='earliest';"
-docker-compose -f test_compatibility/docker-compose.yaml down
+export TEST_DIR="$CUR_DIR/basic_tests"
+python $CUR_DIR/run_compatibility_tests.py
+docker exec proton-server pkill proton-server
+sleep 10
+docker-compose -f "$CUR_DIR/configs/docker-compose.yaml" down -v

tests/stream/test_compatibility/basic_tests/example.yaml

@@ -0,0 +1,9 @@
+example:
+  description: This is an example of test_suit
+  steps:
+    - cmd: select 1;
+      query_time: 3
+      wait: 1
+      query_id: '0000-0000'
+      expect:
+        - [1]

tests/stream/test_compatibility/basic_tests/materialized_view.yaml

@@ -0,0 +1,48 @@
+alter_stream_add_column:
+  description: query from a materialized view on stream, and then alter the stream by adding column
+  steps:
+    - cmd: |
+        select (* except _tp_time) from table(test_1_mv) order by col1;
+      query_time: 3
+      expect:
+        - [ 1, 1.1, 'a1' ]
+        - [ 2, 2.1, 'b2' ]
+        - [ 3, 3.1, 'c3' ]
+        - [ 4, 4.1, 'd4' ]
+        - [ 5, 5.1, 'e5' ]
+        - [ 6, 6.1, 'f6' ]
+    - cmd: |
+        select (* except _tp_time) from test_1_mv where _tp_time > earliest_ts() order by col1;
+      query_time: 3
+      expect:
+        - [ 1, 1.1, 'a1' ]
+        - [ 2, 2.1, 'b2' ]
+        - [ 3, 3.1, 'c3' ]
+        - [ 4, 4.1, 'd4' ]
+        - [ 5, 5.1, 'e5' ]
+        - [ 6, 6.1, 'f6' ]
+
+global_aggregation:
+  description: a materialized view on stream global aggregation
+  steps:
+    - cmd: |
+        select (* except _tp_time) from table(test_2_mv);
+      query_time: 3
+      expect:
+        - [ 1.1, 6.1, 8 ]
+    - cmd: |
+        select (* except _tp_time) from test_2_mv where _tp_time > earliest_ts() limit 1;
+      expect:
+        - [ 1.1, 6.1, 8 ]
+
+func_now:
+  description: test for function now()
+  steps:
+    - cmd: |
+        select (* except _tp_time) from table(test_3_mv) order by i;
+      query_time: 3
+      expect:
+        - [ "any_value", 1 ]
+        - [ "any_value", 2 ]
+        - [ "any_value", 3 ]
+        - [ "any_value", 4 ]

tests/stream/test_compatibility/docker-compose.yaml → tests/stream/test_compatibility/configs/docker-compose.yaml

@@ -8,11 +8,11 @@ services:
       - $GITHUB_WORKSPACE/data/proton-redp/datas:/var/lib/proton
       - $GITHUB_WORKSPACE/data/proton-redp/log:/var/log/proton-server
     ports:
-      - "13218:3218" # HTTP Streaming
-      - "18123:8123" # HTTP Snapshot
-      - "18463:8463" # TCP Streaming
-      - "15432:5432" # Postgres Snapshot
-      - "17587:7587" # TCP Snapshot
+      - "3218:3218" # HTTP Streaming
+      - "8123:8123" # HTTP Snapshot
+      - "8463:8463" # TCP Streaming
+      - "5432:5432" # Postgres Snapshot
+      - "7587:7587" # TCP Snapshot
     deploy:
       replicas: 1
       restart_policy: