From d4314a5dae305deae2e2e7d69a3d78b6f8d7f231 Mon Sep 17 00:00:00 2001 From: Jaime Silvela Date: Mon, 24 Apr 2023 14:03:31 +0200 Subject: [PATCH] feat: compute alerts on systematic failure (#4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stop showing the ignoreFail as a code error. Create a short summary for cases where the full summary would be truncated by GH. Compute an Alerts section and make it available standalone for use in Slackops. Add unit tests. Signed-off-by: Jaime Silvela Signed-off-by: Tao Li Signed-off-by: Niccolò Fei Co-authored-by: Niccolò Fei --- .github/workflows/overflow-test.yaml | 67 +++++++++ .github/workflows/test.yaml | 16 ++- .github/workflows/unit-test.yaml | 19 +++ .gitignore | 1 + DEVELOPERS_DEVELOPERS_DEVELOPERS.md | 12 +- Dockerfile | 2 +- README.md | 75 ++++++++++ action.yaml | 10 ++ ...bb7d97458ca95fd421f325f3896ed239d5d3f.json | 2 +- ...76000698c46de8e7f9763a1a0fe63f70fd5f8.json | 2 +- ...dfe622e341f84dd3d1df1cffee843e1fb84a0.json | 2 +- ...829a4b9f509415ce8da7d997e5ea40bb24af8.json | 19 ++- ...829a4b9f509415ce8da7d997e5ea40bb24af8.json | 19 ++- ...2c1a040e8d51bbeb3f218e998a035ad896d22.json | 19 ++- ...829a4b9f509415ce8da7d997e5ea40bb24af8.json | 19 ++- ...829a4b9f509415ce8da7d997e5ea40bb24af8.json | 19 ++- ...829a4b9f509415ce8da7d997e5ea40bb24af8.json | 19 ++- ...829a4b9f509415ce8da7d997e5ea40bb24af8.json | 19 ++- ...829a4b9f509415ce8da7d997e5ea40bb24af8.json | 19 ++- ...829a4b9f509415ce8da7d997e5ea40bb24af8.json | 19 ++- ...829a4b9f509415ce8da7d997e5ea40bb24af8.json | 19 ++- ...dfe622e341f84dd3d1df1cffee843e1fb84a0.json | 17 +++ ...bb7d97458ca95fd421f325f3896ed239d5d3f.json | 17 +++ ...76000698c46de8e7f9763a1a0fe63f70fd5f8.json | 17 +++ ...dfe622e341f84dd3d1df1cffee843e1fb84a0.json | 17 +++ summarize_test_results.py | 128 +++++++++++++++++- test_summary.py | 70 ++++++++++ 27 files changed, 641 insertions(+), 23 deletions(-) create mode 100644 .github/workflows/overflow-test.yaml create mode 100644 .github/workflows/unit-test.yaml create mode 100644 example-artifacts/local-v1.26.3-PostgreSQL-15.2_9891d0d1caa1ec8fd0adfe622e341f84dd3d1df1cffee843e1fb84a0.json create mode 100644 few-artifacts/id1_0b185c51a60964ecab5bb7d97458ca95fd421f325f3896ed239d5d3f.json create mode 100644 few-artifacts/id1_4902843ff6a60bc4fdb76000698c46de8e7f9763a1a0fe63f70fd5f8.json create mode 100644 few-artifacts/id1_9891d0d1caa1ec8fd0adfe622e341f84dd3d1df1cffee843e1fb84a0.json create mode 100644 test_summary.py diff --git a/.github/workflows/overflow-test.yaml b/.github/workflows/overflow-test.yaml new file mode 100644 index 0000000..7e06d52 --- /dev/null +++ b/.github/workflows/overflow-test.yaml @@ -0,0 +1,67 @@ +on: [push] + +defaults: + run: + # default failure handling for shell scripts in 'run' steps + shell: 'bash -Eeuo pipefail -x {0}' + +jobs: + overflow_test: + runs-on: ubuntu-latest + name: Test CIclops with summary overflow + steps: + - uses: actions/checkout@v3 + + - name: Generate Test Summary + uses: ./ + with: + artifact_directory: example-artifacts/ + output_file: test-summary.md + short_file: short.md + alerts_file: alerts.txt + # NOTE: it does not seem possible to pass $GITHUB_STEP_SUMMARY as a + # regular file to the underlying script. Hence, we create a file and + # in a later step write with >> $GITHUB_STEP_SUMMARY + + - name: Create local file that is bigger than GH limit + run: | + dd if=/dev/zero of=big-test-summary.md bs=1M count=2 + + - name: Check full summary fits within GH limit + # $GITHUB_STEP_SUMMARY will reject content over 1024 bytes + # on exceeding, the workflow WILL FAIL and still count as success() + # With this step, we do proper error flow, and fail if the limit would be + # exceeded. + id: check-overflow + run: | + size=$(stat -c '%s' big-test-summary.md) + if [ "$size" -gt 1024 ]; then + echo "overflow=true" >> $GITHUB_OUTPUT + fi + # Here we force the "big test summary" to overflow GH limits, and we + # create an output that further steps can leverage: steps.check-overflow.outputs.overflow + + - name: If the full summary fits within GH limits, use it + # This step should be skipped, we expect + if: ${{!steps.check-overflow.outputs.overflow}} + run: | + cat big-test-summary.md >> $GITHUB_STEP_SUMMARY + + - name: If the full summary is too big, use short version + if: ${{steps.check-overflow.outputs.overflow}} + run: | + cat short.md >> $GITHUB_STEP_SUMMARY + + - name: If full summary is too big, archive it + if: ${{steps.check-overflow.outputs.overflow}} + uses: actions/upload-artifact@v3 + with: + name: test-summary.md + path: test-summary.md + retention-days: 7 + + - name: Create slack body with alerts + run: | + echo 'slack-message<> $GITHUB_OUTPUT + cat alerts.txt >> $GITHUB_OUTPUT + echo 'EOF' >> $GITHUB_OUTPUT diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index edf3576..285de67 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -1,9 +1,9 @@ on: [push] jobs: - custom_test: + plain_test: runs-on: ubuntu-latest - name: Test action with act + name: Test CIclops with normal summary steps: - uses: actions/checkout@v3 @@ -12,8 +12,18 @@ jobs: with: artifact_directory: example-artifacts/ output_file: test-summary.md + short_file: short.md + alerts_file: alerts.txt # NOTE: it does not seem possible to pass $GITHUB_STEP_SUMMARY as a # regular file to the underlying script. Hence, we create a file and # in a later step write with >> $GITHUB_STEP_SUMMARY + - name: Create GitHub Job Summary from report - run: cat test-summary.md >> $GITHUB_STEP_SUMMARY + run: | + cat test-summary.md >> $GITHUB_STEP_SUMMARY + + - name: Create slack body with alerts + run: | + echo 'slack-message<> $GITHUB_OUTPUT + cat alerts.txt >> $GITHUB_OUTPUT + echo 'EOF' >> $GITHUB_OUTPUT diff --git a/.github/workflows/unit-test.yaml b/.github/workflows/unit-test.yaml new file mode 100644 index 0000000..7520f87 --- /dev/null +++ b/.github/workflows/unit-test.yaml @@ -0,0 +1,19 @@ +on: [push] + +jobs: + plain_test: + runs-on: ubuntu-latest + name: Unit test + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.x' + + - name: Install dependencies + run: python -m pip install --upgrade pip prettytable + + - name: Run suite + run: python test_summary.py -v diff --git a/.gitignore b/.gitignore index 69adda3..7eeb6eb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ # editor and IDE paraphernalia .idea +__pycache__/ diff --git a/DEVELOPERS_DEVELOPERS_DEVELOPERS.md b/DEVELOPERS_DEVELOPERS_DEVELOPERS.md index 6d49fd2..6dbc22d 100644 --- a/DEVELOPERS_DEVELOPERS_DEVELOPERS.md +++ b/DEVELOPERS_DEVELOPERS_DEVELOPERS.md @@ -1,4 +1,4 @@ -# building and testing locally +# Building and testing locally The `ciclops` GitHub Action runs using a Docker container that encapsulates the Python script that does the CI test analysis. @@ -39,6 +39,14 @@ act -b --env GITHUB_STEP_SUMMARY='github-summary.md' Running this should create a file `github-summary.md` with the test summary. +## Unit tests + +CIclops has the beginning of a unit test suite. You can run it with: + +``` sh +python3 -m unittest +``` + ## How it works The files in this repository are needed for the Dockerfile to build and run, of @@ -57,4 +65,4 @@ See [GitHub support for Dockerfile](https://docs.github.com/en/actions/creating- **NOTE**: the behavior of the `COPY` command in Dockerfiles seems quite finicky on whether it's done recursively or not. The invocation used, -`COPY . .`, ensured that the copy was recursive. +`COPY . .`, ensures the copy is done recursively. diff --git a/Dockerfile b/Dockerfile index 8364edf..f969db4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,4 +7,4 @@ RUN pip install --no-cache-dir -r requirements.txt COPY . . ENTRYPOINT [ "python", "/summarize_test_results.py"] -CMD ["--dir", "./test-artifacts", "--out", ""] +CMD ["--dir", "./test-artifacts", "--out", "", "--short", "", "--alerts", ""] diff --git a/README.md b/README.md index 6abe460..48f5269 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,81 @@ For example: run: cat test-summary.md >> $GITHUB_STEP_SUMMARY ``` +## Advanced Usage + +There are two advanced cases we want to call attention to: + +1. Summary overflow. \ + The `GITHUB_STEP_SUMMARY` variable set to receive the CIclops summary will + overflow if the summary is bigger than 1024 bytes. To work around this, + CIclops can be directed to create a short summary on top of the "normal" + summary. The calling workflow can verify if the full summary is too big, and + if so, can display the short summary, and perhaps Archive the full summary. + +2. Slackops \ + CIclops will create a series of alerts when systematic failures are detected. + By "systematic", we mean cases such as: + + - all test combinations have failed + - all combinations fail for a given test + - all tests fail for a given version of Postgres + +The alerts are included in the summary generated by CIclops, but it is also +possible to direct CIclops to put the alerts only in an output file. +This file can then be sent via Slack message to alert DevOps teams. + +The following snippet shows how to use these features: + +``` yaml + - name: Generate Test Summary + uses: cloudnative-pg/ciclops@main + with: + artifact_directory: test-artifacts/data + output_file: test-summary.md + short_file: short.md + alerts_file: alerts.txt + + - name: Check full summary fits within GH limit + id: check-overflow + run: | + size=$(stat -c '%s' test-summary.md) + if [ "$size" -gt 1024 ]; then + echo "overflow=true" >> $GITHUB_OUTPUT + fi + + - name: If the full summary would not overflow, use it + if: ${{!steps.check-overflow.outputs.overflow}} + run: | + cat test-summary.md >> $GITHUB_STEP_SUMMARY + + - name: If the full summary is too big, use short version + if: ${{steps.check-overflow.outputs.overflow}} + run: | + cat short.md >> $GITHUB_STEP_SUMMARY + + - name: If full summary is too big, archive it + if: ${{steps.check-overflow.outputs.overflow}} + uses: actions/upload-artifact@v3 + with: + name: test-summary.md + path: test-summary.md + retention-days: 7 + + - name: Create Slack body with alerts + id: alerts + run: | + echo 'slack-message<> $GITHUB_OUTPUT + cat alerts.txt >> $GITHUB_OUTPUT + echo 'EOF' >> $GITHUB_OUTPUT + + - name: Send Slack Notification + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_USERNAME: cnpg-bot + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + SLACK_MESSAGE: ${{ steps.alerts.outputs.slack-message }} +``` + ## Origin At EDB, working on a series of Kubernetes operators for PostgreSQL, we have an diff --git a/action.yaml b/action.yaml index 09d630f..370bfae 100644 --- a/action.yaml +++ b/action.yaml @@ -14,6 +14,12 @@ inputs: output_file: description: "file where the markdown report should be written" required: false + short_file: + description: "file where the abridged markdown report should be written" + required: false + alerts_file: + description: "file where any alerts found should be written" + required: false runs: using: "docker" image: "Dockerfile" @@ -22,3 +28,7 @@ runs: - "./${{ inputs.artifact_directory }}" - "--out" - "${{ inputs.output_file }}" + - "--short" + - "${{ inputs.short_file }}" + - "--alerts" + - "${{ inputs.alerts_file }}" diff --git a/example-artifacts/id1_0b185c51a60964ecab5bb7d97458ca95fd421f325f3896ed239d5d3f.json b/example-artifacts/id1_0b185c51a60964ecab5bb7d97458ca95fd421f325f3896ed239d5d3f.json index fa02b5b..9ad974f 100644 --- a/example-artifacts/id1_0b185c51a60964ecab5bb7d97458ca95fd421f325f3896ed239d5d3f.json +++ b/example-artifacts/id1_0b185c51a60964ecab5bb7d97458ca95fd421f325f3896ed239d5d3f.json @@ -14,4 +14,4 @@ "workflow_id": 12, "repo": "my-repo", "branch": "my-branch" -} \ No newline at end of file +} diff --git a/example-artifacts/id1_4902843ff6a60bc4fdb76000698c46de8e7f9763a1a0fe63f70fd5f8.json b/example-artifacts/id1_4902843ff6a60bc4fdb76000698c46de8e7f9763a1a0fe63f70fd5f8.json index 857513e..41628dd 100644 --- a/example-artifacts/id1_4902843ff6a60bc4fdb76000698c46de8e7f9763a1a0fe63f70fd5f8.json +++ b/example-artifacts/id1_4902843ff6a60bc4fdb76000698c46de8e7f9763a1a0fe63f70fd5f8.json @@ -14,4 +14,4 @@ "workflow_id": 12, "repo": "my-repo", "branch": "my-branch" -} \ No newline at end of file +} diff --git a/example-artifacts/id1_9891d0d1caa1ec8fd0adfe622e341f84dd3d1df1cffee843e1fb84a0.json b/example-artifacts/id1_9891d0d1caa1ec8fd0adfe622e341f84dd3d1df1cffee843e1fb84a0.json index 5767544..d2d2d66 100644 --- a/example-artifacts/id1_9891d0d1caa1ec8fd0adfe622e341f84dd3d1df1cffee843e1fb84a0.json +++ b/example-artifacts/id1_9891d0d1caa1ec8fd0adfe622e341f84dd3d1df1cffee843e1fb84a0.json @@ -14,4 +14,4 @@ "workflow_id": 12, "repo": "my-repo", "branch": "my-branch" -} \ No newline at end of file +} diff --git a/example-artifacts/local-v1.21.14-PGD-PostgreSQL-12.13-5.0.0-0.0git256.g2a1b38b.1.dev-1_86b346ab9958ad86dc1829a4b9f509415ce8da7d997e5ea40bb24af8.json b/example-artifacts/local-v1.21.14-PGD-PostgreSQL-12.13-5.0.0-0.0git256.g2a1b38b.1.dev-1_86b346ab9958ad86dc1829a4b9f509415ce8da7d997e5ea40bb24af8.json index 4385d32..aca146e 100644 --- a/example-artifacts/local-v1.21.14-PGD-PostgreSQL-12.13-5.0.0-0.0git256.g2a1b38b.1.dev-1_86b346ab9958ad86dc1829a4b9f509415ce8da7d997e5ea40bb24af8.json +++ b/example-artifacts/local-v1.21.14-PGD-PostgreSQL-12.13-5.0.0-0.0git256.g2a1b38b.1.dev-1_86b346ab9958ad86dc1829a4b9f509415ce8da7d997e5ea40bb24af8.json @@ -1 +1,18 @@ -{"name": "Update operand -- upgrades operand version", "state": "failed", "start_time": "2023-01-25T10:26:03.771385171Z", "end_time": "2023-01-25T10:38:44.769651724Z", "error": "Timed out after 600.001s.\nExpected success, but got an error:\n <*errors.errorString | 0xc0003c4c20>: {\n s: \"Assertion in callback at /home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go:266 failed:\\nExpected\\n : 0\\nto equal\\n : 1\",\n }\n Assertion in callback at /home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go:266 failed:\n Expected\n : 0\n to equal\n : 1", "error_file": "/home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go", "error_line": 268, "platform": "local", "postgres_kind": "PostgreSQL", "redwood": false, "matrix_id": "local-v1.21.14-PGD-PostgreSQL-12.13-5.0.0-0.0git256.g2a1b38b.1.dev-1", "postgres_version": "12.13-5.0.0-0.0git256.g2a1b38b.1.dev-1", "k8s_version": "v1.21.14", "workflow_id": 4004501040, "repo": "EnterpriseDB/pg4k-pgd", "branch": "dev/cnp-3285-2"} \ No newline at end of file +{ + "name": "Update operand -- upgrades operand version", + "state": "failed", + "start_time": "2023-01-25T10:26:03.771385171Z", + "end_time": "2023-01-25T10:38:44.769651724Z", + "error": "Timed out after 600.001s.\nExpected success, but got an error:\n <*errors.errorString | 0xc0003c4c20>: {\n s: \"Assertion in callback at /home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go:266 failed:\\nExpected\\n : 0\\nto equal\\n : 1\",\n }\n Assertion in callback at /home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go:266 failed:\n Expected\n : 0\n to equal\n : 1", + "error_file": "/home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go", + "error_line": 268, + "platform": "local", + "postgres_kind": "PostgreSQL", + "redwood": false, + "matrix_id": "local-v1.21.14-PGD-PostgreSQL-12.13-5.0.0-0.0git256.g2a1b38b.1.dev-1", + "postgres_version": "12.13-5.0.0-0.0git256.g2a1b38b.1.dev-1", + "k8s_version": "v1.21.14", + "workflow_id": 4004501040, + "repo": "EnterpriseDB/pg4k-pgd", + "branch": "dev/cnp-3285-2" +} diff --git a/example-artifacts/local-v1.21.14-PGD-PostgreSQL-13.9-5.0.0-0.0git256.g2a1b38b.1.dev-1_86b346ab9958ad86dc1829a4b9f509415ce8da7d997e5ea40bb24af8.json b/example-artifacts/local-v1.21.14-PGD-PostgreSQL-13.9-5.0.0-0.0git256.g2a1b38b.1.dev-1_86b346ab9958ad86dc1829a4b9f509415ce8da7d997e5ea40bb24af8.json index ea83c72..886ae3f 100644 --- a/example-artifacts/local-v1.21.14-PGD-PostgreSQL-13.9-5.0.0-0.0git256.g2a1b38b.1.dev-1_86b346ab9958ad86dc1829a4b9f509415ce8da7d997e5ea40bb24af8.json +++ b/example-artifacts/local-v1.21.14-PGD-PostgreSQL-13.9-5.0.0-0.0git256.g2a1b38b.1.dev-1_86b346ab9958ad86dc1829a4b9f509415ce8da7d997e5ea40bb24af8.json @@ -1 +1,18 @@ -{"name": "Update operand -- upgrades operand version", "state": "failed", "start_time": "2023-01-25T11:08:20.179946291Z", "end_time": "2023-01-25T11:20:54.376679592Z", "error": "Timed out after 600.001s.\nExpected success, but got an error:\n <*errors.errorString | 0xc000a41e40>: {\n s: \"Assertion in callback at /home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go:266 failed:\\nExpected\\n : 0\\nto equal\\n : 1\",\n }\n Assertion in callback at /home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go:266 failed:\n Expected\n : 0\n to equal\n : 1", "error_file": "/home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go", "error_line": 268, "platform": "local", "postgres_kind": "PostgreSQL", "redwood": false, "matrix_id": "local-v1.21.14-PGD-PostgreSQL-13.9-5.0.0-0.0git256.g2a1b38b.1.dev-1", "postgres_version": "13.9-5.0.0-0.0git256.g2a1b38b.1.dev-1", "k8s_version": "v1.21.14", "workflow_id": 4004501040, "repo": "EnterpriseDB/pg4k-pgd", "branch": "dev/cnp-3285-2"} \ No newline at end of file +{ + "name": "Update operand -- upgrades operand version", + "state": "failed", + "start_time": "2023-01-25T11:08:20.179946291Z", + "end_time": "2023-01-25T11:20:54.376679592Z", + "error": "Timed out after 600.001s.\nExpected success, but got an error:\n <*errors.errorString | 0xc000a41e40>: {\n s: \"Assertion in callback at /home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go:266 failed:\\nExpected\\n : 0\\nto equal\\n : 1\",\n }\n Assertion in callback at /home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go:266 failed:\n Expected\n : 0\n to equal\n : 1", + "error_file": "/home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go", + "error_line": 268, + "platform": "local", + "postgres_kind": "PostgreSQL", + "redwood": false, + "matrix_id": "local-v1.21.14-PGD-PostgreSQL-13.9-5.0.0-0.0git256.g2a1b38b.1.dev-1", + "postgres_version": "13.9-5.0.0-0.0git256.g2a1b38b.1.dev-1", + "k8s_version": "v1.21.14", + "workflow_id": 4004501040, + "repo": "EnterpriseDB/pg4k-pgd", + "branch": "dev/cnp-3285-2" +} diff --git a/example-artifacts/local-v1.21.14-PGD-PostgreSQL-13.9-5.0.0-0.0git256.g2a1b38b.1.dev-1_bc078a3b7d0af245efb2c1a040e8d51bbeb3f218e998a035ad896d22.json b/example-artifacts/local-v1.21.14-PGD-PostgreSQL-13.9-5.0.0-0.0git256.g2a1b38b.1.dev-1_bc078a3b7d0af245efb2c1a040e8d51bbeb3f218e998a035ad896d22.json index 3cb9532..1773d7d 100644 --- a/example-artifacts/local-v1.21.14-PGD-PostgreSQL-13.9-5.0.0-0.0git256.g2a1b38b.1.dev-1_bc078a3b7d0af245efb2c1a040e8d51bbeb3f218e998a035ad896d22.json +++ b/example-artifacts/local-v1.21.14-PGD-PostgreSQL-13.9-5.0.0-0.0git256.g2a1b38b.1.dev-1_bc078a3b7d0af245efb2c1a040e8d51bbeb3f218e998a035ad896d22.json @@ -1 +1,18 @@ -{"name": "Smoke test -- sets up a PGD Group according to flexible three regions architecture", "state": "failed", "start_time": "2023-01-25T10:38:50.421338678Z", "end_time": "2023-01-25T10:52:23.675119439Z", "error": "Timed out after 602.794s.\nError: Assertion in callback at /home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go:463 failed:\nExpected\n <*fmt.wrapError | 0xc00007d8e0>: {\n msg: \"context deadline exceeded - \",\n err: {},\n }\nto be nil\n <*errors.errorString | 0xc0003b2650>: {\n s: \"Assertion in callback at /home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go:463 failed:\\nExpected\\n <*fmt.wrapError | 0xc00007d8e0>: {\\n msg: \\\"context deadline exceeded - \\\",\\n err: {},\\n }\\nto be nil\",\n }", "error_file": "/home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go", "error_line": 466, "platform": "local", "postgres_kind": "PostgreSQL", "redwood": false, "matrix_id": "local-v1.21.14-PGD-PostgreSQL-13.9-5.0.0-0.0git256.g2a1b38b.1.dev-1", "postgres_version": "13.9-5.0.0-0.0git256.g2a1b38b.1.dev-1", "k8s_version": "v1.21.14", "workflow_id": 4004501040, "repo": "EnterpriseDB/pg4k-pgd", "branch": "dev/cnp-3285-2"} \ No newline at end of file +{ + "name": "Smoke test -- sets up a PGD Group according to flexible three regions architecture", + "state": "failed", + "start_time": "2023-01-25T10:38:50.421338678Z", + "end_time": "2023-01-25T10:52:23.675119439Z", + "error": "Timed out after 602.794s.\nError: Assertion in callback at /home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go:463 failed:\nExpected\n <*fmt.wrapError | 0xc00007d8e0>: {\n msg: \"context deadline exceeded - \",\n err: {},\n }\nto be nil\n <*errors.errorString | 0xc0003b2650>: {\n s: \"Assertion in callback at /home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go:463 failed:\\nExpected\\n <*fmt.wrapError | 0xc00007d8e0>: {\\n msg: \\\"context deadline exceeded - \\\",\\n err: {},\\n }\\nto be nil\",\n }", + "error_file": "/home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go", + "error_line": 466, + "platform": "local", + "postgres_kind": "PostgreSQL", + "redwood": false, + "matrix_id": "local-v1.21.14-PGD-PostgreSQL-13.9-5.0.0-0.0git256.g2a1b38b.1.dev-1", + "postgres_version": "13.9-5.0.0-0.0git256.g2a1b38b.1.dev-1", + "k8s_version": "v1.21.14", + "workflow_id": 4004501040, + "repo": "EnterpriseDB/pg4k-pgd", + "branch": "dev/cnp-3285-2" +} diff --git a/example-artifacts/local-v1.21.14-PGD-PostgreSQL-14.6-5.0.0-0.0git256.g2a1b38b.1.dev-1_86b346ab9958ad86dc1829a4b9f509415ce8da7d997e5ea40bb24af8.json b/example-artifacts/local-v1.21.14-PGD-PostgreSQL-14.6-5.0.0-0.0git256.g2a1b38b.1.dev-1_86b346ab9958ad86dc1829a4b9f509415ce8da7d997e5ea40bb24af8.json index 5ba39fd..7a17c1e 100644 --- a/example-artifacts/local-v1.21.14-PGD-PostgreSQL-14.6-5.0.0-0.0git256.g2a1b38b.1.dev-1_86b346ab9958ad86dc1829a4b9f509415ce8da7d997e5ea40bb24af8.json +++ b/example-artifacts/local-v1.21.14-PGD-PostgreSQL-14.6-5.0.0-0.0git256.g2a1b38b.1.dev-1_86b346ab9958ad86dc1829a4b9f509415ce8da7d997e5ea40bb24af8.json @@ -1 +1,18 @@ -{"name": "Update operand -- upgrades operand version", "state": "failed", "start_time": "2023-01-25T10:42:32.45376546Z", "end_time": "2023-01-25T10:54:59.12104098Z", "error": "Timed out after 600.001s.\nExpected success, but got an error:\n <*errors.errorString | 0xc0000b3ce0>: {\n s: \"Assertion in callback at /home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go:266 failed:\\nExpected\\n : 0\\nto equal\\n : 1\",\n }\n Assertion in callback at /home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go:266 failed:\n Expected\n : 0\n to equal\n : 1", "error_file": "/home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go", "error_line": 268, "platform": "local", "postgres_kind": "PostgreSQL", "redwood": false, "matrix_id": "local-v1.21.14-PGD-PostgreSQL-14.6-5.0.0-0.0git256.g2a1b38b.1.dev-1", "postgres_version": "14.6-5.0.0-0.0git256.g2a1b38b.1.dev-1", "k8s_version": "v1.21.14", "workflow_id": 4004501040, "repo": "EnterpriseDB/pg4k-pgd", "branch": "dev/cnp-3285-2"} \ No newline at end of file +{ + "name": "Update operand -- upgrades operand version", + "state": "failed", + "start_time": "2023-01-25T10:42:32.45376546Z", + "end_time": "2023-01-25T10:54:59.12104098Z", + "error": "Timed out after 600.001s.\nExpected success, but got an error:\n <*errors.errorString | 0xc0000b3ce0>: {\n s: \"Assertion in callback at /home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go:266 failed:\\nExpected\\n : 0\\nto equal\\n : 1\",\n }\n Assertion in callback at /home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go:266 failed:\n Expected\n : 0\n to equal\n : 1", + "error_file": "/home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go", + "error_line": 268, + "platform": "local", + "postgres_kind": "PostgreSQL", + "redwood": false, + "matrix_id": "local-v1.21.14-PGD-PostgreSQL-14.6-5.0.0-0.0git256.g2a1b38b.1.dev-1", + "postgres_version": "14.6-5.0.0-0.0git256.g2a1b38b.1.dev-1", + "k8s_version": "v1.21.14", + "workflow_id": 4004501040, + "repo": "EnterpriseDB/pg4k-pgd", + "branch": "dev/cnp-3285-2" +} diff --git a/example-artifacts/local-v1.22.15-PGD-PostgreSQL-14.6-5.0.0-0.0git256.g2a1b38b.1.dev-1_86b346ab9958ad86dc1829a4b9f509415ce8da7d997e5ea40bb24af8.json b/example-artifacts/local-v1.22.15-PGD-PostgreSQL-14.6-5.0.0-0.0git256.g2a1b38b.1.dev-1_86b346ab9958ad86dc1829a4b9f509415ce8da7d997e5ea40bb24af8.json index 15ce832..9ed1b73 100644 --- a/example-artifacts/local-v1.22.15-PGD-PostgreSQL-14.6-5.0.0-0.0git256.g2a1b38b.1.dev-1_86b346ab9958ad86dc1829a4b9f509415ce8da7d997e5ea40bb24af8.json +++ b/example-artifacts/local-v1.22.15-PGD-PostgreSQL-14.6-5.0.0-0.0git256.g2a1b38b.1.dev-1_86b346ab9958ad86dc1829a4b9f509415ce8da7d997e5ea40bb24af8.json @@ -1 +1,18 @@ -{"name": "Update operand -- upgrades operand version", "state": "failed", "start_time": "2023-01-25T10:47:20.438364457Z", "end_time": "2023-01-25T10:59:16.061423411Z", "error": "Timed out after 600.001s.\nExpected success, but got an error:\n <*errors.errorString | 0xc000aafa70>: {\n s: \"Assertion in callback at /home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go:266 failed:\\nExpected\\n : 0\\nto equal\\n : 1\",\n }\n Assertion in callback at /home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go:266 failed:\n Expected\n : 0\n to equal\n : 1", "error_file": "/home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go", "error_line": 268, "platform": "local", "postgres_kind": "PostgreSQL", "redwood": false, "matrix_id": "local-v1.22.15-PGD-PostgreSQL-14.6-5.0.0-0.0git256.g2a1b38b.1.dev-1", "postgres_version": "14.6-5.0.0-0.0git256.g2a1b38b.1.dev-1", "k8s_version": "v1.22.15", "workflow_id": 4004501040, "repo": "EnterpriseDB/pg4k-pgd", "branch": "dev/cnp-3285-2"} \ No newline at end of file +{ + "name": "Update operand -- upgrades operand version", + "state": "failed", + "start_time": "2023-01-25T10:47:20.438364457Z", + "end_time": "2023-01-25T10:59:16.061423411Z", + "error": "Timed out after 600.001s.\nExpected success, but got an error:\n <*errors.errorString | 0xc000aafa70>: {\n s: \"Assertion in callback at /home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go:266 failed:\\nExpected\\n : 0\\nto equal\\n : 1\",\n }\n Assertion in callback at /home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go:266 failed:\n Expected\n : 0\n to equal\n : 1", + "error_file": "/home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go", + "error_line": 268, + "platform": "local", + "postgres_kind": "PostgreSQL", + "redwood": false, + "matrix_id": "local-v1.22.15-PGD-PostgreSQL-14.6-5.0.0-0.0git256.g2a1b38b.1.dev-1", + "postgres_version": "14.6-5.0.0-0.0git256.g2a1b38b.1.dev-1", + "k8s_version": "v1.22.15", + "workflow_id": 4004501040, + "repo": "EnterpriseDB/pg4k-pgd", + "branch": "dev/cnp-3285-2" +} diff --git a/example-artifacts/local-v1.23.13-PGD-PostgreSQL-14.6-5.0.0-0.0git256.g2a1b38b.1.dev-1_86b346ab9958ad86dc1829a4b9f509415ce8da7d997e5ea40bb24af8.json b/example-artifacts/local-v1.23.13-PGD-PostgreSQL-14.6-5.0.0-0.0git256.g2a1b38b.1.dev-1_86b346ab9958ad86dc1829a4b9f509415ce8da7d997e5ea40bb24af8.json index 4b2f793..be90133 100644 --- a/example-artifacts/local-v1.23.13-PGD-PostgreSQL-14.6-5.0.0-0.0git256.g2a1b38b.1.dev-1_86b346ab9958ad86dc1829a4b9f509415ce8da7d997e5ea40bb24af8.json +++ b/example-artifacts/local-v1.23.13-PGD-PostgreSQL-14.6-5.0.0-0.0git256.g2a1b38b.1.dev-1_86b346ab9958ad86dc1829a4b9f509415ce8da7d997e5ea40bb24af8.json @@ -1 +1,18 @@ -{"name": "Update operand -- upgrades operand version", "state": "failed", "start_time": "2023-01-25T09:48:56.116429738Z", "end_time": "2023-01-25T10:01:22.651439285Z", "error": "Timed out after 600.001s.\nExpected success, but got an error:\n <*errors.errorString | 0xc0003f5da0>: {\n s: \"Assertion in callback at /home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go:266 failed:\\nExpected\\n : 0\\nto equal\\n : 1\",\n }\n Assertion in callback at /home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go:266 failed:\n Expected\n : 0\n to equal\n : 1", "error_file": "/home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go", "error_line": 268, "platform": "local", "postgres_kind": "PostgreSQL", "redwood": false, "matrix_id": "local-v1.23.13-PGD-PostgreSQL-14.6-5.0.0-0.0git256.g2a1b38b.1.dev-1", "postgres_version": "14.6-5.0.0-0.0git256.g2a1b38b.1.dev-1", "k8s_version": "v1.23.13", "workflow_id": 4004501040, "repo": "EnterpriseDB/pg4k-pgd", "branch": "dev/cnp-3285-2"} \ No newline at end of file +{ + "name": "Update operand -- upgrades operand version", + "state": "failed", + "start_time": "2023-01-25T09:48:56.116429738Z", + "end_time": "2023-01-25T10:01:22.651439285Z", + "error": "Timed out after 600.001s.\nExpected success, but got an error:\n <*errors.errorString | 0xc0003f5da0>: {\n s: \"Assertion in callback at /home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go:266 failed:\\nExpected\\n : 0\\nto equal\\n : 1\",\n }\n Assertion in callback at /home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go:266 failed:\n Expected\n : 0\n to equal\n : 1", + "error_file": "/home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go", + "error_line": 268, + "platform": "local", + "postgres_kind": "PostgreSQL", + "redwood": false, + "matrix_id": "local-v1.23.13-PGD-PostgreSQL-14.6-5.0.0-0.0git256.g2a1b38b.1.dev-1", + "postgres_version": "14.6-5.0.0-0.0git256.g2a1b38b.1.dev-1", + "k8s_version": "v1.23.13", + "workflow_id": 4004501040, + "repo": "EnterpriseDB/pg4k-pgd", + "branch": "dev/cnp-3285-2" +} diff --git a/example-artifacts/local-v1.24.7-PGD-PostgreSQL-14.6-5.0.0-0.0git256.g2a1b38b.1.dev-1_86b346ab9958ad86dc1829a4b9f509415ce8da7d997e5ea40bb24af8.json b/example-artifacts/local-v1.24.7-PGD-PostgreSQL-14.6-5.0.0-0.0git256.g2a1b38b.1.dev-1_86b346ab9958ad86dc1829a4b9f509415ce8da7d997e5ea40bb24af8.json index 7495b91..0e51d3d 100644 --- a/example-artifacts/local-v1.24.7-PGD-PostgreSQL-14.6-5.0.0-0.0git256.g2a1b38b.1.dev-1_86b346ab9958ad86dc1829a4b9f509415ce8da7d997e5ea40bb24af8.json +++ b/example-artifacts/local-v1.24.7-PGD-PostgreSQL-14.6-5.0.0-0.0git256.g2a1b38b.1.dev-1_86b346ab9958ad86dc1829a4b9f509415ce8da7d997e5ea40bb24af8.json @@ -1 +1,18 @@ -{"name": "Update operand -- upgrades operand version", "state": "failed", "start_time": "2023-01-25T10:42:28.730589019Z", "end_time": "2023-01-25T10:54:53.661654127Z", "error": "Timed out after 600.000s.\nExpected success, but got an error:\n <*errors.errorString | 0xc000c581b0>: {\n s: \"Assertion in callback at /home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go:266 failed:\\nExpected\\n : 0\\nto equal\\n : 1\",\n }\n Assertion in callback at /home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go:266 failed:\n Expected\n : 0\n to equal\n : 1", "error_file": "/home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go", "error_line": 268, "platform": "local", "postgres_kind": "PostgreSQL", "redwood": false, "matrix_id": "local-v1.24.7-PGD-PostgreSQL-14.6-5.0.0-0.0git256.g2a1b38b.1.dev-1", "postgres_version": "14.6-5.0.0-0.0git256.g2a1b38b.1.dev-1", "k8s_version": "v1.24.7", "workflow_id": 4004501040, "repo": "EnterpriseDB/pg4k-pgd", "branch": "dev/cnp-3285-2"} \ No newline at end of file +{ + "name": "Update operand -- upgrades operand version", + "state": "failed", + "start_time": "2023-01-25T10:42:28.730589019Z", + "end_time": "2023-01-25T10:54:53.661654127Z", + "error": "Timed out after 600.000s.\nExpected success, but got an error:\n <*errors.errorString | 0xc000c581b0>: {\n s: \"Assertion in callback at /home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go:266 failed:\\nExpected\\n : 0\\nto equal\\n : 1\",\n }\n Assertion in callback at /home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go:266 failed:\n Expected\n : 0\n to equal\n : 1", + "error_file": "/home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go", + "error_line": 268, + "platform": "local", + "postgres_kind": "PostgreSQL", + "redwood": false, + "matrix_id": "local-v1.24.7-PGD-PostgreSQL-14.6-5.0.0-0.0git256.g2a1b38b.1.dev-1", + "postgres_version": "14.6-5.0.0-0.0git256.g2a1b38b.1.dev-1", + "k8s_version": "v1.24.7", + "workflow_id": 4004501040, + "repo": "EnterpriseDB/pg4k-pgd", + "branch": "dev/cnp-3285-2" +} diff --git a/example-artifacts/local-v1.25.3-PGD-PostgreSQL-14.6-5.0.0-0.0git256.g2a1b38b.1.dev-1_86b346ab9958ad86dc1829a4b9f509415ce8da7d997e5ea40bb24af8.json b/example-artifacts/local-v1.25.3-PGD-PostgreSQL-14.6-5.0.0-0.0git256.g2a1b38b.1.dev-1_86b346ab9958ad86dc1829a4b9f509415ce8da7d997e5ea40bb24af8.json index 95bb59f..7b33f9f 100644 --- a/example-artifacts/local-v1.25.3-PGD-PostgreSQL-14.6-5.0.0-0.0git256.g2a1b38b.1.dev-1_86b346ab9958ad86dc1829a4b9f509415ce8da7d997e5ea40bb24af8.json +++ b/example-artifacts/local-v1.25.3-PGD-PostgreSQL-14.6-5.0.0-0.0git256.g2a1b38b.1.dev-1_86b346ab9958ad86dc1829a4b9f509415ce8da7d997e5ea40bb24af8.json @@ -1 +1,18 @@ -{"name": "Update operand -- upgrades operand version", "state": "failed", "start_time": "2023-01-25T10:18:24.30448771Z", "end_time": "2023-01-25T10:30:42.854980755Z", "error": "Timed out after 600.000s.\nExpected success, but got an error:\n <*errors.errorString | 0xc00047d7b0>: {\n s: \"Assertion in callback at /home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go:266 failed:\\nExpected\\n : 0\\nto equal\\n : 1\",\n }\n Assertion in callback at /home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go:266 failed:\n Expected\n : 0\n to equal\n : 1", "error_file": "/home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go", "error_line": 268, "platform": "local", "postgres_kind": "PostgreSQL", "redwood": false, "matrix_id": "local-v1.25.3-PGD-PostgreSQL-14.6-5.0.0-0.0git256.g2a1b38b.1.dev-1", "postgres_version": "14.6-5.0.0-0.0git256.g2a1b38b.1.dev-1", "k8s_version": "v1.25.3", "workflow_id": 4004501040, "repo": "EnterpriseDB/pg4k-pgd", "branch": "dev/cnp-3285-2"} \ No newline at end of file +{ + "name": "Update operand -- upgrades operand version", + "state": "failed", + "start_time": "2023-01-25T10:18:24.30448771Z", + "end_time": "2023-01-25T10:30:42.854980755Z", + "error": "Timed out after 600.000s.\nExpected success, but got an error:\n <*errors.errorString | 0xc00047d7b0>: {\n s: \"Assertion in callback at /home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go:266 failed:\\nExpected\\n : 0\\nto equal\\n : 1\",\n }\n Assertion in callback at /home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go:266 failed:\n Expected\n : 0\n to equal\n : 1", + "error_file": "/home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go", + "error_line": 268, + "platform": "local", + "postgres_kind": "PostgreSQL", + "redwood": false, + "matrix_id": "local-v1.25.3-PGD-PostgreSQL-14.6-5.0.0-0.0git256.g2a1b38b.1.dev-1", + "postgres_version": "14.6-5.0.0-0.0git256.g2a1b38b.1.dev-1", + "k8s_version": "v1.25.3", + "workflow_id": 4004501040, + "repo": "EnterpriseDB/pg4k-pgd", + "branch": "dev/cnp-3285-2" +} diff --git a/example-artifacts/local-v1.26.0-PGD-PostgreSQL-12.13-5.0.0-0.0git256.g2a1b38b.1.dev-1_86b346ab9958ad86dc1829a4b9f509415ce8da7d997e5ea40bb24af8.json b/example-artifacts/local-v1.26.0-PGD-PostgreSQL-12.13-5.0.0-0.0git256.g2a1b38b.1.dev-1_86b346ab9958ad86dc1829a4b9f509415ce8da7d997e5ea40bb24af8.json index c378dc4..889bbb7 100644 --- a/example-artifacts/local-v1.26.0-PGD-PostgreSQL-12.13-5.0.0-0.0git256.g2a1b38b.1.dev-1_86b346ab9958ad86dc1829a4b9f509415ce8da7d997e5ea40bb24af8.json +++ b/example-artifacts/local-v1.26.0-PGD-PostgreSQL-12.13-5.0.0-0.0git256.g2a1b38b.1.dev-1_86b346ab9958ad86dc1829a4b9f509415ce8da7d997e5ea40bb24af8.json @@ -1 +1,18 @@ -{"name": "Update operand -- upgrades operand version", "state": "failed", "start_time": "2023-01-25T10:37:14.031311318Z", "end_time": "2023-01-25T10:49:32.42470834Z", "error": "Timed out after 600.001s.\nExpected success, but got an error:\n <*errors.errorString | 0xc0000b3770>: {\n s: \"Assertion in callback at /home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go:266 failed:\\nExpected\\n : 0\\nto equal\\n : 1\",\n }\n Assertion in callback at /home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go:266 failed:\n Expected\n : 0\n to equal\n : 1", "error_file": "/home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go", "error_line": 268, "platform": "local", "postgres_kind": "PostgreSQL", "redwood": false, "matrix_id": "local-v1.26.0-PGD-PostgreSQL-12.13-5.0.0-0.0git256.g2a1b38b.1.dev-1", "postgres_version": "12.13-5.0.0-0.0git256.g2a1b38b.1.dev-1", "k8s_version": "v1.26.0", "workflow_id": 4004501040, "repo": "EnterpriseDB/pg4k-pgd", "branch": "dev/cnp-3285-2"} \ No newline at end of file +{ + "name": "Update operand -- upgrades operand version", + "state": "failed", + "start_time": "2023-01-25T10:37:14.031311318Z", + "end_time": "2023-01-25T10:49:32.42470834Z", + "error": "Timed out after 600.001s.\nExpected success, but got an error:\n <*errors.errorString | 0xc0000b3770>: {\n s: \"Assertion in callback at /home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go:266 failed:\\nExpected\\n : 0\\nto equal\\n : 1\",\n }\n Assertion in callback at /home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go:266 failed:\n Expected\n : 0\n to equal\n : 1", + "error_file": "/home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go", + "error_line": 268, + "platform": "local", + "postgres_kind": "PostgreSQL", + "redwood": false, + "matrix_id": "local-v1.26.0-PGD-PostgreSQL-12.13-5.0.0-0.0git256.g2a1b38b.1.dev-1", + "postgres_version": "12.13-5.0.0-0.0git256.g2a1b38b.1.dev-1", + "k8s_version": "v1.26.0", + "workflow_id": 4004501040, + "repo": "EnterpriseDB/pg4k-pgd", + "branch": "dev/cnp-3285-2" +} diff --git a/example-artifacts/local-v1.26.0-PGD-PostgreSQL-14.6-5.0.0-0.0git256.g2a1b38b.1.dev-1_86b346ab9958ad86dc1829a4b9f509415ce8da7d997e5ea40bb24af8.json b/example-artifacts/local-v1.26.0-PGD-PostgreSQL-14.6-5.0.0-0.0git256.g2a1b38b.1.dev-1_86b346ab9958ad86dc1829a4b9f509415ce8da7d997e5ea40bb24af8.json index 02693e7..31ef0bd 100644 --- a/example-artifacts/local-v1.26.0-PGD-PostgreSQL-14.6-5.0.0-0.0git256.g2a1b38b.1.dev-1_86b346ab9958ad86dc1829a4b9f509415ce8da7d997e5ea40bb24af8.json +++ b/example-artifacts/local-v1.26.0-PGD-PostgreSQL-14.6-5.0.0-0.0git256.g2a1b38b.1.dev-1_86b346ab9958ad86dc1829a4b9f509415ce8da7d997e5ea40bb24af8.json @@ -1 +1,18 @@ -{"name": "Update operand -- upgrades operand version", "state": "failed", "start_time": "2023-01-25T09:56:02.837927223Z", "end_time": "2023-01-25T10:07:59.547469753Z", "error": "Timed out after 600.000s.\nExpected success, but got an error:\n <*errors.errorString | 0xc000992790>: {\n s: \"Assertion in callback at /home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go:266 failed:\\nExpected\\n : 0\\nto equal\\n : 1\",\n }\n Assertion in callback at /home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go:266 failed:\n Expected\n : 0\n to equal\n : 1", "error_file": "/home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go", "error_line": 268, "platform": "local", "postgres_kind": "PostgreSQL", "redwood": false, "matrix_id": "local-v1.26.0-PGD-PostgreSQL-14.6-5.0.0-0.0git256.g2a1b38b.1.dev-1", "postgres_version": "14.6-5.0.0-0.0git256.g2a1b38b.1.dev-1", "k8s_version": "v1.26.0", "workflow_id": 4004501040, "repo": "EnterpriseDB/pg4k-pgd", "branch": "dev/cnp-3285-2"} \ No newline at end of file +{ + "name": "Update operand -- upgrades operand version", + "state": "failed", + "start_time": "2023-01-25T09:56:02.837927223Z", + "end_time": "2023-01-25T10:07:59.547469753Z", + "error": "Timed out after 600.000s.\nExpected success, but got an error:\n <*errors.errorString | 0xc000992790>: {\n s: \"Assertion in callback at /home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go:266 failed:\\nExpected\\n : 0\\nto equal\\n : 1\",\n }\n Assertion in callback at /home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go:266 failed:\n Expected\n : 0\n to equal\n : 1", + "error_file": "/home/runner/work/pg4k-pgd/pg4k-pgd/pgd-operator/tests/e2e/asserts_test.go", + "error_line": 268, + "platform": "local", + "postgres_kind": "PostgreSQL", + "redwood": false, + "matrix_id": "local-v1.26.0-PGD-PostgreSQL-14.6-5.0.0-0.0git256.g2a1b38b.1.dev-1", + "postgres_version": "14.6-5.0.0-0.0git256.g2a1b38b.1.dev-1", + "k8s_version": "v1.26.0", + "workflow_id": 4004501040, + "repo": "EnterpriseDB/pg4k-pgd", + "branch": "dev/cnp-3285-2" +} diff --git a/example-artifacts/local-v1.26.3-PostgreSQL-15.2_9891d0d1caa1ec8fd0adfe622e341f84dd3d1df1cffee843e1fb84a0.json b/example-artifacts/local-v1.26.3-PostgreSQL-15.2_9891d0d1caa1ec8fd0adfe622e341f84dd3d1df1cffee843e1fb84a0.json new file mode 100644 index 0000000..2337f13 --- /dev/null +++ b/example-artifacts/local-v1.26.3-PostgreSQL-15.2_9891d0d1caa1ec8fd0adfe622e341f84dd3d1df1cffee843e1fb84a0.json @@ -0,0 +1,17 @@ +{ + "name": "ignoreFails on e2e tests -- generates a failing tests that should be ignored", + "state": "ignoreFailed", + "start_time": "2023-04-10T03:05:10.146218086Z", + "end_time": "2023-04-10T03:05:10.182348615Z", + "error": "Expected\n : true\nto be false", + "error_file": "/home/runner/work/cloudnative-pg/cloudnative-pg/tests/e2e/ignorefails_test.go", + "error_line": 31, + "platform": "local", + "postgres_kind": "PostgreSQL", + "matrix_id": "local-v1.26.3-PostgreSQL-15.2", + "postgres_version": "15.2", + "k8s_version": "v1.26.3", + "workflow_id": 4653592818, + "repo": "cloudnative-pg/cloudnative-pg", + "branch": "main" +} diff --git a/few-artifacts/id1_0b185c51a60964ecab5bb7d97458ca95fd421f325f3896ed239d5d3f.json b/few-artifacts/id1_0b185c51a60964ecab5bb7d97458ca95fd421f325f3896ed239d5d3f.json new file mode 100644 index 0000000..9ad974f --- /dev/null +++ b/few-artifacts/id1_0b185c51a60964ecab5bb7d97458ca95fd421f325f3896ed239d5d3f.json @@ -0,0 +1,17 @@ +{ + "name": "InitDB settings - custom default locale -- use the custom default locale specified", + "state": "passed", + "start_time": "2021-11-29T18:29:12.613387+01:00", + "end_time": "2021-11-29T18:31:07.988145+01:00", + "error": "", + "error_file": "", + "error_line": 0, + "platform": "local", + "postgres_kind": "PostgreSQL", + "matrix_id": "id1", + "postgres_version": "11.1", + "k8s_version": "22", + "workflow_id": 12, + "repo": "my-repo", + "branch": "my-branch" +} diff --git a/few-artifacts/id1_4902843ff6a60bc4fdb76000698c46de8e7f9763a1a0fe63f70fd5f8.json b/few-artifacts/id1_4902843ff6a60bc4fdb76000698c46de8e7f9763a1a0fe63f70fd5f8.json new file mode 100644 index 0000000..41628dd --- /dev/null +++ b/few-artifacts/id1_4902843ff6a60bc4fdb76000698c46de8e7f9763a1a0fe63f70fd5f8.json @@ -0,0 +1,17 @@ +{ + "name": "InitDB settings - initdb custom post-init SQL scripts -- can find the tables created by the post-init SQL queries", + "state": "failed", + "start_time": "2021-11-29T18:28:37.198931+01:00", + "end_time": "2021-11-29T18:29:12.613157+01:00", + "error": "Expected\n : 1\\n\nto equal\n : 2\\n", + "error_file": "/Users/jaime.silvela/repos/cloud-native-postgresql/tests/e2e/initdb_test.go", + "error_line": 80, + "platform": "local", + "postgres_kind": "PostgreSQL", + "matrix_id": "id1", + "postgres_version": "11.1", + "k8s_version": "22", + "workflow_id": 12, + "repo": "my-repo", + "branch": "my-branch" +} diff --git a/few-artifacts/id1_9891d0d1caa1ec8fd0adfe622e341f84dd3d1df1cffee843e1fb84a0.json b/few-artifacts/id1_9891d0d1caa1ec8fd0adfe622e341f84dd3d1df1cffee843e1fb84a0.json new file mode 100644 index 0000000..d2d2d66 --- /dev/null +++ b/few-artifacts/id1_9891d0d1caa1ec8fd0adfe622e341f84dd3d1df1cffee843e1fb84a0.json @@ -0,0 +1,17 @@ +{ + "name": "ignoreFails on e2e tests -- generates a failing tests that should be ignored", + "state": "ignoreFailed", + "start_time": "0001-01-01T00:00:00Z", + "end_time": "0001-01-01T00:00:00Z", + "error": "", + "error_file": "", + "error_line": 0, + "platform": "local", + "postgres_kind": "PostgreSQL", + "matrix_id": "id1", + "postgres_version": "11.1", + "k8s_version": "22", + "workflow_id": 12, + "repo": "my-repo", + "branch": "my-branch" +} diff --git a/summarize_test_results.py b/summarize_test_results.py index 5de01ba..d8327c0 100644 --- a/summarize_test_results.py +++ b/summarize_test_results.py @@ -190,6 +190,7 @@ def track_time_taken(test_results, test_times, suite_times): if suite_times["end_time"][platform][matrix_id] < end_time: suite_times["end_time"][platform][matrix_id] = end_time + def count_bucketed_by_test(test_results, by_test): """counts the successes, failures, failing versions of kubernetes, failing versions of postgres, bucketed by test name. @@ -223,12 +224,13 @@ def count_bucketed_by_test(test_results, by_test): by_test["pg_versions_failed"][name][pg_version] = True by_test["platforms_failed"][name][platform] = True + def count_bucketed_by_code(test_results, by_failing_code): """buckets by failed code, with a list of tests where the assertion fails, and a view of the stack trace. """ name = test_results["name"] - if test_results["error"] == "": + if test_results["error"] == "" or test_results["state"] == "ignoreFailed": return errfile = test_results["error_file"] @@ -254,6 +256,7 @@ def count_bucketed_by_code(test_results, by_failing_code): if err_desc not in by_failing_code["errors"]: by_failing_code["errors"][err_desc] = test_results["error"] + def count_bucketized_stats(test_results, buckets, field_id): """counts the success/failures onto a bucket. This means there are two dictionaries: one for `total` tests, one for `failed` tests. @@ -397,6 +400,57 @@ def compile_overview(summary): } +def format_alerts(summary, with_header=True, file_out=None): + """print Alerts for tests that have failed systematically + + We want to capture: + - all test combinations failed (if this happens, no more investigation needed) + - a test failed systematically (all combinations of this test failed) + - a PG version failed systematically (all tests in that PG version failed) + - a K8s version failed systematically (all tests in that K8s version failed) + - a Platform failed systematically (all tests in that Platform failed) + + We require more than 1 "failure" to avoid flooding + """ + has_systematic_failures = False + + if summary["total_run"] == summary["total_failed"]: + if with_header: + print(f"## Alerts\n", file=file_out) + print(f"All test combinations failed\n", file=file_out) + return + + metric_name = { + "by_test": "Tests", + "by_k8s": "Kubernetes versions", + "by_postgres": "Postgres versions", + "by_platform": "Platforms", + } + + output = "" + for metric in ["by_test", "by_k8s", "by_postgres", "by_platform"]: + has_failure_in_metric = False + for bucket_hits in summary[metric]["failed"].items(): + bucket = bucket_hits[0] # the items() call returns (bucket, hits) pairs + failures = summary[metric]["failed"][bucket] + runs = summary[metric]["total"][bucket] + if failures == runs and failures > 1: + if not has_failure_in_metric: + output += f"{metric_name[metric]} with systematic failures:\n\n" + has_failure_in_metric = True + has_systematic_failures = True + output += f"- {bucket}: ({failures} out of {runs} tests failed)\n" + if has_failure_in_metric: + output += f"\n" + + if not has_systematic_failures: + return + + if with_header: + print(f"## Alerts\n", file=file_out) + print(f"{output}", end="", file=file_out) + + def format_overview(summary, structure, file_out=None): """print general test metrics""" print("## " + structure["title"] + "\n", file=file_out) @@ -473,6 +527,7 @@ def format_by_test(summary, structure, file_out=None): print(table, file=file_out) + def format_by_code(summary, structure, file_out=None): """print metrics bucketed by failing code""" title = structure["title"] @@ -494,7 +549,11 @@ def format_by_code(summary, structure, file_out=None): for bucket in sorted_by_code: tests = ", ".join(summary["by_code"]["tests"][bucket].keys()) # replace newlines and pipes to avoid interference with markdown tables - errors = summary["by_code"]["errors"][bucket].replace("\n", "
").replace("|", "—") + errors = ( + summary["by_code"]["errors"][bucket] + .replace("\n", "
") + .replace("|", "—") + ) err_cell = f"
Click to expand{errors}
" table.add_row( [ @@ -507,6 +566,7 @@ def format_by_code(summary, structure, file_out=None): print(table, file=file_out) + def format_duration(duration): """pretty-print duration""" minutes = duration.seconds // 60 @@ -537,6 +597,7 @@ def format_durations_table(test_times, structure, file_out=None): print(table, file=file_out) + def format_suite_durations_table(suite_times, structure, file_out=None): """print the table of durations for the whole suite, per platform""" title = structure["title"] @@ -557,7 +618,10 @@ def format_suite_durations_table(suite_times, structure, file_out=None): } for platform in suite_times["start_time"]: for matrix_id in suite_times["start_time"][platform]: - duration = suite_times["end_time"][platform][matrix_id] - suite_times["start_time"][platform][matrix_id] + duration = ( + suite_times["end_time"][platform][matrix_id] + - suite_times["start_time"][platform][matrix_id] + ) if platform not in suite_durations["max"]: suite_durations["max"][platform] = duration if platform not in suite_durations["min"]: @@ -584,6 +648,7 @@ def format_suite_durations_table(suite_times, structure, file_out=None): print(table, file=file_out) + def format_test_failures(summary, file_out=None): """creates the part of the test report that drills into the failures""" by_test_section = { @@ -687,6 +752,7 @@ def format_test_summary(summary, file_out=None): ], } + format_alerts(summary, file_out=file_out) format_overview(overview, overview_section, file_out=file_out) if summary["total_failed"] == 0: @@ -710,7 +776,9 @@ def format_test_summary(summary, file_out=None): ], } - format_suite_durations_table(summary["suite_durations"], suite_timing_section, file_out=file_out) + format_suite_durations_table( + summary["suite_durations"], suite_timing_section, file_out=file_out + ) timing_section = { "title": "Test times", @@ -726,6 +794,38 @@ def format_test_summary(summary, file_out=None): format_durations_table(summary["test_durations"], timing_section, file_out=file_out) +def format_short_test_summary(summary, file_out=None): + """creates a Markdown document with a short test overview, useful as fallback + if the proper test summary exceeds GitHub capacity. + Outputs to stdout like a good 12-factor-app citizen, unless the `file_out` + argument is provided + """ + + print( + "This is an abridged test summary, in place of the full test summary which exceeds" + + " the GitHub limit for a summary. Please look for the full summary as an Artifact.", + file=file_out, + ) + print(file=file_out) + overview = compile_overview(summary) + + overview_section = { + "title": "Overview", + "header": ["failed", "out of", ""], + "rows": [ + ["test combinations", "total_failed", "total_run"], + ["unique tests", "unique_failed", "unique_run"], + ["matrix branches", "matrix_failed", "matrix_run"], + ["k8s versions", "k8s_failed", "k8s_run"], + ["postgres versions", "postgres_failed", "postgres_run"], + ["platforms", "platform_failed", "platform_run"], + ], + } + + format_alerts(summary, file_out=file_out) + format_overview(overview, overview_section, file_out=file_out) + + if __name__ == "__main__": parser = argparse.ArgumentParser(description="Summarize the E2E Suite results") @@ -741,6 +841,18 @@ def format_test_summary(summary, file_out=None): type=str, help="output file", ) + parser.add_argument( + "-s", + "--short", + type=str, + help="short output file", + ) + parser.add_argument( + "-a", + "--alerts", + type=str, + help="short output file", + ) args = parser.parse_args() @@ -750,3 +862,11 @@ def format_test_summary(summary, file_out=None): format_test_summary(test_summary, file_out=f) else: format_test_summary(test_summary) + + if args.short: + with open(args.short, "w") as f: + format_short_test_summary(test_summary, file_out=f) + + if args.alerts: + with open(args.alerts, "w") as f: + format_alerts(test_summary, with_header=False, file_out=f) diff --git a/test_summary.py b/test_summary.py new file mode 100644 index 0000000..d1b81cf --- /dev/null +++ b/test_summary.py @@ -0,0 +1,70 @@ +# +# Copyright The CloudNativePG Contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import unittest +import summarize_test_results +import datetime + + +class TestIsFailed(unittest.TestCase): + def test_compute_summary(self): + self.maxDiff = None + summary = summarize_test_results.compute_test_summary("few-artifacts") + self.assertEqual(summary["total_run"], 3) + self.assertEqual(summary["total_failed"], 1) + + self.assertEqual( + summary["by_code"]["total"], + { + "/Users/jaime.silvela/repos/cloud-native-postgresql/tests/e2e/initdb_test.go:80": 1 + }, + "unexpected summary", + ) + self.assertEqual( + summary["by_code"]["tests"], + { + "/Users/jaime.silvela/repos/cloud-native-postgresql/tests/e2e/initdb_test.go:80": { + "InitDB settings - initdb custom post-init SQL scripts -- can find the tables created by the post-init SQL queries": True + } + }, + "unexpected summary", + ) + self.assertEqual( + summary["by_matrix"], {"total": {"id1": 3}, "failed": {"id1": 1}} + ) + self.assertEqual(summary["by_k8s"], {"total": {"22": 3}, "failed": {"22": 1}}) + self.assertEqual( + summary["by_platform"], {"total": {"local": 3}, "failed": {"local": 1}} + ) + self.assertEqual( + summary["by_postgres"], + {"total": {"PostgreSQL-11.1": 3}, "failed": {"PostgreSQL-11.1": 1}}, + ) + self.assertEqual( + summary["suite_durations"], + { + "end_time": { + "local": {"id1": datetime.datetime(2021, 11, 29, 18, 31, 7)} + }, + "start_time": { + "local": {"id1": datetime.datetime(2021, 11, 29, 18, 28, 37)} + }, + }, + ) + + +if __name__ == "__main__": + unittest.main()