From d18304513b29e5dcecac037a1310247d7fe34797 Mon Sep 17 00:00:00 2001 From: Stephen Rosen Date: Wed, 8 Feb 2023 06:14:30 +0000 Subject: [PATCH 1/2] Add a check for tests which are always skipped This takes the form of a new script which crawls pytest XML reports and collates them into a single aggregate. It checks for tests which are skipped or missing in all of the reports. The aggregator can be run along with a suite of tox environments via `make collatd-test-report`, and a new CI job runs this in a build. --- .github/workflows/build.yaml | 27 ++++++++++++++++++ Makefile | 10 +++++++ scripts/aggregate-pytest-reports.py | 43 +++++++++++++++++++++++++++++ 3 files changed, 80 insertions(+) create mode 100644 scripts/aggregate-pytest-reports.py diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index ca74b8be1..09aad1e6a 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -78,6 +78,33 @@ jobs: python -m tox run-parallel -m ci python -m tox run -e cov + collate-tests: + runs-on: ubuntu-latest + name: "Collate results to check for skipped tests" + steps: + - uses: actions/checkout@v3 + + - name: get date for caching + run: /bin/date -u "+%U" > cachedate.txt + shell: bash + + - uses: actions/setup-python@v4 + with: + python-version: | + 3.7 + 3.10 + 3.11 + cache: "pip" + cache-dependency-path: | + .github/workflows/build.yaml + setup.cfg + tox.ini + cachedate.txt + + - run: python -m pip install tox + + - run: make collated-test-report + self-check: name: "Self-Check" runs-on: ubuntu-latest diff --git a/Makefile b/Makefile index b08b23793..1cf0f9ee4 100644 --- a/Makefile +++ b/Makefile @@ -17,6 +17,16 @@ release: -git push $(shell git rev-parse --abbrev-ref @{push} | cut -d '/' -f1) refs/tags/$(PKG_VERSION) tox run -e publish-release +.PHONY: collated-test-report +collated-test-report: + tox p -e py37-mindeps,py311,py310-notoml,py310-tomli-format,py311-json5 -- '--junitxml={envdir}/pytest.xml' + python ./scripts/aggregate-pytest-reports.py \ + .tox/py37-mindeps/pytest.xml \ + .tox/py311/pytest.xml \ + .tox/py310-notoml/pytest.xml \ + .tox/py310-tomli-format/pytest.xml \ + .tox/py311-json5/pytest.xml + .PHONY: clean clean: rm -rf dist build *.egg-info .tox .coverage.* diff --git a/scripts/aggregate-pytest-reports.py b/scripts/aggregate-pytest-reports.py new file mode 100644 index 000000000..5d108f787 --- /dev/null +++ b/scripts/aggregate-pytest-reports.py @@ -0,0 +1,43 @@ +import argparse +import sys +from collections import defaultdict +from xml.etree import ElementTree # nosec + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("FILES", nargs="+") + args = parser.parse_args() + + tests_by_name = defaultdict(dict) + for filename in args.FILES: + tree = ElementTree.parse(filename) + root = tree.getroot() + + for testcase in root.findall("./testsuite/testcase"): + classname = testcase.get("classname") + name = testcase.get("name") + nodename = f"{classname.replace('.', '/')}.py::{name}" + + skip_node = testcase.find("skipped") + if skip_node is not None: + if "skipped" not in tests_by_name[nodename]: + tests_by_name[nodename]["skipped"] = True + else: + tests_by_name[nodename]["skipped"] = False + + fail = False + for nodename, attributes in tests_by_name.items(): + if attributes.get("skipped") is True: + print(f"ALWAYS SKIPPED: {nodename}") + fail = True + + if fail: + print("fail") + sys.exit(1) + print("ok") + sys.exit(0) + + +if __name__ == "__main__": + main() From 42d21dc9557358b7a4859b8e7ff94d29c5d8d46e Mon Sep 17 00:00:00 2001 From: Stephen Rosen Date: Wed, 8 Feb 2023 07:20:39 +0000 Subject: [PATCH 2/2] Don't run tests twice in CI As a first draft, tests ran twice (one for the "test" and once for the skipped test collator). Switch to upload/download of junitxml report data to pass the reports from the matrix to the collator. --- .github/workflows/build.yaml | 34 ++++++++++++++-------------------- Makefile | 9 ++------- tox.ini | 5 +++-- 3 files changed, 19 insertions(+), 29 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 09aad1e6a..8dc5c49ff 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -75,35 +75,29 @@ jobs: - name: test run: | - python -m tox run-parallel -m ci + python -m tox run-parallel -m ci -- --junitxml pytest.{envname}.xml python -m tox run -e cov + - uses: actions/upload-artifact@v3 + with: + name: pytest-report-py${{ matrix.py }}-${{ matrix.os }} + path: pytest.*.xml + collate-tests: + needs: [ci-test-matrix] runs-on: ubuntu-latest name: "Collate results to check for skipped tests" steps: - uses: actions/checkout@v3 - - - name: get date for caching - run: /bin/date -u "+%U" > cachedate.txt - shell: bash - - uses: actions/setup-python@v4 with: - python-version: | - 3.7 - 3.10 - 3.11 - cache: "pip" - cache-dependency-path: | - .github/workflows/build.yaml - setup.cfg - tox.ini - cachedate.txt - - - run: python -m pip install tox - - - run: make collated-test-report + python-version: "3.x" + # download everything + - uses: actions/download-artifact@v3 + with: + path: artifacts/ + # collate and report + - run: python ./scripts/aggregate-pytest-reports.py artifacts/*/pytest.*.xml self-check: name: "Self-Check" diff --git a/Makefile b/Makefile index 1cf0f9ee4..0858d9d04 100644 --- a/Makefile +++ b/Makefile @@ -19,13 +19,8 @@ release: .PHONY: collated-test-report collated-test-report: - tox p -e py37-mindeps,py311,py310-notoml,py310-tomli-format,py311-json5 -- '--junitxml={envdir}/pytest.xml' - python ./scripts/aggregate-pytest-reports.py \ - .tox/py37-mindeps/pytest.xml \ - .tox/py311/pytest.xml \ - .tox/py310-notoml/pytest.xml \ - .tox/py310-tomli-format/pytest.xml \ - .tox/py311-json5/pytest.xml + tox p + python ./scripts/aggregate-pytest-reports.py .tox/*/pytest.xml .PHONY: clean clean: diff --git a/tox.ini b/tox.ini index c11d936a4..8f2df9a06 100644 --- a/tox.ini +++ b/tox.ini @@ -11,7 +11,7 @@ skip_missing_interpreters = true minversion = 4.0.0 labels = - ci = py-notoml, py-tomli-format, py-json5, py-pyjson5 + ci = py, py-notoml, py-tomli-format, py-json5, py-pyjson5 [testenv] description = "run tests with pytest" @@ -28,7 +28,8 @@ deps = format: jsonschema[format] set_env = notoml: FORCE_TOML_DISABLED=1 -commands = coverage run -m pytest {posargs} +commands = + coverage run -m pytest {posargs:--junitxml={envdir}/pytest.xml} [testenv:cov_clean] description = "erase coverage data to prepare for a new run"