From 8d283aa235fd2dc95e3bda5d75bbb3e9133e118a Mon Sep 17 00:00:00 2001 From: AlexWells Date: Tue, 30 May 2023 12:23:29 +0100 Subject: [PATCH 01/11] Improve container build workflow This will test the container before it is pushed to GHCR --- .github/workflows/code.yml | 65 +++++++++++++++++++++++++++----------- 1 file changed, 46 insertions(+), 19 deletions(-) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index fb468b23d..4e5e7e233 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -115,6 +115,9 @@ jobs: contents: read packages: write + env: + TEST_TAG: "testing" + steps: - name: Checkout uses: actions/checkout@v3 @@ -136,42 +139,66 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Docker meta + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + + - name: Build and export to Docker local cache + uses: docker/build-push-action@v4 + with: + # Note build-args, context, file, and target must all match between this + # step and the later build-push-action, otherwise the second build-push-action + # will attempt to build the image again + build-args: | + PIP_OPTIONS=-r lockfiles/requirements.txt dist/*.whl + context: artifacts/ + file: ./Dockerfile + target: runtime + load: true + tags: ${{ env.TEST_TAG }} + # If you have a long docker build (2+ minutes), uncomment the + # following to turn on caching. For short build times this + # makes it a little slower + #cache-from: type=gha + #cache-to: type=gha,mode=max + + - name: Test cli works in cached runtime image + run: docker run docker.io/library/${{ env.TEST_TAG }} --version + + - name: Create tags for publishing image id: meta uses: docker/metadata-action@v4 with: images: ${{ env.IMAGE_REPOSITORY }} tags: | type=ref,event=tag - type=raw,value=latest + type=raw,value=latest, enable=${{ github.ref_type == 'tag' }} + # type=edge,branch=main + # Add line above to generate image for every commit to given branch, + # and uncomment the end of if clause in next step - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v2 - - - name: Build runtime image + - name: Push cached image to container registry + if: github.ref_type == 'tag' # || github.ref_name == 'main' uses: docker/build-push-action@v3 + # This does not build the image again, it will find the image in the + # Docker cache and publish it with: + # Note build-args, context, file, and target must all match between this + # step and the previous build-push-action, otherwise this step will + # attempt to build the image again build-args: | PIP_OPTIONS=-r lockfiles/requirements.txt dist/*.whl - push: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags') }} - load: ${{ ! (github.event_name == 'push' && startsWith(github.ref, 'refs/tags')) }} - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} context: artifacts/ file: ./Dockerfile - # If you have a long docker build, uncomment the following to turn on caching - # For short build times this makes it a little slower - #cache-from: type=gha - #cache-to: type=gha,mode=max - - - name: Test cli works in runtime image - run: docker run ${{ env.IMAGE_REPOSITORY }} --version + target: runtime + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} release: # upload to PyPI and make a release on every tag needs: [lint, dist, test] - if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags') }} + if: ${{ github.event_name == 'push' && github.ref_type == 'tag' }} runs-on: ubuntu-latest env: HAS_PYPI_TOKEN: ${{ secrets.PYPI_TOKEN != '' }} From c68c75e2fa2578d2f2609abaf20a09b19b1be03e Mon Sep 17 00:00:00 2001 From: Eva Lott Date: Fri, 2 Jun 2023 15:46:56 +0100 Subject: [PATCH 02/11] Changed coverage parameters --- .github/workflows/code.yml | 2 +- .vscode/settings.json | 7 +++++-- pyproject.toml | 3 +-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index 4e5e7e233..cbc3e2801 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -63,7 +63,7 @@ jobs: run: pipdeptree - name: Run tests - run: pytest + run: tox -e pytest - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 diff --git a/.vscode/settings.json b/.vscode/settings.json index 2472acfd6..35402dc50 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -3,7 +3,10 @@ "python.linting.flake8Enabled": true, "python.linting.mypyEnabled": true, "python.linting.enabled": true, - "python.testing.pytestArgs": [], + "python.testing.pytestArgs": [ + "--cov=python3_pip_skeleton", + "--cov-report xml:cov.xml" + ], "python.testing.unittestEnabled": false, "python.testing.pytestEnabled": true, "python.formatting.provider": "black", @@ -12,4 +15,4 @@ "editor.codeActionsOnSave": { "source.organizeImports": true } -} \ No newline at end of file +} diff --git a/pyproject.toml b/pyproject.toml index 2aa5efc0d..ec8e6931d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,7 +75,6 @@ exclude = [".tox", "venv"] # Run pytest with all our checkers, and don't spam us with massive tracebacks on error addopts = """ --tb=native -vv --doctest-modules --doctest-glob="*.rst" - --cov=python3_pip_skeleton --cov-report term --cov-report xml:cov.xml """ # https://iscinumpy.gitlab.io/post/bound-version-constraints/#watch-for-warnings filterwarnings = "error" @@ -107,7 +106,7 @@ allowlist_externals = sphinx-build sphinx-autobuild commands = - pytest: pytest {posargs} + pytest: pytest --cov=python3_pip_skeleton --cov-report term --cov-report xml:cov.xml {posargs} mypy: mypy src tests {posargs} pre-commit: pre-commit run --all-files {posargs} docs: sphinx-{posargs:build -EW --keep-going} -T docs build/html From f02b12ec81d82a39cb4022ffd6e8c3f3f80ac711 Mon Sep 17 00:00:00 2001 From: Eva Lott Date: Mon, 5 Jun 2023 08:58:00 +0100 Subject: [PATCH 03/11] Added vscode settings --- .vscode/settings.json | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 35402dc50..b8cc9ad67 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -3,10 +3,7 @@ "python.linting.flake8Enabled": true, "python.linting.mypyEnabled": true, "python.linting.enabled": true, - "python.testing.pytestArgs": [ - "--cov=python3_pip_skeleton", - "--cov-report xml:cov.xml" - ], + "python.testing.pytestArgs": ["--cov=python3_pip_skeleton", "--cov-report", "xml:cov.xml"], "python.testing.unittestEnabled": false, "python.testing.pytestEnabled": true, "python.formatting.provider": "black", From 3415f1c8b7c9305600d06a6be460cab081556950 Mon Sep 17 00:00:00 2001 From: Eva Lott Date: Tue, 20 Jun 2023 11:29:18 +0100 Subject: [PATCH 04/11] Slight changes to the sed command, and ensuring output is utf-8 on windows --- .github/actions/install_requirements/action.yml | 2 +- .github/pages/make_switcher.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/install_requirements/action.yml b/.github/actions/install_requirements/action.yml index 25a146d16..c5d4db43b 100644 --- a/.github/actions/install_requirements/action.yml +++ b/.github/actions/install_requirements/action.yml @@ -32,7 +32,7 @@ runs: mkdir -p lockfiles pip freeze --exclude-editable > lockfiles/${{ inputs.requirements_file }} # delete the self referencing line and make sure it isn't blank - sed -i '/file:/d' lockfiles/${{ inputs.requirements_file }} + sed -i'' -e '/file:/d' lockfiles/${{ inputs.requirements_file }} shell: bash - name: Upload lockfiles diff --git a/.github/pages/make_switcher.py b/.github/pages/make_switcher.py index 39c127726..d70367aec 100755 --- a/.github/pages/make_switcher.py +++ b/.github/pages/make_switcher.py @@ -64,7 +64,7 @@ def write_json(path: Path, repository: str, versions: str): ] text = json.dumps(struct, indent=2) print(f"JSON switcher:\n{text}") - path.write_text(text) + path.write_text(text, encoding="utf-8") def main(args=None): From 553608123944a5e8c6e7843754f350fb6b52bc7f Mon Sep 17 00:00:00 2001 From: AlexWells Date: Fri, 1 Sep 2023 13:47:50 +0100 Subject: [PATCH 05/11] Remove trailing spaces and rationalise newlines Some tools remove trailing whitespace by default on save, so may as well correct the originals. Ensures every file ends with an empty blank line. Again some tools do this automatically. --- .github/actions/install_requirements/action.yml | 1 - .github/pages/index.html | 2 +- .github/workflows/linkcheck.yml | 2 +- .gitignore | 1 - .vscode/extensions.json | 2 +- .vscode/launch.json | 2 +- .vscode/tasks.json | 2 +- docs/developer/how-to/build-docs.rst | 2 +- docs/developer/how-to/lint.rst | 2 -- docs/developer/how-to/make-release.rst | 2 +- docs/developer/reference/standards.rst | 2 +- docs/developer/tutorials/dev-install.rst | 10 +++++----- pyproject.toml | 4 ++-- 13 files changed, 15 insertions(+), 19 deletions(-) diff --git a/.github/actions/install_requirements/action.yml b/.github/actions/install_requirements/action.yml index c5d4db43b..20d7a3adf 100644 --- a/.github/actions/install_requirements/action.yml +++ b/.github/actions/install_requirements/action.yml @@ -55,4 +55,3 @@ runs: fi fi shell: bash - diff --git a/.github/pages/index.html b/.github/pages/index.html index 80f0a0091..c495f39f2 100644 --- a/.github/pages/index.html +++ b/.github/pages/index.html @@ -8,4 +8,4 @@ - \ No newline at end of file + diff --git a/.github/workflows/linkcheck.yml b/.github/workflows/linkcheck.yml index 6b64fdea9..70cba115f 100644 --- a/.github/workflows/linkcheck.yml +++ b/.github/workflows/linkcheck.yml @@ -24,4 +24,4 @@ jobs: run: tox -e docs build -- -b linkcheck - name: Keepalive Workflow - uses: gautamkrishnar/keepalive-workflow@v1 \ No newline at end of file + uses: gautamkrishnar/keepalive-workflow@v1 diff --git a/.gitignore b/.gitignore index 9fbb6bfe0..62dcd9a8d 100644 --- a/.gitignore +++ b/.gitignore @@ -66,4 +66,3 @@ venv* # further build artifacts lockfiles/ - diff --git a/.vscode/extensions.json b/.vscode/extensions.json index 819229919..cda78d45e 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -6,4 +6,4 @@ "redhat.vscode-yaml", "ryanluker.vscode-coverage-gutters" ] -} \ No newline at end of file +} diff --git a/.vscode/launch.json b/.vscode/launch.json index f65cb376b..3cda74327 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -22,4 +22,4 @@ }, } ] -} \ No newline at end of file +} diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 946e69d4b..c999e8646 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -13,4 +13,4 @@ "problemMatcher": [], } ] -} \ No newline at end of file +} diff --git a/docs/developer/how-to/build-docs.rst b/docs/developer/how-to/build-docs.rst index 0174fc82d..11a5e6386 100644 --- a/docs/developer/how-to/build-docs.rst +++ b/docs/developer/how-to/build-docs.rst @@ -35,4 +35,4 @@ changes in this directory too:: $ tox -e docs autobuild -- --watch src -.. _sphinx: https://www.sphinx-doc.org/ \ No newline at end of file +.. _sphinx: https://www.sphinx-doc.org/ diff --git a/docs/developer/how-to/lint.rst b/docs/developer/how-to/lint.rst index 8f4e92dbb..ffb6ee5af 100644 --- a/docs/developer/how-to/lint.rst +++ b/docs/developer/how-to/lint.rst @@ -37,5 +37,3 @@ VSCode support The ``.vscode/settings.json`` will run black and isort formatters as well as flake8 checking on save. Issues will be highlighted in the editor window. - - diff --git a/docs/developer/how-to/make-release.rst b/docs/developer/how-to/make-release.rst index 747e44a22..d8f92f854 100644 --- a/docs/developer/how-to/make-release.rst +++ b/docs/developer/how-to/make-release.rst @@ -13,4 +13,4 @@ To make a new release, please follow this checklist: Note that tagging and pushing to the main branch has the same effect except that you will not get the option to edit the release notes. -.. _release: https://github.com/DiamondLightSource/python3-pip-skeleton/releases \ No newline at end of file +.. _release: https://github.com/DiamondLightSource/python3-pip-skeleton/releases diff --git a/docs/developer/reference/standards.rst b/docs/developer/reference/standards.rst index b78a719e1..06c4af539 100644 --- a/docs/developer/reference/standards.rst +++ b/docs/developer/reference/standards.rst @@ -61,4 +61,4 @@ Docs follow the underlining convention:: .. seealso:: - How-to guide `../how-to/build-docs` \ No newline at end of file + How-to guide `../how-to/build-docs` diff --git a/docs/developer/tutorials/dev-install.rst b/docs/developer/tutorials/dev-install.rst index 3a6627a14..183b1893a 100644 --- a/docs/developer/tutorials/dev-install.rst +++ b/docs/developer/tutorials/dev-install.rst @@ -37,10 +37,10 @@ requires python 3.8 or later) or to run in a container under `VSCode $ code python3-pip-skeleton # Click on 'Reopen in Container' when prompted # Open a new terminal - - .. note:: - - See the epics-containers_ documentation for more complex + + .. note:: + + See the epics-containers_ documentation for more complex use cases, such as integration with podman. See what was installed @@ -65,4 +65,4 @@ This will run in parallel the following checks: - `../how-to/lint` -.. _epics-containers: https://epics-containers.github.io/main/user/tutorials/devcontainer.html \ No newline at end of file +.. _epics-containers: https://epics-containers.github.io/main/user/tutorials/devcontainer.html diff --git a/pyproject.toml b/pyproject.toml index ec8e6931d..c84d39364 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -99,8 +99,8 @@ skipsdist=True # Don't create a virtualenv for the command, requires tox-direct plugin direct = True passenv = * -allowlist_externals = - pytest +allowlist_externals = + pytest pre-commit mypy sphinx-build From 454498e85022497c056984a0f1f2b291101b17c5 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Tue, 12 Sep 2023 13:21:36 +0100 Subject: [PATCH 06/11] test_database_render uses subst format --- src/ibek/gen_scripts.py | 11 +-- src/ibek/render.py | 43 ------------ src/ibek/render_db.py | 107 ++++++++++++++++++++++++++++++ src/ibek/templates/db.subst.jinja | 8 +++ src/ibek/templates/make_db.jinja | 2 - tests/conftest.py | 5 ++ tests/test_cli.py | 7 ++ tests/test_render.py | 28 ++++++-- 8 files changed, 155 insertions(+), 56 deletions(-) create mode 100644 src/ibek/render_db.py create mode 100644 src/ibek/templates/db.subst.jinja delete mode 100644 src/ibek/templates/make_db.jinja diff --git a/src/ibek/gen_scripts.py b/src/ibek/gen_scripts.py index 7fff7c5fc..3577c19b2 100644 --- a/src/ibek/gen_scripts.py +++ b/src/ibek/gen_scripts.py @@ -11,6 +11,7 @@ from .ioc import IOC, clear_entity_model_ids, make_entity_models, make_ioc_model from .render import Render +from .render_db import RenderDb from .support import Support log = logging.getLogger(__name__) @@ -66,13 +67,13 @@ def create_db_script(ioc_instance: IOC) -> str: Create make_db.sh script for expanding the database templates """ with open(TEMPLATES / "make_db.jinja", "r") as f: - template = Template(f.read()) + jinja_txt = f.read() - renderer = Render() + renderer = RenderDb(ioc_instance) - return template.render( - database_elements=renderer.render_database_elements(ioc_instance), - ) + templates = renderer.render_database() + + return Template(jinja_txt).render(templates=templates) def create_boot_script(ioc_instance: IOC) -> str: diff --git a/src/ibek/render.py b/src/ibek/render.py index 686be8e15..a0a6160e8 100644 --- a/src/ibek/render.py +++ b/src/ibek/render.py @@ -111,43 +111,6 @@ def render_post_ioc_init(self, instance: Entity) -> Optional[str]: post_init = instance.__definition__.post_init return self.render_script(instance, post_init) - def render_database(self, instance: Entity) -> Optional[str]: - """ - render the lines required to instantiate database by combining the - templates from the Entity's database list with the arguments from - an Entity - """ - templates = instance.__definition__.databases - # The entity may not instantiate any database templates - if not templates: - return None - jinja_txt = "" - - for template in templates: - db_file = template.file.strip("\n") - - macros = [] - for arg, value in template.args.items(): - if value is None: - if arg not in instance.__dict__: - raise ValueError( - f"database arg '{arg}' in database template " - f"'{template.file}' not found in context" - ) - macros.append(f"{arg}={{{{ {arg} }}}}") - else: - macros.append(f"{arg}={value}") - - db_arg_string = ", ".join(macros) - - jinja_txt += ( - f'msi -I${{EPICS_DB_INCLUDE_PATH}} -M"{db_arg_string}" "{db_file}"\n' - ) - - db_txt = render_with_utils(instance, jinja_txt) # type: ignore - - return db_txt + "\n" - def render_environment_variables(self, instance: Entity) -> Optional[str]: """ render the environment variable elements by combining the jinja template @@ -193,12 +156,6 @@ def render_post_ioc_init_elements(self, ioc: IOC) -> str: """ return self.render_elements(ioc, self.render_post_ioc_init) - def render_database_elements(self, ioc: IOC) -> str: - """ - Render all of the DBLoadRecords entries for a given IOC instance - """ - return self.render_elements(ioc, self.render_database) - def render_environment_variable_elements(self, ioc: IOC) -> str: """ Render all of the environment variable entries for a given IOC instance diff --git a/src/ibek/render_db.py b/src/ibek/render_db.py new file mode 100644 index 000000000..2c781ad20 --- /dev/null +++ b/src/ibek/render_db.py @@ -0,0 +1,107 @@ +""" +A class for rendering a substitution file from multiple instantiations of +support module definitions +""" + +from dataclasses import dataclass +from typing import Any, Dict, List + +from ibek.globals import render_with_utils +from ibek.ioc import IOC, Entity + + +class RenderDb: + @dataclass + class RenderDbTemplate: + filename: str + rows: List[List[str]] + columns: List[int] + + def __init__(self, ioc_instance: IOC) -> None: + self.ioc_instance = ioc_instance + # a mapping from template file name to details of instances of that template + self.render_templates: Dict[str, RenderDb.RenderDbTemplate] = {} + + def add_row(self, filename: str, args: Dict[str, Any], entity: Entity) -> None: + """ + Accumulate rows of arguments for each template file, + Adding a new template file if it does not already exist. + Convert all arguments to strings. + """ + if filename not in self.render_templates: + # for new filenames create a new RenderDbTemplate entry + headings = [str(i) for i in list(args.keys())] + self.render_templates[filename] = RenderDb.RenderDbTemplate( + filename=filename, + rows=[headings], # first row is the headings + columns=[0] * len(args), + ) + + # add a new row of argument values + row = [str(i) for i in args.values()] + + # render any Jinja fields in the arguments + for i, line in enumerate(row): + row[i] = render_with_utils(dict(entity), row[i]) + + # save the new row + self.render_templates[filename].rows.append(row) + + def parse_instances(self) -> None: + """ + Gather the database template instantiations from all entities + while validating the arguments + """ + for entity in self.ioc_instance.entities: + templates = entity.__definition__.databases + + # Not all entities instantiate database templates + if templates is None or not entity.entity_enabled: + continue + + for template in templates: + template.file = template.file.strip("\n") + + for arg, value in template.args.items(): + if value is None: + if arg not in entity.__dict__: + raise ValueError( + f"database arg '{arg}' in database template " + f"'{template.file}' not found in context" + ) + self.add_row(template.file, template.args, entity) + + def align_columns(self) -> None: + """ + Make sure columns will line up for each template file, also + provide escaping for spaces and quotes + """ + + # first calculate the column width for each template + # including escaping spaces and quotes + for template in self.render_templates.values(): + for row in template.rows: + for i, arg in enumerate(row): + row[i] = row[i].replace('"', r"\"") + row[i] = f'"{row[i]}"' if " " in arg else row[i] + template.columns[i] = max(template.columns[i], len(arg) + 1) + + # now pad each column to the maximum width + for template in self.render_templates.values(): + for row in template.rows: + for i, arg in enumerate(row): + row[i] = arg.ljust(template.columns[i]) + + def render_database(self) -> Dict[str, List[str]]: + """ + Render a database substitution file + """ + self.parse_instances() + self.align_columns() + + results = {} + + for template in self.render_templates.values(): + results[template.filename] = ["".join(row) + "\n" for row in template.rows] + + return results diff --git a/src/ibek/templates/db.subst.jinja b/src/ibek/templates/db.subst.jinja new file mode 100644 index 000000000..26e5727f8 --- /dev/null +++ b/src/ibek/templates/db.subst.jinja @@ -0,0 +1,8 @@ +################################################################################ +# DB substitution file generated by http://github.com/epics-containers/ibek +################################################################################ +{% for template_file, template_instances in template_list.items() %} +file "{{ template_file }}" { +pattern { {{ " ".join(template_instances.headings) }} } +} +{% endfor %} \ No newline at end of file diff --git a/src/ibek/templates/make_db.jinja b/src/ibek/templates/make_db.jinja deleted file mode 100644 index bfec9cf3b..000000000 --- a/src/ibek/templates/make_db.jinja +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -{{ database_elements -}} \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index efa342148..edf8db620 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -29,6 +29,11 @@ def get_support(yaml_file: str) -> Support: return support +@fixture +def templates(): + return Path(__file__).parent.parent / "src" / "ibek" / "templates" + + @fixture def samples(): return Path(__file__).parent / "samples" diff --git a/tests/test_cli.py b/tests/test_cli.py index 45bb01ee5..a45b765a2 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,3 +1,7 @@ +""" +System tests that run the CLI commands and compare the output to expected +results. +""" import json import subprocess import sys @@ -96,6 +100,9 @@ def test_build_startup_multiple(tmp_path: Path, samples: Path): """ build an ioc startup script from an IOC instance entity file and multiple support module definition files + + Also verifies database subst file generation for multiple + entity instantiations. """ clear_entity_model_ids() ioc_yaml = samples / "yaml" / "all.ibek.ioc.yaml" diff --git a/tests/test_render.py b/tests/test_render.py index 18e94e0ba..676800029 100644 --- a/tests/test_render.py +++ b/tests/test_render.py @@ -4,7 +4,9 @@ """ from typing import Literal +from ibek.ioc import IOC, clear_entity_model_ids from ibek.render import Render +from ibek.render_db import RenderDb def find_entity_class(entity_classes, entity_type): @@ -45,21 +47,35 @@ def test_obj_ref_script(objects_classes): ) -def test_database_render(objects_classes): +def test_database_render(objects_classes, templates): ref_cls = find_entity_class(objects_classes, "object_module.RefObject") consumer = find_entity_class(objects_classes, "object_module.ConsumerTwo") ref_cls(name="test_ref_object") my_consumer = consumer(name="test_consumer", PORT="test_ref_object") - render = Render() - db_txt = render.render_database(my_consumer) + # make a dummy IOC with two entities as database render works against + # a whole IOC rather than a single entity at a time. + clear_entity_model_ids() - assert ( - db_txt == 'msi -I${EPICS_DB_INCLUDE_PATH} -M"name=test_consumer, ' - 'ip=127.0.0.1, value=test_ref_object.127.0.0.1" "test.db"\n' + ioc = IOC( + ioc_name="test_ioc", + description="for testing", + generic_ioc_image="test_ioc_img", + entities=[], ) + ioc.entities.append(my_consumer) + render_db = RenderDb(ioc) + templates = render_db.render_database() + + assert templates == { + "test.db": [ + "name ip value \n", + "None 127.0.0.1 test_ref_object.127.0.0.1 \n", + ] + } + def test_environment_variables(objects_classes): ref_cls = find_entity_class(objects_classes, "object_module.RefObject") From 40793b76ddf30b97e4970b7a0e65b1e2f2823531 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Tue, 12 Sep 2023 14:25:52 +0100 Subject: [PATCH 07/11] added subst file generation --- src/ibek/__main__.py | 2 +- src/ibek/gen_scripts.py | 2 +- src/ibek/render_db.py | 16 ++++++++++------ src/ibek/templates/db.subst.jinja | 6 ++++-- tests/generate_samples.sh | 4 ++-- tests/samples/outputs/all.db.subst | 3 +++ tests/samples/outputs/all.make_db.sh | 1 - tests/samples/outputs/objects.db.subst | 11 +++++++++++ tests/samples/outputs/objects.make_db.sh | 3 --- tests/test_cli.py | 4 ++-- tests/test_render.py | 4 ++-- 11 files changed, 36 insertions(+), 20 deletions(-) create mode 100644 tests/samples/outputs/all.db.subst delete mode 100644 tests/samples/outputs/all.make_db.sh create mode 100644 tests/samples/outputs/objects.db.subst delete mode 100644 tests/samples/outputs/objects.make_db.sh diff --git a/src/ibek/__main__.py b/src/ibek/__main__.py index 49f7a172b..5ef94bd8f 100644 --- a/src/ibek/__main__.py +++ b/src/ibek/__main__.py @@ -74,7 +74,7 @@ def build_startup( help="Path to output startup script", ), db_out: Path = typer.Option( - default="config/make_db.sh", + default="config/db.subst", help="Path to output database expansion shell script", ), ): diff --git a/src/ibek/gen_scripts.py b/src/ibek/gen_scripts.py index 3577c19b2..00c0b6a19 100644 --- a/src/ibek/gen_scripts.py +++ b/src/ibek/gen_scripts.py @@ -66,7 +66,7 @@ def create_db_script(ioc_instance: IOC) -> str: """ Create make_db.sh script for expanding the database templates """ - with open(TEMPLATES / "make_db.jinja", "r") as f: + with open(TEMPLATES / "db.subst.jinja", "r") as f: jinja_txt = f.read() renderer = RenderDb(ioc_instance) diff --git a/src/ibek/render_db.py b/src/ibek/render_db.py index 2c781ad20..3575f83d2 100644 --- a/src/ibek/render_db.py +++ b/src/ibek/render_db.py @@ -52,7 +52,8 @@ def parse_instances(self) -> None: Gather the database template instantiations from all entities while validating the arguments """ - for entity in self.ioc_instance.entities: + for entity_root in self.ioc_instance.entities: + entity = entity_root.root templates = entity.__definition__.databases # Not all entities instantiate database templates @@ -80,11 +81,14 @@ def align_columns(self) -> None: # first calculate the column width for each template # including escaping spaces and quotes for template in self.render_templates.values(): - for row in template.rows: + for n, row in enumerate(template.rows): for i, arg in enumerate(row): - row[i] = row[i].replace('"', r"\"") - row[i] = f'"{row[i]}"' if " " in arg else row[i] - template.columns[i] = max(template.columns[i], len(arg) + 1) + row[i] = arg.replace(",", r"\,") + if n > 0: + row[i] = f'"{row[i]}"' + if i < len(template.columns) - 1: + row[i] += "," + template.columns[i] = max(template.columns[i], len(row[i]) + 1) # now pad each column to the maximum width for template in self.render_templates.values(): @@ -102,6 +106,6 @@ def render_database(self) -> Dict[str, List[str]]: results = {} for template in self.render_templates.values(): - results[template.filename] = ["".join(row) + "\n" for row in template.rows] + results[template.filename] = ["".join(row) for row in template.rows] return results diff --git a/src/ibek/templates/db.subst.jinja b/src/ibek/templates/db.subst.jinja index 26e5727f8..bdc40cb82 100644 --- a/src/ibek/templates/db.subst.jinja +++ b/src/ibek/templates/db.subst.jinja @@ -1,8 +1,10 @@ ################################################################################ # DB substitution file generated by http://github.com/epics-containers/ibek ################################################################################ -{% for template_file, template_instances in template_list.items() %} +{% for template_file, rows in templates.items() %} file "{{ template_file }}" { -pattern { {{ " ".join(template_instances.headings) }} } +pattern {% for row in rows %} +{{"\t"}}{ {{ row }}} {% endfor %} } + {% endfor %} \ No newline at end of file diff --git a/tests/generate_samples.sh b/tests/generate_samples.sh index 8279f40d7..fa2454ff9 100755 --- a/tests/generate_samples.sh +++ b/tests/generate_samples.sh @@ -26,7 +26,7 @@ echo making an ioc schema using multiple support yaml files ibek ioc-schema yaml/objects.ibek.support.yaml yaml/all.ibek.support.yaml schemas/multiple.ibek.ioc.schema.json # echo making ioc based on objects support yaml -ibek build-startup yaml/objects.ibek.ioc.yaml yaml/objects.ibek.support.yaml --out outputs/objects.st.cmd --db-out outputs/objects.make_db.sh +ibek build-startup yaml/objects.ibek.ioc.yaml yaml/objects.ibek.support.yaml --out outputs/objects.st.cmd --db-out outputs/objects.db.subst # echo making ioc based on mutiple support yaml -ibek build-startup yaml/all.ibek.ioc.yaml yaml/objects.ibek.support.yaml yaml/all.ibek.support.yaml --out outputs/all.st.cmd --db-out outputs/all.make_db.sh +ibek build-startup yaml/all.ibek.ioc.yaml yaml/objects.ibek.support.yaml yaml/all.ibek.support.yaml --out outputs/all.st.cmd --db-out outputs/all.db.subst diff --git a/tests/samples/outputs/all.db.subst b/tests/samples/outputs/all.db.subst new file mode 100644 index 000000000..5e306ee97 --- /dev/null +++ b/tests/samples/outputs/all.db.subst @@ -0,0 +1,3 @@ +################################################################################ +# DB substitution file generated by http://github.com/epics-containers/ibek +################################################################################ diff --git a/tests/samples/outputs/all.make_db.sh b/tests/samples/outputs/all.make_db.sh deleted file mode 100644 index a9bf588e2..000000000 --- a/tests/samples/outputs/all.make_db.sh +++ /dev/null @@ -1 +0,0 @@ -#!/bin/bash diff --git a/tests/samples/outputs/objects.db.subst b/tests/samples/outputs/objects.db.subst new file mode 100644 index 000000000..e535c5508 --- /dev/null +++ b/tests/samples/outputs/objects.db.subst @@ -0,0 +1,11 @@ +################################################################################ +# DB substitution file generated by http://github.com/epics-containers/ibek +################################################################################ + +file "test.db" { +pattern + { name, ip, value} + { "None", "10.0.0.2", "AsynPort2.10.0.0.2"} + { "None", "10.0.0.2", "AsynPort2.10.0.0.2"} +} + diff --git a/tests/samples/outputs/objects.make_db.sh b/tests/samples/outputs/objects.make_db.sh deleted file mode 100644 index c94639a98..000000000 --- a/tests/samples/outputs/objects.make_db.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -msi -I${EPICS_DB_INCLUDE_PATH} -M"name=Consumer of another port, ip=10.0.0.2, value=AsynPort2.10.0.0.2" "test.db" -msi -I${EPICS_DB_INCLUDE_PATH} -M"name=Another Consumer of the 2nd port, ip=10.0.0.2, value=AsynPort2.10.0.0.2" "test.db" diff --git a/tests/test_cli.py b/tests/test_cli.py index a45b765a2..b536d50ae 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -91,7 +91,7 @@ def test_build_startup_single(tmp_path: Path, samples: Path): actual_boot = out_file.read_text() assert example_boot == actual_boot - example_db = (samples / "outputs" / "objects.make_db.sh").read_text() + example_db = (samples / "outputs" / "objects.db.subst").read_text() actual_db = out_db.read_text() assert example_db == actual_db @@ -126,6 +126,6 @@ def test_build_startup_multiple(tmp_path: Path, samples: Path): actual_boot = out_file.read_text() assert example_boot == actual_boot - example_db = (samples / "outputs" / "all.make_db.sh").read_text() + example_db = (samples / "outputs" / "all.db.subst").read_text() actual_db = out_db.read_text() assert example_db == actual_db diff --git a/tests/test_render.py b/tests/test_render.py index 676800029..d12254c3f 100644 --- a/tests/test_render.py +++ b/tests/test_render.py @@ -71,8 +71,8 @@ def test_database_render(objects_classes, templates): assert templates == { "test.db": [ - "name ip value \n", - "None 127.0.0.1 test_ref_object.127.0.0.1 \n", + "name, ip, value", + '"None", "127.0.0.1", "test_ref_object.127.0.0.1"', ] } From bb14248bd94eaf43e1e9002e08dfaf5ee11d9fc1 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Tue, 12 Sep 2023 15:02:02 +0100 Subject: [PATCH 08/11] improved tests for subst file generation --- src/ibek/render_db.py | 16 ++++++++++--- src/ibek/templates/db.subst.jinja | 2 +- tests/samples/outputs/all.db.subst | 23 +++++++++++++++++++ tests/samples/outputs/all.st.cmd | 3 +++ tests/samples/outputs/objects.db.subst | 6 ++--- .../schemas/multiple.ibek.ioc.schema.json | 4 ++-- tests/samples/yaml/all.ibek.ioc.yaml | 4 ++++ tests/samples/yaml/all.ibek.support.yaml | 17 +++++++++++++- tests/test_cli.py | 6 ++--- tests/test_render.py | 4 ++-- 10 files changed, 69 insertions(+), 16 deletions(-) diff --git a/src/ibek/render_db.py b/src/ibek/render_db.py index 3575f83d2..70c9e2bb1 100644 --- a/src/ibek/render_db.py +++ b/src/ibek/render_db.py @@ -37,8 +37,10 @@ def add_row(self, filename: str, args: Dict[str, Any], entity: Entity) -> None: columns=[0] * len(args), ) - # add a new row of argument values - row = [str(i) for i in args.values()] + # Add a new row of argument values. + # Note the case where no value was specified for the argument + # meaning that the name is to be rendered in Jinja + row = ["{{ %s }}" % k if v is None else v for k, v in args.items()] # render any Jinja fields in the arguments for i, line in enumerate(row): @@ -53,7 +55,15 @@ def parse_instances(self) -> None: while validating the arguments """ for entity_root in self.ioc_instance.entities: - entity = entity_root.root + # TODO this is a highly suspect way of coping with the difference + # between deserialized entities and the original entity class. + # The deserializer inserts an extra layer of nesting + # with "root". This issue comes up because test_database_render + # directly creates an IOC object and does not get the extra "root". + # + # see definition of EntityModel in ioc.py + entity = getattr(entity_root, "root", entity_root) + templates = entity.__definition__.databases # Not all entities instantiate database templates diff --git a/src/ibek/templates/db.subst.jinja b/src/ibek/templates/db.subst.jinja index bdc40cb82..686adee3e 100644 --- a/src/ibek/templates/db.subst.jinja +++ b/src/ibek/templates/db.subst.jinja @@ -4,7 +4,7 @@ {% for template_file, rows in templates.items() %} file "{{ template_file }}" { pattern {% for row in rows %} -{{"\t"}}{ {{ row }}} {% endfor %} +{{"\t"}}{ {{ row }} } {% endfor %} } {% endfor %} \ No newline at end of file diff --git a/tests/samples/outputs/all.db.subst b/tests/samples/outputs/all.db.subst index 5e306ee97..5710a2923 100644 --- a/tests/samples/outputs/all.db.subst +++ b/tests/samples/outputs/all.db.subst @@ -1,3 +1,26 @@ ################################################################################ # DB substitution file generated by http://github.com/epics-containers/ibek ################################################################################ + +file "another_test.db" { +pattern + { name, my_int_enum, clock_rate, db_calculated, calculated_one } + { "AllObject One", "2", "dummy", "HELLO AllObject One", "AllObject One.AllObject One String.1.1.0.True" } + { "AllObject Two", "1", "1", "HELLO AllObject Two", "AllObject Two.AllObject Two String.1.1.0.True" } +} + + +file "yet_another.db" { +pattern + { name, my_object, my_float, my_bool } + { "AllObject One", "Ref1", "1.0", "True" } + { "AllObject Two", "Ref1", "1.0", "True" } +} + + +file "test.db" { +pattern + { name, ip, value } + { "Consumer Two With DB", "127.0.0.1", "Ref1.127.0.0.1" } +} + diff --git a/tests/samples/outputs/all.st.cmd b/tests/samples/outputs/all.st.cmd index a7f056089..5f3d2d401 100644 --- a/tests/samples/outputs/all.st.cmd +++ b/tests/samples/outputs/all.st.cmd @@ -25,6 +25,9 @@ clock_rate = 1 my_mixed_enum_no_default = . +# ExampleTestFunction asynPortIP name port value +ExampleTestFunction 127.0.0.1 Consumer Two With DB Ref1 Ref1.127.0.0.1 + dbLoadRecords /tmp/ioc.db iocInit diff --git a/tests/samples/outputs/objects.db.subst b/tests/samples/outputs/objects.db.subst index e535c5508..f9e179719 100644 --- a/tests/samples/outputs/objects.db.subst +++ b/tests/samples/outputs/objects.db.subst @@ -4,8 +4,8 @@ file "test.db" { pattern - { name, ip, value} - { "None", "10.0.0.2", "AsynPort2.10.0.0.2"} - { "None", "10.0.0.2", "AsynPort2.10.0.0.2"} + { name, ip, value } + { "Consumer of another port", "10.0.0.2", "AsynPort2.10.0.0.2" } + { "Another Consumer of the 2nd port", "10.0.0.2", "AsynPort2.10.0.0.2" } } diff --git a/tests/samples/schemas/multiple.ibek.ioc.schema.json b/tests/samples/schemas/multiple.ibek.ioc.schema.json index edde6228f..e0da01654 100644 --- a/tests/samples/schemas/multiple.ibek.ioc.schema.json +++ b/tests/samples/schemas/multiple.ibek.ioc.schema.json @@ -109,10 +109,10 @@ "title": "My Bool", "type": "boolean" }, - "a value calculated from args": { + "calculated_one": { "default": "{{ name }}.{{ my_str }}.{{ my_int }}.{{ my_float }}.{{ my_bool }}", "description": "test jinja render of arg values", - "title": "A Value Calculated From Args", + "title": "Calculated One", "type": "string" } }, diff --git a/tests/samples/yaml/all.ibek.ioc.yaml b/tests/samples/yaml/all.ibek.ioc.yaml index ca2d02f0f..87c89d0c6 100644 --- a/tests/samples/yaml/all.ibek.ioc.yaml +++ b/tests/samples/yaml/all.ibek.ioc.yaml @@ -24,6 +24,10 @@ entities: clock_rate: 2Hz my_str: AllObject Two String + - type: object_module.ConsumerTwo + name: Consumer Two With DB + PORT: Ref1 + - type: object_module.ConsumerTwo name: Disabled Consumer PORT: Ref1 diff --git a/tests/samples/yaml/all.ibek.support.yaml b/tests/samples/yaml/all.ibek.support.yaml index df706b155..b8fd97021 100644 --- a/tests/samples/yaml/all.ibek.support.yaml +++ b/tests/samples/yaml/all.ibek.support.yaml @@ -71,7 +71,7 @@ defs: default: true values: - - name: a value calculated from args + - name: calculated_one value: "{{ name }}.{{ my_str }}.{{ my_int }}.{{ my_float }}.{{ my_bool }}" description: test jinja render of arg values @@ -96,3 +96,18 @@ defs: args: identifier: "{{ name }}" TestValue: test_value:{{ test_value }} + + databases: + - file: another_test.db + args: + name: + my_int_enum: + clock_rate: + db_calculated: "HELLO {{ name }}" + calculated_one: + - file: yet_another.db + args: + name: + my_object: + my_float: + my_bool: diff --git a/tests/test_cli.py b/tests/test_cli.py index b536d50ae..8c2d25715 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -24,8 +24,6 @@ def test_ibek_schema(tmp_path: Path, samples: Path): expected = json.loads( (samples / "schemas" / "ibek.support.schema.json").read_text() ) - # Don't care if version number didn't update to match if the rest is the same - # expected["title"] = mock.ANY actual = json.loads((schema_path).read_text()) assert expected == actual @@ -75,7 +73,7 @@ def test_build_startup_single(tmp_path: Path, samples: Path): ioc_yaml = samples / "yaml" / "objects.ibek.ioc.yaml" support_yaml = samples / "yaml" / "objects.ibek.support.yaml" out_file = tmp_path / "new_dir" / "st.cmd" - out_db = tmp_path / "new_dir" / "make_db.sh" + out_db = tmp_path / "new_dir" / "objects.db.subst" run_cli( "build-startup", @@ -109,7 +107,7 @@ def test_build_startup_multiple(tmp_path: Path, samples: Path): support_yaml = samples / "yaml" / "objects.ibek.support.yaml" support_yaml2 = samples / "yaml" / "all.ibek.support.yaml" out_file = tmp_path / "st.cmd" - out_db = tmp_path / "make_db.sh" + out_db = tmp_path / "all.db.subst" run_cli( "build-startup", diff --git a/tests/test_render.py b/tests/test_render.py index d12254c3f..e30462de4 100644 --- a/tests/test_render.py +++ b/tests/test_render.py @@ -71,8 +71,8 @@ def test_database_render(objects_classes, templates): assert templates == { "test.db": [ - "name, ip, value", - '"None", "127.0.0.1", "test_ref_object.127.0.0.1"', + "name, ip, value", + '"test_consumer", "127.0.0.1", "test_ref_object.127.0.0.1"', ] } From 34391eb933e6c2feab4ba44a310d5f2894a82e00 Mon Sep 17 00:00:00 2001 From: Tom Willemsen Date: Tue, 12 Sep 2023 14:49:29 +0100 Subject: [PATCH 09/11] Use ruff as a linter as a replacement for flake8/isort --- .gitignore | 3 +++ .pre-commit-config.yaml | 6 ++--- .vscode/extensions.json | 3 ++- .vscode/settings.json | 15 ++++++++---- docs/conf.py | 3 +-- docs/developer/how-to/lint.rst | 12 +++++----- docs/developer/reference/standards.rst | 3 +-- pyproject.toml | 33 +++++++++++++------------- 8 files changed, 44 insertions(+), 34 deletions(-) diff --git a/.gitignore b/.gitignore index 62dcd9a8d..a37be99b3 100644 --- a/.gitignore +++ b/.gitignore @@ -66,3 +66,6 @@ venv* # further build artifacts lockfiles/ + +# ruff cache +.ruff_cache/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index aa2a4cb2c..5bc9f001c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,9 +15,9 @@ repos: entry: black --check --diff types: [python] - - id: flake8 - name: Run flake8 + - id: ruff + name: Run ruff stages: [commit] language: system - entry: flake8 + entry: ruff types: [python] diff --git a/.vscode/extensions.json b/.vscode/extensions.json index cda78d45e..a1227b348 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -4,6 +4,7 @@ "ms-python.python", "tamasfe.even-better-toml", "redhat.vscode-yaml", - "ryanluker.vscode-coverage-gutters" + "ryanluker.vscode-coverage-gutters", + "charliermarsh.Ruff" ] } diff --git a/.vscode/settings.json b/.vscode/settings.json index b8cc9ad67..f5b8508fb 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,15 +1,22 @@ { "python.linting.pylintEnabled": false, - "python.linting.flake8Enabled": true, + "python.linting.flake8Enabled": false, "python.linting.mypyEnabled": true, "python.linting.enabled": true, - "python.testing.pytestArgs": ["--cov=python3_pip_skeleton", "--cov-report", "xml:cov.xml"], + "python.testing.pytestArgs": [ + "--cov=python3_pip_skeleton", + "--cov-report", + "xml:cov.xml" + ], "python.testing.unittestEnabled": false, "python.testing.pytestEnabled": true, "python.formatting.provider": "black", "python.languageServer": "Pylance", "editor.formatOnSave": true, - "editor.codeActionsOnSave": { - "source.organizeImports": true + "[python]": { + "editor.codeActionsOnSave": { + "source.fixAll.ruff": false, + "source.organizeImports.ruff": true + } } } diff --git a/docs/conf.py b/docs/conf.py index 7022f617c..1c4d4e644 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -107,8 +107,7 @@ rst_epilog = """ .. _Diamond Light Source: http://www.diamond.ac.uk .. _black: https://github.com/psf/black -.. _flake8: https://flake8.pycqa.org/en/latest/ -.. _isort: https://github.com/PyCQA/isort +.. _ruff: https://beta.ruff.rs/docs/ .. _mypy: http://mypy-lang.org/ .. _pre-commit: https://pre-commit.com/ """ diff --git a/docs/developer/how-to/lint.rst b/docs/developer/how-to/lint.rst index ffb6ee5af..2df258d8f 100644 --- a/docs/developer/how-to/lint.rst +++ b/docs/developer/how-to/lint.rst @@ -1,7 +1,7 @@ Run linting using pre-commit ============================ -Code linting is handled by black_, flake8_ and isort_ run under pre-commit_. +Code linting is handled by black_ and ruff_ run under pre-commit_. Running pre-commit ------------------ @@ -26,14 +26,14 @@ repository:: $ black . -Likewise with isort:: +Likewise with ruff:: - $ isort . + $ ruff --fix . -If you get any flake8 issues you will have to fix those manually. +Ruff may not be able to automatically fix all issues; in this case, you will have to fix those manually. VSCode support -------------- -The ``.vscode/settings.json`` will run black and isort formatters as well as -flake8 checking on save. Issues will be highlighted in the editor window. +The ``.vscode/settings.json`` will run black formatting as well as +ruff checking on save. Issues will be highlighted in the editor window. diff --git a/docs/developer/reference/standards.rst b/docs/developer/reference/standards.rst index 06c4af539..5a1fd4782 100644 --- a/docs/developer/reference/standards.rst +++ b/docs/developer/reference/standards.rst @@ -10,8 +10,7 @@ Code Standards The code in this repository conforms to standards set by the following tools: - black_ for code formatting -- flake8_ for style checks -- isort_ for import ordering +- ruff_ for style checks - mypy_ for static type checking .. seealso:: diff --git a/pyproject.toml b/pyproject.toml index c84d39364..3adc36b57 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,13 +26,12 @@ requires-python = ">=3.7" dev = [ "black", "mypy", - "flake8-isort", - "Flake8-pyproject", "pipdeptree", "pre-commit", "pydata-sphinx-theme>=0.12", "pytest", "pytest-cov", + "ruff", "sphinx-autobuild", "sphinx-copybutton", "sphinx-design", @@ -57,20 +56,6 @@ write_to = "src/python3_pip_skeleton/_version.py" [tool.mypy] ignore_missing_imports = true # Ignore missing stubs in imported modules -[tool.isort] -float_to_top = true -profile = "black" - -[tool.flake8] -extend-ignore = [ - "E203", # See https://github.com/PyCQA/pycodestyle/issues/373 - "F811", # support typing.overload decorator - "F722", # allow Annotated[typ, some_func("some string")] -] -max-line-length = 88 # Respect black's line length (default 88), -exclude = [".tox", "venv"] - - [tool.pytest.ini_options] # Run pytest with all our checkers, and don't spam us with massive tracebacks on error addopts = """ @@ -111,3 +96,19 @@ commands = pre-commit: pre-commit run --all-files {posargs} docs: sphinx-{posargs:build -EW --keep-going} -T docs build/html """ + + +[tool.ruff] +src = ["src", "tests"] +ignore = [ + "C408", # Unnecessary collection call - e.g. list(...) instead of [...] + "E501", # Line too long, should be fixed by black. +] +line-length = 88 +select = [ + "C4", # flake8-comprehensions - https://beta.ruff.rs/docs/rules/#flake8-comprehensions-c4 + "E", # pycodestyle errors - https://beta.ruff.rs/docs/rules/#error-e + "F", # pyflakes rules - https://beta.ruff.rs/docs/rules/#pyflakes-f + "W", # pycodestyle warnings - https://beta.ruff.rs/docs/rules/#warning-w + "I001", # isort +] From 360e8dda86bfced9dc990dd49ff3f089a09071ce Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Wed, 13 Sep 2023 08:19:14 +0100 Subject: [PATCH 10/11] Rename python3-pip-skeleton -> ibek --- .github/CONTRIBUTING.rst | 8 ++--- .github/pages/make_switcher.py | 10 +++--- Dockerfile | 2 +- LICENSE | 2 +- README.rst | 34 +++++++++---------- docs/conf.py | 12 +++---- .../0002-switched-to-pip-skeleton.rst | 6 ++-- docs/developer/how-to/make-release.rst | 4 +-- docs/developer/tutorials/dev-install.rst | 8 ++--- docs/index.rst | 4 +-- docs/user/how-to/run-container.rst | 8 ++--- docs/user/reference/api.rst | 8 ++--- docs/user/tutorials/installation.rst | 6 ++-- pyproject.toml | 16 ++++----- .../__init__.py | 2 +- .../__main__.py | 10 +++--- tests/test_boilerplate_removed.py | 2 +- tests/test_cli.py | 4 +-- 18 files changed, 73 insertions(+), 73 deletions(-) rename src/{python3_pip_skeleton => ibek}/__init__.py (80%) rename src/{python3_pip_skeleton => ibek}/__main__.py (55%) diff --git a/.github/CONTRIBUTING.rst b/.github/CONTRIBUTING.rst index 19ab494ff..fa16425fe 100644 --- a/.github/CONTRIBUTING.rst +++ b/.github/CONTRIBUTING.rst @@ -7,7 +7,7 @@ filing a new one. If you have a great idea but it involves big changes, please file a ticket before making a pull request! We want to make sure you don't spend your time coding something that might not fit the scope of the project. -.. _GitHub: https://github.com/DiamondLightSource/python3-pip-skeleton/issues +.. _GitHub: https://github.com/epics-containers/ibek/issues Issue or Discussion? -------------------- @@ -16,13 +16,13 @@ Github also offers discussions_ as a place to ask questions and share ideas. If your issue is open ended and it is not obvious when it can be "closed", please raise it as a discussion instead. -.. _discussions: https://github.com/DiamondLightSource/python3-pip-skeleton/discussions +.. _discussions: https://github.com/epics-containers/ibek/discussions Code coverage ------------- While 100% code coverage does not make a library bug-free, it significantly -reduces the number of easily caught bugs! Please make sure coverage remains the +reduces the number of easily caught bugs! Please make sure coverage rechange_linter_to_ruffs the same or is improved by a pull request! Developer guide @@ -32,4 +32,4 @@ The `Developer Guide`_ contains information on setting up a development environment, running the tests and what standards the code and documentation should follow. -.. _Developer Guide: https://diamondlightsource.github.io/python3-pip-skeleton/main/developer/how-to/contribute.html +.. _Developer Guide: https://diamondlightsource.github.io/ibek/change_linter_to_ruff/developer/how-to/contribute.html diff --git a/.github/pages/make_switcher.py b/.github/pages/make_switcher.py index d70367aec..6728a5bf7 100755 --- a/.github/pages/make_switcher.py +++ b/.github/pages/make_switcher.py @@ -43,9 +43,9 @@ def get_versions(ref: str, add: Optional[str], remove: Optional[str]) -> List[st # Get a sorted list of tags tags = get_sorted_tags_list() - # Make the sorted versions list from main branches and tags + # Make the sorted versions list from change_linter_to_ruff branches and tags versions: List[str] = [] - for version in ["master", "main"] + tags: + for version in ["master", "change_linter_to_ruff"] + tags: if version in builds: versions.append(version) builds.remove(version) @@ -67,7 +67,7 @@ def write_json(path: Path, repository: str, versions: str): path.write_text(text, encoding="utf-8") -def main(args=None): +def change_linter_to_ruff(args=None): parser = ArgumentParser( description="Make a versions.txt file from gh-pages directories" ) @@ -95,5 +95,5 @@ def main(args=None): write_json(args.output, args.repository, versions) -if __name__ == "__main__": - main() +if __name__ == "__change_linter_to_ruff__": + change_linter_to_ruff() diff --git a/Dockerfile b/Dockerfile index 31d05606f..31f40ecfb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,5 +33,5 @@ COPY --from=build /venv/ /venv/ ENV PATH=/venv/bin:$PATH # change this entrypoint if it is not the same as the repo -ENTRYPOINT ["python3-pip-skeleton"] +ENTRYPOINT ["ibek"] CMD ["--version"] diff --git a/LICENSE b/LICENSE index 8dada3eda..adb931e88 100644 --- a/LICENSE +++ b/LICENSE @@ -41,7 +41,7 @@ form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain + of this License, Derivative Works shall not include works that rechange_linter_to_ruff separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. diff --git a/README.rst b/README.rst index a014631eb..8f6680cbd 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -python3-pip-skeleton +ibek =========================== |code_ci| |docs_ci| |coverage| |pypi_version| |license| @@ -13,10 +13,10 @@ This is where you should write a short paragraph that describes what your module how it does it, and why people should use it. ============== ============================================================== -PyPI ``pip install python3-pip-skeleton`` -Source code https://github.com/DiamondLightSource/python3-pip-skeleton -Documentation https://DiamondLightSource.github.io/python3-pip-skeleton -Releases https://github.com/DiamondLightSource/python3-pip-skeleton/releases +PyPI ``pip install ibek`` +Source code https://github.com/epics-containers/ibek +Documentation https://epics-containers.github.io/ibek +Releases https://github.com/epics-containers/ibek/releases ============== ============================================================== This is where you should put some images or code snippets that illustrate @@ -25,28 +25,28 @@ introductory code here: .. code-block:: python - from python3_pip_skeleton import __version__ + from ibek import __version__ - print(f"Hello python3_pip_skeleton {__version__}") + print(f"Hello ibek {__version__}") Or if it is a commandline tool then you might put some example commands here:: - $ python -m python3_pip_skeleton --version + $ python -m ibek --version -.. |code_ci| image:: https://github.com/DiamondLightSource/python3-pip-skeleton/actions/workflows/code.yml/badge.svg?branch=main - :target: https://github.com/DiamondLightSource/python3-pip-skeleton/actions/workflows/code.yml +.. |code_ci| image:: https://github.com/epics-containers/ibek/actions/workflows/code.yml/badge.svg?branch=change_linter_to_ruff + :target: https://github.com/epics-containers/ibek/actions/workflows/code.yml :alt: Code CI -.. |docs_ci| image:: https://github.com/DiamondLightSource/python3-pip-skeleton/actions/workflows/docs.yml/badge.svg?branch=main - :target: https://github.com/DiamondLightSource/python3-pip-skeleton/actions/workflows/docs.yml +.. |docs_ci| image:: https://github.com/epics-containers/ibek/actions/workflows/docs.yml/badge.svg?branch=change_linter_to_ruff + :target: https://github.com/epics-containers/ibek/actions/workflows/docs.yml :alt: Docs CI -.. |coverage| image:: https://codecov.io/gh/DiamondLightSource/python3-pip-skeleton/branch/main/graph/badge.svg - :target: https://codecov.io/gh/DiamondLightSource/python3-pip-skeleton +.. |coverage| image:: https://codecov.io/gh/epics-containers/ibek/branch/change_linter_to_ruff/graph/badge.svg + :target: https://codecov.io/gh/epics-containers/ibek :alt: Test Coverage -.. |pypi_version| image:: https://img.shields.io/pypi/v/python3-pip-skeleton.svg - :target: https://pypi.org/project/python3-pip-skeleton +.. |pypi_version| image:: https://img.shields.io/pypi/v/ibek.svg + :target: https://pypi.org/project/ibek :alt: Latest PyPI version .. |license| image:: https://img.shields.io/badge/License-Apache%202.0-blue.svg @@ -57,4 +57,4 @@ Or if it is a commandline tool then you might put some example commands here:: Anything below this line is used when viewing README.rst and will be replaced when included in index.rst -See https://DiamondLightSource.github.io/python3-pip-skeleton for more detailed documentation. +See https://epics-containers.github.io/ibek for more detailed documentation. diff --git a/docs/conf.py b/docs/conf.py index 1c4d4e644..d2f5811b8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -10,15 +10,15 @@ import requests -import python3_pip_skeleton +import ibek # -- General configuration ------------------------------------------------ # General information about the project. -project = "python3-pip-skeleton" +project = "ibek" # The full version, including alpha/beta/rc tags. -release = python3_pip_skeleton.__version__ +release = ibek.__version__ # The short X.Y version. if "+" in release: @@ -52,7 +52,7 @@ # A list of (type, target) tuples (by default empty) that should be ignored when # generating warnings in "nitpicky mode". Note that type should include the -# domain name if present. Example entries would be ('py:func', 'int') or +# dochange_linter_to_ruff name if present. Example entries would be ('py:func', 'int') or # ('envvar', 'LD_LIBRARY_PATH'). nitpick_ignore = [ ("py:class", "NoneType"), @@ -66,7 +66,7 @@ ] # Both the class’ and the __init__ method’s docstring are concatenated and -# inserted into the main body of the autoclass directive +# inserted into the change_linter_to_ruff body of the autoclass directive autoclass_content = "both" # Order the members by the order they appear in the source code @@ -127,7 +127,7 @@ # html_theme = "pydata_sphinx_theme" github_repo = project -github_user = "DiamondLightSource" +github_user = "epics-containers" switcher_json = f"https://{github_user}.github.io/{github_repo}/switcher.json" switcher_exists = requests.get(switcher_json).ok if not switcher_exists: diff --git a/docs/developer/explanations/decisions/0002-switched-to-pip-skeleton.rst b/docs/developer/explanations/decisions/0002-switched-to-pip-skeleton.rst index 41d90fd4a..823c68369 100644 --- a/docs/developer/explanations/decisions/0002-switched-to-pip-skeleton.rst +++ b/docs/developer/explanations/decisions/0002-switched-to-pip-skeleton.rst @@ -1,4 +1,4 @@ -2. Adopt python3-pip-skeleton for project structure +2. Adopt ibek for project structure =================================================== Date: 2022-02-18 @@ -11,7 +11,7 @@ Accepted Context ------- -We should use the following `pip-skeleton `_. +We should use the following `pip-skeleton `_. The skeleton will ensure consistency in developer environments and package management. @@ -23,7 +23,7 @@ We have switched to using the skeleton. Consequences ------------ -This module will use a fixed set of tools as developed in python3-pip-skeleton +This module will use a fixed set of tools as developed in ibek and can pull from this skeleton to update the packaging to the latest techniques. As such, the developer environment may have changed, the following could be diff --git a/docs/developer/how-to/make-release.rst b/docs/developer/how-to/make-release.rst index d8f92f854..1449a6143 100644 --- a/docs/developer/how-to/make-release.rst +++ b/docs/developer/how-to/make-release.rst @@ -10,7 +10,7 @@ To make a new release, please follow this checklist: - Click ``Generate release notes``, review and edit these notes - Choose a title and click ``Publish Release`` -Note that tagging and pushing to the main branch has the same effect except that +Note that tagging and pushing to the change_linter_to_ruff branch has the same effect except that you will not get the option to edit the release notes. -.. _release: https://github.com/DiamondLightSource/python3-pip-skeleton/releases +.. _release: https://github.com/epics-containers/ibek/releases diff --git a/docs/developer/tutorials/dev-install.rst b/docs/developer/tutorials/dev-install.rst index 183b1893a..203297041 100644 --- a/docs/developer/tutorials/dev-install.rst +++ b/docs/developer/tutorials/dev-install.rst @@ -10,7 +10,7 @@ Clone the repository First clone the repository locally using `Git `_:: - $ git clone git://github.com/DiamondLightSource/python3-pip-skeleton.git + $ git clone git://github.com/epics-containers/ibek.git Install dependencies -------------------- @@ -25,7 +25,7 @@ requires python 3.8 or later) or to run in a container under `VSCode .. code:: - $ cd python3-pip-skeleton + $ cd ibek $ python3 -m venv venv $ source venv/bin/activate $ pip install -e '.[dev]' @@ -34,7 +34,7 @@ requires python 3.8 or later) or to run in a container under `VSCode .. code:: - $ code python3-pip-skeleton + $ code ibek # Click on 'Reopen in Container' when prompted # Open a new terminal @@ -65,4 +65,4 @@ This will run in parallel the following checks: - `../how-to/lint` -.. _epics-containers: https://epics-containers.github.io/main/user/tutorials/devcontainer.html +.. _epics-containers: https://epics-containers.github.io/change_linter_to_ruff/user/tutorials/devcontainer.html diff --git a/docs/index.rst b/docs/index.rst index df33c8e67..f23dd4fa1 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -14,13 +14,13 @@ The documentation is split into 2 sections: :link: user/index :link-type: doc - The User Guide contains documentation on how to install and use python3-pip-skeleton. + The User Guide contains documentation on how to install and use ibek. .. grid-item-card:: :material-regular:`code;4em` :link: developer/index :link-type: doc - The Developer Guide contains documentation on how to develop and contribute changes back to python3-pip-skeleton. + The Developer Guide contains documentation on how to develop and contribute changes back to ibek. .. toctree:: :hidden: diff --git a/docs/user/how-to/run-container.rst b/docs/user/how-to/run-container.rst index 84f857af7..8814ce033 100644 --- a/docs/user/how-to/run-container.rst +++ b/docs/user/how-to/run-container.rst @@ -1,15 +1,15 @@ Run in a container ================== -Pre-built containers with python3-pip-skeleton and its dependencies already +Pre-built containers with ibek and its dependencies already installed are available on `Github Container Registry -`_. +`_. Starting the container ---------------------- To pull the container from github container registry and run:: - $ docker run ghcr.io/DiamondLightSource/python3-pip-skeleton:main --version + $ docker run ghcr.io/epics-containers/ibek:change_linter_to_ruff --version -To get a released version, use a numbered release instead of ``main``. +To get a released version, use a numbered release instead of ``change_linter_to_ruff``. diff --git a/docs/user/reference/api.rst b/docs/user/reference/api.rst index 8544e1728..1ae13111d 100644 --- a/docs/user/reference/api.rst +++ b/docs/user/reference/api.rst @@ -1,14 +1,14 @@ API === -.. automodule:: python3_pip_skeleton +.. automodule:: ibek - ``python3_pip_skeleton`` + ``ibek`` ----------------------------------- -This is the internal API reference for python3_pip_skeleton +This is the internal API reference for ibek -.. data:: python3_pip_skeleton.__version__ +.. data:: ibek.__version__ :type: str Version number as calculated by https://github.com/pypa/setuptools_scm diff --git a/docs/user/tutorials/installation.rst b/docs/user/tutorials/installation.rst index e90d3efb9..2f209fcc7 100644 --- a/docs/user/tutorials/installation.rst +++ b/docs/user/tutorials/installation.rst @@ -25,14 +25,14 @@ Installing the library You can now use ``pip`` to install the library and its dependencies:: - $ python3 -m pip install python3-pip-skeleton + $ python3 -m pip install ibek If you require a feature that is not currently released you can also install from github:: - $ python3 -m pip install git+https://github.com/DiamondLightSource/python3-pip-skeleton.git + $ python3 -m pip install git+https://github.com/epics-containers/ibek.git The library should now be installed and the commandline interface on your path. You can check the version that has been installed by typing:: - $ python3-pip-skeleton --version + $ ibek --version diff --git a/pyproject.toml b/pyproject.toml index 3adc36b57..cda0fa0d0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=64", "setuptools_scm[toml]>=6.2", "wheel"] build-backend = "setuptools.build_meta" [project] -name = "python3-pip-skeleton" +name = "ibek" classifiers = [ "Development Status :: 3 - Alpha", "License :: OSI Approved :: Apache Software License", @@ -40,18 +40,18 @@ dev = [ ] [project.scripts] -python3-pip-skeleton = "python3_pip_skeleton.__main__:main" +ibek = "ibek.__change_linter_to_ruff__:change_linter_to_ruff" [project.urls] -GitHub = "https://github.com/DiamondLightSource/python3-pip-skeleton" +GitHub = "https://github.com/epics-containers/ibek" [[project.authors]] # Further authors may be added by duplicating this section -email = "email@address.com" -name = "Firstname Lastname" +email = "tom.cobb@diamond.ac.uk" +name = "Tom Cobb" [tool.setuptools_scm] -write_to = "src/python3_pip_skeleton/_version.py" +write_to = "src/ibek/_version.py" [tool.mypy] ignore_missing_imports = true # Ignore missing stubs in imported modules @@ -67,7 +67,7 @@ filterwarnings = "error" testpaths = "docs src tests" [tool.coverage.run] -data_file = "/tmp/python3_pip_skeleton.coverage" +data_file = "/tmp/ibek.coverage" [tool.coverage.paths] # Tests are run from installed location, map back to the src directory @@ -91,7 +91,7 @@ allowlist_externals = sphinx-build sphinx-autobuild commands = - pytest: pytest --cov=python3_pip_skeleton --cov-report term --cov-report xml:cov.xml {posargs} + pytest: pytest --cov=ibek --cov-report term --cov-report xml:cov.xml {posargs} mypy: mypy src tests {posargs} pre-commit: pre-commit run --all-files {posargs} docs: sphinx-{posargs:build -EW --keep-going} -T docs build/html diff --git a/src/python3_pip_skeleton/__init__.py b/src/ibek/__init__.py similarity index 80% rename from src/python3_pip_skeleton/__init__.py rename to src/ibek/__init__.py index 82cce3c2d..6d2899555 100644 --- a/src/python3_pip_skeleton/__init__.py +++ b/src/ibek/__init__.py @@ -5,7 +5,7 @@ else: from importlib.metadata import version # noqa -__version__ = version("python3-pip-skeleton") +__version__ = version("ibek") del version __all__ = ["__version__"] diff --git a/src/python3_pip_skeleton/__main__.py b/src/ibek/__main__.py similarity index 55% rename from src/python3_pip_skeleton/__main__.py rename to src/ibek/__main__.py index e348a31e4..f02dfb4ed 100644 --- a/src/python3_pip_skeleton/__main__.py +++ b/src/ibek/__main__.py @@ -2,15 +2,15 @@ from . import __version__ -__all__ = ["main"] +__all__ = ["change_linter_to_ruff"] -def main(args=None): +def change_linter_to_ruff(args=None): parser = ArgumentParser() parser.add_argument("-v", "--version", action="version", version=__version__) args = parser.parse_args(args) -# test with: python -m python3_pip_skeleton -if __name__ == "__main__": - main() +# test with: python -m ibek +if __name__ == "__change_linter_to_ruff__": + change_linter_to_ruff() diff --git a/tests/test_boilerplate_removed.py b/tests/test_boilerplate_removed.py index a0675d9c8..0f05f08aa 100644 --- a/tests/test_boilerplate_removed.py +++ b/tests/test_boilerplate_removed.py @@ -31,7 +31,7 @@ def assert_not_contains_text(path: str, text: str, explanation: str): # pyproject.toml def test_module_summary(): - summary = metadata("python3-pip-skeleton")["summary"] + summary = metadata("ibek")["summary"] skeleton_check( "One line description of your module" in summary, "Please change project.description in ./pyproject.toml " diff --git a/tests/test_cli.py b/tests/test_cli.py index 2ff648c03..beab7e4e3 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,9 +1,9 @@ import subprocess import sys -from python3_pip_skeleton import __version__ +from ibek import __version__ def test_cli_version(): - cmd = [sys.executable, "-m", "python3_pip_skeleton", "--version"] + cmd = [sys.executable, "-m", "ibek", "--version"] assert subprocess.check_output(cmd).decode().strip() == __version__ From 0e575e20cf8ab16c7a32ffcd36e69c15e52f8540 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Wed, 13 Sep 2023 09:37:59 +0100 Subject: [PATCH 11/11] fixes for update: docs and ruff --fix --- docs/conf.py | 3 ++- pyproject.toml | 1 + src/ibek/ioc.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 8164ad8a2..a4ceae7ed 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -63,11 +63,12 @@ ("py:class", "'object'"), ("py:class", "'id'"), ("py:class", "typing_extensions.Literal"), + ("py:class", "type"), ] # Both the class’ and the __init__ method’s docstring are concatenated and # inserted into the change_linter_to_ruff body of the autoclass directive -autoclass_content = "both" +autoclass_content = "class" # Order the members by the order they appear in the source code autodoc_member_order = "bysource" diff --git a/pyproject.toml b/pyproject.toml index 81bdd5d0f..9c07ea861 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ dev = [ "pytest", "pytest-cov", "ruff", + "Sphinx==6.2.1", "sphinx-autobuild", "sphinx-copybutton", "sphinx-design", diff --git a/src/ibek/ioc.py b/src/ibek/ioc.py index 0131408b4..f6e5dc07c 100644 --- a/src/ibek/ioc.py +++ b/src/ibek/ioc.py @@ -45,7 +45,7 @@ def add_ibek_attributes(cls, entity: Entity): """ # find the id field in this Entity if it has one - ids = set(a.name for a in entity.__definition__.args if isinstance(a, IdArg)) + ids = {a.name for a in entity.__definition__.args if isinstance(a, IdArg)} entity_dict = entity.model_dump() for arg, value in entity_dict.items():