diff --git a/.github/CONTRIBUTING.rst b/.github/CONTRIBUTING.rst index 19ab494ff..fa16425fe 100644 --- a/.github/CONTRIBUTING.rst +++ b/.github/CONTRIBUTING.rst @@ -7,7 +7,7 @@ filing a new one. If you have a great idea but it involves big changes, please file a ticket before making a pull request! We want to make sure you don't spend your time coding something that might not fit the scope of the project. -.. _GitHub: https://github.com/DiamondLightSource/python3-pip-skeleton/issues +.. _GitHub: https://github.com/epics-containers/ibek/issues Issue or Discussion? -------------------- @@ -16,13 +16,13 @@ Github also offers discussions_ as a place to ask questions and share ideas. If your issue is open ended and it is not obvious when it can be "closed", please raise it as a discussion instead. -.. _discussions: https://github.com/DiamondLightSource/python3-pip-skeleton/discussions +.. _discussions: https://github.com/epics-containers/ibek/discussions Code coverage ------------- While 100% code coverage does not make a library bug-free, it significantly -reduces the number of easily caught bugs! Please make sure coverage remains the +reduces the number of easily caught bugs! Please make sure coverage rechange_linter_to_ruffs the same or is improved by a pull request! Developer guide @@ -32,4 +32,4 @@ The `Developer Guide`_ contains information on setting up a development environment, running the tests and what standards the code and documentation should follow. -.. _Developer Guide: https://diamondlightsource.github.io/python3-pip-skeleton/main/developer/how-to/contribute.html +.. _Developer Guide: https://diamondlightsource.github.io/ibek/change_linter_to_ruff/developer/how-to/contribute.html diff --git a/.github/actions/install_requirements/action.yml b/.github/actions/install_requirements/action.yml index 9a17a8f15..6cc2e5437 100644 --- a/.github/actions/install_requirements/action.yml +++ b/.github/actions/install_requirements/action.yml @@ -32,7 +32,7 @@ runs: mkdir -p lockfiles pip freeze --exclude-editable > lockfiles/${{ inputs.requirements_file }} # delete the self referencing line and make sure it isn't blank - sed -i '/file:/d' lockfiles/${{ inputs.requirements_file }} + sed -i'' -e '/file:/d' lockfiles/${{ inputs.requirements_file }} shell: bash - name: Upload lockfiles diff --git a/.github/pages/index.html b/.github/pages/index.html index 80f0a0091..c495f39f2 100644 --- a/.github/pages/index.html +++ b/.github/pages/index.html @@ -8,4 +8,4 @@ - \ No newline at end of file + diff --git a/.github/pages/make_switcher.py b/.github/pages/make_switcher.py index 39c127726..6728a5bf7 100755 --- a/.github/pages/make_switcher.py +++ b/.github/pages/make_switcher.py @@ -43,9 +43,9 @@ def get_versions(ref: str, add: Optional[str], remove: Optional[str]) -> List[st # Get a sorted list of tags tags = get_sorted_tags_list() - # Make the sorted versions list from main branches and tags + # Make the sorted versions list from change_linter_to_ruff branches and tags versions: List[str] = [] - for version in ["master", "main"] + tags: + for version in ["master", "change_linter_to_ruff"] + tags: if version in builds: versions.append(version) builds.remove(version) @@ -64,10 +64,10 @@ def write_json(path: Path, repository: str, versions: str): ] text = json.dumps(struct, indent=2) print(f"JSON switcher:\n{text}") - path.write_text(text) + path.write_text(text, encoding="utf-8") -def main(args=None): +def change_linter_to_ruff(args=None): parser = ArgumentParser( description="Make a versions.txt file from gh-pages directories" ) @@ -95,5 +95,5 @@ def main(args=None): write_json(args.output, args.repository, versions) -if __name__ == "__main__": - main() +if __name__ == "__change_linter_to_ruff__": + change_linter_to_ruff() diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index 92f2a1e4b..f5c3f4d17 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -66,7 +66,7 @@ jobs: run: pipdeptree - name: Run tests - run: pytest + run: tox -e pytest - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 @@ -183,12 +183,7 @@ jobs: - name: Push cached image to container registry if: github.ref_type == 'tag' # || github.ref_name == 'main' uses: docker/build-push-action@v3 - # This does not build the image again, it will find the image in the - # Docker cache and publish it with: - # Note build-args, context, file, and target must all match between this - # step and the previous build-push-action, otherwise this step will - # attempt to build the image again build-args: | PIP_OPTIONS=-r lockfiles/requirements.txt dist/*.whl context: artifacts/ diff --git a/.github/workflows/linkcheck.yml b/.github/workflows/linkcheck.yml index 3b24af558..d2a80410e 100644 --- a/.github/workflows/linkcheck.yml +++ b/.github/workflows/linkcheck.yml @@ -24,4 +24,4 @@ jobs: run: tox -e docs build -- -b linkcheck - name: Keepalive Workflow - uses: gautamkrishnar/keepalive-workflow@v1 \ No newline at end of file + uses: gautamkrishnar/keepalive-workflow@v1 diff --git a/.gitignore b/.gitignore index e0cf8114a..a37be99b3 100644 --- a/.gitignore +++ b/.gitignore @@ -67,5 +67,5 @@ venv* # further build artifacts lockfiles/ -# tests testing folder -tests/samples/yaml/config +# ruff cache +.ruff_cache/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index aa2a4cb2c..5bc9f001c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,9 +15,9 @@ repos: entry: black --check --diff types: [python] - - id: flake8 - name: Run flake8 + - id: ruff + name: Run ruff stages: [commit] language: system - entry: flake8 + entry: ruff types: [python] diff --git a/.vscode/extensions.json b/.vscode/extensions.json index 819229919..a1227b348 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -4,6 +4,7 @@ "ms-python.python", "tamasfe.even-better-toml", "redhat.vscode-yaml", - "ryanluker.vscode-coverage-gutters" + "ryanluker.vscode-coverage-gutters", + "charliermarsh.Ruff" ] -} \ No newline at end of file +} diff --git a/.vscode/launch.json b/.vscode/launch.json index d41428bfb..bd20e4384 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -22,4 +22,4 @@ }, } ] -} \ No newline at end of file +} diff --git a/.vscode/settings.json b/.vscode/settings.json index 2472acfd6..f5b8508fb 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,15 +1,22 @@ { "python.linting.pylintEnabled": false, - "python.linting.flake8Enabled": true, + "python.linting.flake8Enabled": false, "python.linting.mypyEnabled": true, "python.linting.enabled": true, - "python.testing.pytestArgs": [], + "python.testing.pytestArgs": [ + "--cov=python3_pip_skeleton", + "--cov-report", + "xml:cov.xml" + ], "python.testing.unittestEnabled": false, "python.testing.pytestEnabled": true, "python.formatting.provider": "black", "python.languageServer": "Pylance", "editor.formatOnSave": true, - "editor.codeActionsOnSave": { - "source.organizeImports": true + "[python]": { + "editor.codeActionsOnSave": { + "source.fixAll.ruff": false, + "source.organizeImports.ruff": true + } } -} \ No newline at end of file +} diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 946e69d4b..c999e8646 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -13,4 +13,4 @@ "problemMatcher": [], } ] -} \ No newline at end of file +} diff --git a/LICENSE b/LICENSE index 8dada3eda..adb931e88 100644 --- a/LICENSE +++ b/LICENSE @@ -41,7 +41,7 @@ form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain + of this License, Derivative Works shall not include works that rechange_linter_to_ruff separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. diff --git a/README.rst b/README.rst index 836d0f642..93d65eeba 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,5 @@ ibek -==== +=========================== |code_ci| |docs_ci| |coverage| |pypi_version| |license| @@ -27,31 +27,25 @@ TODO This project is approaching completion. The following items are still to do: - - Complete documentation in general + from ibek import __version__ - - Add a diagram and more details. Use draw.io for image, and save as SVG - with source embed in it, save as something.draw.io.svg + print(f"Hello ibek {__version__}") - DONE: Add ability to define embedded objects e.g. AsynIp and AsynSerial would both be defined by embedding AsynPort (so similar to how original builder.py works) - - Add support for enums where you define the possible values inline + $ python -m ibek --version - - DONE: update helm-template/config/start.sh to call 'ibek build-startup' when it - sees a st.cmd.yaml file in the startup directory. This change requires - embedding ibek into the epics-base container image. - - -.. |code_ci| image:: https://github.com/epics-containers/ibek/actions/workflows/code.yml/badge.svg?branch=main +.. |code_ci| image:: https://github.com/epics-containers/ibek/actions/workflows/code.yml/badge.svg?branch=change_linter_to_ruff :target: https://github.com/epics-containers/ibek/actions/workflows/code.yml :alt: Code CI -.. |docs_ci| image:: https://github.com/epics-containers/ibek/actions/workflows/docs.yml/badge.svg?branch=main +.. |docs_ci| image:: https://github.com/epics-containers/ibek/actions/workflows/docs.yml/badge.svg?branch=change_linter_to_ruff :target: https://github.com/epics-containers/ibek/actions/workflows/docs.yml :alt: Docs CI -.. |coverage| image:: https://codecov.io/gh/epics-containers/ibek/branch/master/graph/badge.svg +.. |coverage| image:: https://codecov.io/gh/epics-containers/ibek/branch/change_linter_to_ruff/graph/badge.svg :target: https://codecov.io/gh/epics-containers/ibek :alt: Test Coverage diff --git a/docs/conf.py b/docs/conf.py index 16a5f2e28..a4ceae7ed 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -52,7 +52,7 @@ # A list of (type, target) tuples (by default empty) that should be ignored when # generating warnings in "nitpicky mode". Note that type should include the -# domain name if present. Example entries would be ('py:func', 'int') or +# dochange_linter_to_ruff name if present. Example entries would be ('py:func', 'int') or # ('envvar', 'LD_LIBRARY_PATH'). nitpick_ignore = [ ("py:class", "NoneType"), @@ -63,10 +63,11 @@ ("py:class", "'object'"), ("py:class", "'id'"), ("py:class", "typing_extensions.Literal"), + ("py:class", "type"), ] -# Dont use the __init__ docstring because pydantic base classes cause sphinx -# to generate a lot of warnings +# Both the class’ and the __init__ method’s docstring are concatenated and +# inserted into the change_linter_to_ruff body of the autoclass directive autoclass_content = "class" # Order the members by the order they appear in the source code @@ -108,8 +109,7 @@ rst_epilog = """ .. _Diamond Light Source: http://www.diamond.ac.uk .. _black: https://github.com/psf/black -.. _flake8: https://flake8.pycqa.org/en/latest/ -.. _isort: https://github.com/PyCQA/isort +.. _ruff: https://beta.ruff.rs/docs/ .. _mypy: http://mypy-lang.org/ .. _pre-commit: https://pre-commit.com/ """ @@ -130,7 +130,6 @@ html_theme = "pydata_sphinx_theme" github_repo = project github_user = "epics-containers" - switcher_json = f"https://{github_user}.github.io/{github_repo}/switcher.json" switcher_exists = requests.get(switcher_json).ok if not switcher_exists: diff --git a/docs/developer/explanations/decisions/0002-switched-to-pip-skeleton.rst b/docs/developer/explanations/decisions/0002-switched-to-pip-skeleton.rst index 41d90fd4a..823c68369 100644 --- a/docs/developer/explanations/decisions/0002-switched-to-pip-skeleton.rst +++ b/docs/developer/explanations/decisions/0002-switched-to-pip-skeleton.rst @@ -1,4 +1,4 @@ -2. Adopt python3-pip-skeleton for project structure +2. Adopt ibek for project structure =================================================== Date: 2022-02-18 @@ -11,7 +11,7 @@ Accepted Context ------- -We should use the following `pip-skeleton `_. +We should use the following `pip-skeleton `_. The skeleton will ensure consistency in developer environments and package management. @@ -23,7 +23,7 @@ We have switched to using the skeleton. Consequences ------------ -This module will use a fixed set of tools as developed in python3-pip-skeleton +This module will use a fixed set of tools as developed in ibek and can pull from this skeleton to update the packaging to the latest techniques. As such, the developer environment may have changed, the following could be diff --git a/docs/developer/how-to/build-docs.rst b/docs/developer/how-to/build-docs.rst index 0174fc82d..11a5e6386 100644 --- a/docs/developer/how-to/build-docs.rst +++ b/docs/developer/how-to/build-docs.rst @@ -35,4 +35,4 @@ changes in this directory too:: $ tox -e docs autobuild -- --watch src -.. _sphinx: https://www.sphinx-doc.org/ \ No newline at end of file +.. _sphinx: https://www.sphinx-doc.org/ diff --git a/docs/developer/how-to/lint.rst b/docs/developer/how-to/lint.rst index 8f4e92dbb..2df258d8f 100644 --- a/docs/developer/how-to/lint.rst +++ b/docs/developer/how-to/lint.rst @@ -1,7 +1,7 @@ Run linting using pre-commit ============================ -Code linting is handled by black_, flake8_ and isort_ run under pre-commit_. +Code linting is handled by black_ and ruff_ run under pre-commit_. Running pre-commit ------------------ @@ -26,16 +26,14 @@ repository:: $ black . -Likewise with isort:: +Likewise with ruff:: - $ isort . + $ ruff --fix . -If you get any flake8 issues you will have to fix those manually. +Ruff may not be able to automatically fix all issues; in this case, you will have to fix those manually. VSCode support -------------- -The ``.vscode/settings.json`` will run black and isort formatters as well as -flake8 checking on save. Issues will be highlighted in the editor window. - - +The ``.vscode/settings.json`` will run black formatting as well as +ruff checking on save. Issues will be highlighted in the editor window. diff --git a/docs/developer/how-to/make-release.rst b/docs/developer/how-to/make-release.rst index 747e44a22..1449a6143 100644 --- a/docs/developer/how-to/make-release.rst +++ b/docs/developer/how-to/make-release.rst @@ -10,7 +10,7 @@ To make a new release, please follow this checklist: - Click ``Generate release notes``, review and edit these notes - Choose a title and click ``Publish Release`` -Note that tagging and pushing to the main branch has the same effect except that +Note that tagging and pushing to the change_linter_to_ruff branch has the same effect except that you will not get the option to edit the release notes. -.. _release: https://github.com/DiamondLightSource/python3-pip-skeleton/releases \ No newline at end of file +.. _release: https://github.com/epics-containers/ibek/releases diff --git a/docs/developer/reference/standards.rst b/docs/developer/reference/standards.rst index b78a719e1..5a1fd4782 100644 --- a/docs/developer/reference/standards.rst +++ b/docs/developer/reference/standards.rst @@ -10,8 +10,7 @@ Code Standards The code in this repository conforms to standards set by the following tools: - black_ for code formatting -- flake8_ for style checks -- isort_ for import ordering +- ruff_ for style checks - mypy_ for static type checking .. seealso:: @@ -61,4 +60,4 @@ Docs follow the underlining convention:: .. seealso:: - How-to guide `../how-to/build-docs` \ No newline at end of file + How-to guide `../how-to/build-docs` diff --git a/docs/developer/tutorials/dev-install.rst b/docs/developer/tutorials/dev-install.rst index d7c0cd069..203297041 100644 --- a/docs/developer/tutorials/dev-install.rst +++ b/docs/developer/tutorials/dev-install.rst @@ -65,4 +65,4 @@ This will run in parallel the following checks: - `../how-to/lint` -.. _epics-containers: https://epics-containers.github.io/main/user/tutorials/devcontainer.html \ No newline at end of file +.. _epics-containers: https://epics-containers.github.io/change_linter_to_ruff/user/tutorials/devcontainer.html diff --git a/docs/index.rst b/docs/index.rst index df33c8e67..f23dd4fa1 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -14,13 +14,13 @@ The documentation is split into 2 sections: :link: user/index :link-type: doc - The User Guide contains documentation on how to install and use python3-pip-skeleton. + The User Guide contains documentation on how to install and use ibek. .. grid-item-card:: :material-regular:`code;4em` :link: developer/index :link-type: doc - The Developer Guide contains documentation on how to develop and contribute changes back to python3-pip-skeleton. + The Developer Guide contains documentation on how to develop and contribute changes back to ibek. .. toctree:: :hidden: diff --git a/docs/user/how-to/run-container.rst b/docs/user/how-to/run-container.rst index d5bd5ce91..3bad9811c 100644 --- a/docs/user/how-to/run-container.rst +++ b/docs/user/how-to/run-container.rst @@ -12,4 +12,4 @@ To pull the container from github container registry and run:: $ docker run ghcr.io/epics-containers/ibek:main --version -To get a released version, use a numbered release instead of ``main``. +To get a released version, use a numbered release instead of ``change_linter_to_ruff``. diff --git a/pyproject.toml b/pyproject.toml index 30205642f..9c07ea861 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,6 @@ build-backend = "setuptools.build_meta" [project] name = "ibek" - classifiers = [ "Development Status :: 3 - Alpha", "License :: OSI Approved :: Apache Software License", @@ -29,13 +28,12 @@ requires-python = ">=3.8" dev = [ "black", "mypy", - "flake8-isort", - "Flake8-pyproject", "pipdeptree", "pre-commit", "pydata-sphinx-theme>=0.12", "pytest", "pytest-cov", + "ruff", "Sphinx==6.2.1", "sphinx-autobuild", "sphinx-copybutton", @@ -61,20 +59,6 @@ write_to = "src/ibek/_version.py" [tool.mypy] ignore_missing_imports = true # Ignore missing stubs in imported modules -[tool.isort] -float_to_top = true -profile = "black" - -[tool.flake8] -extend-ignore = [ - "E203", # See https://github.com/PyCQA/pycodestyle/issues/373 - "F811", # support typing.overload decorator - "F722", # allow Annotated[typ, some_func("some string")] -] -max-line-length = 88 # Respect black's line length (default 88), -exclude = [".tox", "venv"] - - [tool.pytest.ini_options] # Run pytest with all our checkers, and don't spam us with massive tracebacks on error addopts = """ @@ -111,8 +95,24 @@ allowlist_externals = sphinx-build sphinx-autobuild commands = - pytest: pytest {posargs} + pytest: pytest --cov=ibek --cov-report term --cov-report xml:cov.xml {posargs} mypy: mypy src tests {posargs} pre-commit: pre-commit run --all-files {posargs} docs: sphinx-{posargs:build -EW --keep-going} -T docs build/html """ + + +[tool.ruff] +src = ["src", "tests"] +ignore = [ + "C408", # Unnecessary collection call - e.g. list(...) instead of [...] + "E501", # Line too long, should be fixed by black. +] +line-length = 88 +select = [ + "C4", # flake8-comprehensions - https://beta.ruff.rs/docs/rules/#flake8-comprehensions-c4 + "E", # pycodestyle errors - https://beta.ruff.rs/docs/rules/#error-e + "F", # pyflakes rules - https://beta.ruff.rs/docs/rules/#pyflakes-f + "W", # pycodestyle warnings - https://beta.ruff.rs/docs/rules/#warning-w + "I001", # isort +] diff --git a/src/ibek/__main__.py b/src/ibek/__main__.py index 49f7a172b..5ef94bd8f 100644 --- a/src/ibek/__main__.py +++ b/src/ibek/__main__.py @@ -74,7 +74,7 @@ def build_startup( help="Path to output startup script", ), db_out: Path = typer.Option( - default="config/make_db.sh", + default="config/db.subst", help="Path to output database expansion shell script", ), ): diff --git a/src/ibek/gen_scripts.py b/src/ibek/gen_scripts.py index 7fff7c5fc..00c0b6a19 100644 --- a/src/ibek/gen_scripts.py +++ b/src/ibek/gen_scripts.py @@ -11,6 +11,7 @@ from .ioc import IOC, clear_entity_model_ids, make_entity_models, make_ioc_model from .render import Render +from .render_db import RenderDb from .support import Support log = logging.getLogger(__name__) @@ -65,14 +66,14 @@ def create_db_script(ioc_instance: IOC) -> str: """ Create make_db.sh script for expanding the database templates """ - with open(TEMPLATES / "make_db.jinja", "r") as f: - template = Template(f.read()) + with open(TEMPLATES / "db.subst.jinja", "r") as f: + jinja_txt = f.read() - renderer = Render() + renderer = RenderDb(ioc_instance) - return template.render( - database_elements=renderer.render_database_elements(ioc_instance), - ) + templates = renderer.render_database() + + return Template(jinja_txt).render(templates=templates) def create_boot_script(ioc_instance: IOC) -> str: diff --git a/src/ibek/ioc.py b/src/ibek/ioc.py index 0131408b4..f6e5dc07c 100644 --- a/src/ibek/ioc.py +++ b/src/ibek/ioc.py @@ -45,7 +45,7 @@ def add_ibek_attributes(cls, entity: Entity): """ # find the id field in this Entity if it has one - ids = set(a.name for a in entity.__definition__.args if isinstance(a, IdArg)) + ids = {a.name for a in entity.__definition__.args if isinstance(a, IdArg)} entity_dict = entity.model_dump() for arg, value in entity_dict.items(): diff --git a/src/ibek/render.py b/src/ibek/render.py index 686be8e15..a0a6160e8 100644 --- a/src/ibek/render.py +++ b/src/ibek/render.py @@ -111,43 +111,6 @@ def render_post_ioc_init(self, instance: Entity) -> Optional[str]: post_init = instance.__definition__.post_init return self.render_script(instance, post_init) - def render_database(self, instance: Entity) -> Optional[str]: - """ - render the lines required to instantiate database by combining the - templates from the Entity's database list with the arguments from - an Entity - """ - templates = instance.__definition__.databases - # The entity may not instantiate any database templates - if not templates: - return None - jinja_txt = "" - - for template in templates: - db_file = template.file.strip("\n") - - macros = [] - for arg, value in template.args.items(): - if value is None: - if arg not in instance.__dict__: - raise ValueError( - f"database arg '{arg}' in database template " - f"'{template.file}' not found in context" - ) - macros.append(f"{arg}={{{{ {arg} }}}}") - else: - macros.append(f"{arg}={value}") - - db_arg_string = ", ".join(macros) - - jinja_txt += ( - f'msi -I${{EPICS_DB_INCLUDE_PATH}} -M"{db_arg_string}" "{db_file}"\n' - ) - - db_txt = render_with_utils(instance, jinja_txt) # type: ignore - - return db_txt + "\n" - def render_environment_variables(self, instance: Entity) -> Optional[str]: """ render the environment variable elements by combining the jinja template @@ -193,12 +156,6 @@ def render_post_ioc_init_elements(self, ioc: IOC) -> str: """ return self.render_elements(ioc, self.render_post_ioc_init) - def render_database_elements(self, ioc: IOC) -> str: - """ - Render all of the DBLoadRecords entries for a given IOC instance - """ - return self.render_elements(ioc, self.render_database) - def render_environment_variable_elements(self, ioc: IOC) -> str: """ Render all of the environment variable entries for a given IOC instance diff --git a/src/ibek/render_db.py b/src/ibek/render_db.py new file mode 100644 index 000000000..70c9e2bb1 --- /dev/null +++ b/src/ibek/render_db.py @@ -0,0 +1,121 @@ +""" +A class for rendering a substitution file from multiple instantiations of +support module definitions +""" + +from dataclasses import dataclass +from typing import Any, Dict, List + +from ibek.globals import render_with_utils +from ibek.ioc import IOC, Entity + + +class RenderDb: + @dataclass + class RenderDbTemplate: + filename: str + rows: List[List[str]] + columns: List[int] + + def __init__(self, ioc_instance: IOC) -> None: + self.ioc_instance = ioc_instance + # a mapping from template file name to details of instances of that template + self.render_templates: Dict[str, RenderDb.RenderDbTemplate] = {} + + def add_row(self, filename: str, args: Dict[str, Any], entity: Entity) -> None: + """ + Accumulate rows of arguments for each template file, + Adding a new template file if it does not already exist. + Convert all arguments to strings. + """ + if filename not in self.render_templates: + # for new filenames create a new RenderDbTemplate entry + headings = [str(i) for i in list(args.keys())] + self.render_templates[filename] = RenderDb.RenderDbTemplate( + filename=filename, + rows=[headings], # first row is the headings + columns=[0] * len(args), + ) + + # Add a new row of argument values. + # Note the case where no value was specified for the argument + # meaning that the name is to be rendered in Jinja + row = ["{{ %s }}" % k if v is None else v for k, v in args.items()] + + # render any Jinja fields in the arguments + for i, line in enumerate(row): + row[i] = render_with_utils(dict(entity), row[i]) + + # save the new row + self.render_templates[filename].rows.append(row) + + def parse_instances(self) -> None: + """ + Gather the database template instantiations from all entities + while validating the arguments + """ + for entity_root in self.ioc_instance.entities: + # TODO this is a highly suspect way of coping with the difference + # between deserialized entities and the original entity class. + # The deserializer inserts an extra layer of nesting + # with "root". This issue comes up because test_database_render + # directly creates an IOC object and does not get the extra "root". + # + # see definition of EntityModel in ioc.py + entity = getattr(entity_root, "root", entity_root) + + templates = entity.__definition__.databases + + # Not all entities instantiate database templates + if templates is None or not entity.entity_enabled: + continue + + for template in templates: + template.file = template.file.strip("\n") + + for arg, value in template.args.items(): + if value is None: + if arg not in entity.__dict__: + raise ValueError( + f"database arg '{arg}' in database template " + f"'{template.file}' not found in context" + ) + self.add_row(template.file, template.args, entity) + + def align_columns(self) -> None: + """ + Make sure columns will line up for each template file, also + provide escaping for spaces and quotes + """ + + # first calculate the column width for each template + # including escaping spaces and quotes + for template in self.render_templates.values(): + for n, row in enumerate(template.rows): + for i, arg in enumerate(row): + row[i] = arg.replace(",", r"\,") + if n > 0: + row[i] = f'"{row[i]}"' + if i < len(template.columns) - 1: + row[i] += "," + template.columns[i] = max(template.columns[i], len(row[i]) + 1) + + # now pad each column to the maximum width + for template in self.render_templates.values(): + for row in template.rows: + for i, arg in enumerate(row): + row[i] = arg.ljust(template.columns[i]) + + def render_database(self) -> Dict[str, List[str]]: + """ + Render a database substitution file + """ + self.parse_instances() + self.align_columns() + + results = {} + + for template in self.render_templates.values(): + results[template.filename] = ["".join(row) for row in template.rows] + + return results diff --git a/src/ibek/templates/db.subst.jinja b/src/ibek/templates/db.subst.jinja new file mode 100644 index 000000000..686adee3e --- /dev/null +++ b/src/ibek/templates/db.subst.jinja @@ -0,0 +1,10 @@ +################################################################################ +# DB substitution file generated by http://github.com/epics-containers/ibek +################################################################################ +{% for template_file, rows in templates.items() %} +file "{{ template_file }}" { +pattern {% for row in rows %} +{{"\t"}}{ {{ row }} } {% endfor %} +} + +{% endfor %} \ No newline at end of file diff --git a/src/ibek/templates/make_db.jinja b/src/ibek/templates/make_db.jinja deleted file mode 100644 index bfec9cf3b..000000000 --- a/src/ibek/templates/make_db.jinja +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -{{ database_elements -}} \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index efa342148..edf8db620 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -29,6 +29,11 @@ def get_support(yaml_file: str) -> Support: return support +@fixture +def templates(): + return Path(__file__).parent.parent / "src" / "ibek" / "templates" + + @fixture def samples(): return Path(__file__).parent / "samples" diff --git a/tests/generate_samples.sh b/tests/generate_samples.sh index 8279f40d7..fa2454ff9 100755 --- a/tests/generate_samples.sh +++ b/tests/generate_samples.sh @@ -26,7 +26,7 @@ echo making an ioc schema using multiple support yaml files ibek ioc-schema yaml/objects.ibek.support.yaml yaml/all.ibek.support.yaml schemas/multiple.ibek.ioc.schema.json # echo making ioc based on objects support yaml -ibek build-startup yaml/objects.ibek.ioc.yaml yaml/objects.ibek.support.yaml --out outputs/objects.st.cmd --db-out outputs/objects.make_db.sh +ibek build-startup yaml/objects.ibek.ioc.yaml yaml/objects.ibek.support.yaml --out outputs/objects.st.cmd --db-out outputs/objects.db.subst # echo making ioc based on mutiple support yaml -ibek build-startup yaml/all.ibek.ioc.yaml yaml/objects.ibek.support.yaml yaml/all.ibek.support.yaml --out outputs/all.st.cmd --db-out outputs/all.make_db.sh +ibek build-startup yaml/all.ibek.ioc.yaml yaml/objects.ibek.support.yaml yaml/all.ibek.support.yaml --out outputs/all.st.cmd --db-out outputs/all.db.subst diff --git a/tests/samples/outputs/all.db.subst b/tests/samples/outputs/all.db.subst new file mode 100644 index 000000000..5710a2923 --- /dev/null +++ b/tests/samples/outputs/all.db.subst @@ -0,0 +1,26 @@ +################################################################################ +# DB substitution file generated by http://github.com/epics-containers/ibek +################################################################################ + +file "another_test.db" { +pattern + { name, my_int_enum, clock_rate, db_calculated, calculated_one } + { "AllObject One", "2", "dummy", "HELLO AllObject One", "AllObject One.AllObject One String.1.1.0.True" } + { "AllObject Two", "1", "1", "HELLO AllObject Two", "AllObject Two.AllObject Two String.1.1.0.True" } +} + + +file "yet_another.db" { +pattern + { name, my_object, my_float, my_bool } + { "AllObject One", "Ref1", "1.0", "True" } + { "AllObject Two", "Ref1", "1.0", "True" } +} + + +file "test.db" { +pattern + { name, ip, value } + { "Consumer Two With DB", "127.0.0.1", "Ref1.127.0.0.1" } +} + diff --git a/tests/samples/outputs/all.make_db.sh b/tests/samples/outputs/all.make_db.sh deleted file mode 100644 index a9bf588e2..000000000 --- a/tests/samples/outputs/all.make_db.sh +++ /dev/null @@ -1 +0,0 @@ -#!/bin/bash diff --git a/tests/samples/outputs/all.st.cmd b/tests/samples/outputs/all.st.cmd index a7f056089..5f3d2d401 100644 --- a/tests/samples/outputs/all.st.cmd +++ b/tests/samples/outputs/all.st.cmd @@ -25,6 +25,9 @@ clock_rate = 1 my_mixed_enum_no_default = . +# ExampleTestFunction asynPortIP name port value +ExampleTestFunction 127.0.0.1 Consumer Two With DB Ref1 Ref1.127.0.0.1 + dbLoadRecords /tmp/ioc.db iocInit diff --git a/tests/samples/outputs/objects.db.subst b/tests/samples/outputs/objects.db.subst new file mode 100644 index 000000000..f9e179719 --- /dev/null +++ b/tests/samples/outputs/objects.db.subst @@ -0,0 +1,11 @@ +################################################################################ +# DB substitution file generated by http://github.com/epics-containers/ibek +################################################################################ + +file "test.db" { +pattern + { name, ip, value } + { "Consumer of another port", "10.0.0.2", "AsynPort2.10.0.0.2" } + { "Another Consumer of the 2nd port", "10.0.0.2", "AsynPort2.10.0.0.2" } +} + diff --git a/tests/samples/outputs/objects.make_db.sh b/tests/samples/outputs/objects.make_db.sh deleted file mode 100644 index c94639a98..000000000 --- a/tests/samples/outputs/objects.make_db.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -msi -I${EPICS_DB_INCLUDE_PATH} -M"name=Consumer of another port, ip=10.0.0.2, value=AsynPort2.10.0.0.2" "test.db" -msi -I${EPICS_DB_INCLUDE_PATH} -M"name=Another Consumer of the 2nd port, ip=10.0.0.2, value=AsynPort2.10.0.0.2" "test.db" diff --git a/tests/samples/schemas/multiple.ibek.ioc.schema.json b/tests/samples/schemas/multiple.ibek.ioc.schema.json index edde6228f..e0da01654 100644 --- a/tests/samples/schemas/multiple.ibek.ioc.schema.json +++ b/tests/samples/schemas/multiple.ibek.ioc.schema.json @@ -109,10 +109,10 @@ "title": "My Bool", "type": "boolean" }, - "a value calculated from args": { + "calculated_one": { "default": "{{ name }}.{{ my_str }}.{{ my_int }}.{{ my_float }}.{{ my_bool }}", "description": "test jinja render of arg values", - "title": "A Value Calculated From Args", + "title": "Calculated One", "type": "string" } }, diff --git a/tests/samples/yaml/all.ibek.ioc.yaml b/tests/samples/yaml/all.ibek.ioc.yaml index ca2d02f0f..87c89d0c6 100644 --- a/tests/samples/yaml/all.ibek.ioc.yaml +++ b/tests/samples/yaml/all.ibek.ioc.yaml @@ -24,6 +24,10 @@ entities: clock_rate: 2Hz my_str: AllObject Two String + - type: object_module.ConsumerTwo + name: Consumer Two With DB + PORT: Ref1 + - type: object_module.ConsumerTwo name: Disabled Consumer PORT: Ref1 diff --git a/tests/samples/yaml/all.ibek.support.yaml b/tests/samples/yaml/all.ibek.support.yaml index df706b155..b8fd97021 100644 --- a/tests/samples/yaml/all.ibek.support.yaml +++ b/tests/samples/yaml/all.ibek.support.yaml @@ -71,7 +71,7 @@ defs: default: true values: - - name: a value calculated from args + - name: calculated_one value: "{{ name }}.{{ my_str }}.{{ my_int }}.{{ my_float }}.{{ my_bool }}" description: test jinja render of arg values @@ -96,3 +96,18 @@ defs: args: identifier: "{{ name }}" TestValue: test_value:{{ test_value }} + + databases: + - file: another_test.db + args: + name: + my_int_enum: + clock_rate: + db_calculated: "HELLO {{ name }}" + calculated_one: + - file: yet_another.db + args: + name: + my_object: + my_float: + my_bool: diff --git a/tests/test_cli.py b/tests/test_cli.py index 45bb01ee5..8c2d25715 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,3 +1,7 @@ +""" +System tests that run the CLI commands and compare the output to expected +results. +""" import json import subprocess import sys @@ -20,8 +24,6 @@ def test_ibek_schema(tmp_path: Path, samples: Path): expected = json.loads( (samples / "schemas" / "ibek.support.schema.json").read_text() ) - # Don't care if version number didn't update to match if the rest is the same - # expected["title"] = mock.ANY actual = json.loads((schema_path).read_text()) assert expected == actual @@ -71,7 +73,7 @@ def test_build_startup_single(tmp_path: Path, samples: Path): ioc_yaml = samples / "yaml" / "objects.ibek.ioc.yaml" support_yaml = samples / "yaml" / "objects.ibek.support.yaml" out_file = tmp_path / "new_dir" / "st.cmd" - out_db = tmp_path / "new_dir" / "make_db.sh" + out_db = tmp_path / "new_dir" / "objects.db.subst" run_cli( "build-startup", @@ -87,7 +89,7 @@ def test_build_startup_single(tmp_path: Path, samples: Path): actual_boot = out_file.read_text() assert example_boot == actual_boot - example_db = (samples / "outputs" / "objects.make_db.sh").read_text() + example_db = (samples / "outputs" / "objects.db.subst").read_text() actual_db = out_db.read_text() assert example_db == actual_db @@ -96,13 +98,16 @@ def test_build_startup_multiple(tmp_path: Path, samples: Path): """ build an ioc startup script from an IOC instance entity file and multiple support module definition files + + Also verifies database subst file generation for multiple + entity instantiations. """ clear_entity_model_ids() ioc_yaml = samples / "yaml" / "all.ibek.ioc.yaml" support_yaml = samples / "yaml" / "objects.ibek.support.yaml" support_yaml2 = samples / "yaml" / "all.ibek.support.yaml" out_file = tmp_path / "st.cmd" - out_db = tmp_path / "make_db.sh" + out_db = tmp_path / "all.db.subst" run_cli( "build-startup", @@ -119,6 +124,6 @@ def test_build_startup_multiple(tmp_path: Path, samples: Path): actual_boot = out_file.read_text() assert example_boot == actual_boot - example_db = (samples / "outputs" / "all.make_db.sh").read_text() + example_db = (samples / "outputs" / "all.db.subst").read_text() actual_db = out_db.read_text() assert example_db == actual_db diff --git a/tests/test_render.py b/tests/test_render.py index 18e94e0ba..e30462de4 100644 --- a/tests/test_render.py +++ b/tests/test_render.py @@ -4,7 +4,9 @@ """ from typing import Literal +from ibek.ioc import IOC, clear_entity_model_ids from ibek.render import Render +from ibek.render_db import RenderDb def find_entity_class(entity_classes, entity_type): @@ -45,21 +47,35 @@ def test_obj_ref_script(objects_classes): ) -def test_database_render(objects_classes): +def test_database_render(objects_classes, templates): ref_cls = find_entity_class(objects_classes, "object_module.RefObject") consumer = find_entity_class(objects_classes, "object_module.ConsumerTwo") ref_cls(name="test_ref_object") my_consumer = consumer(name="test_consumer", PORT="test_ref_object") - render = Render() - db_txt = render.render_database(my_consumer) + # make a dummy IOC with two entities as database render works against + # a whole IOC rather than a single entity at a time. + clear_entity_model_ids() - assert ( - db_txt == 'msi -I${EPICS_DB_INCLUDE_PATH} -M"name=test_consumer, ' - 'ip=127.0.0.1, value=test_ref_object.127.0.0.1" "test.db"\n' + ioc = IOC( + ioc_name="test_ioc", + description="for testing", + generic_ioc_image="test_ioc_img", + entities=[], ) + ioc.entities.append(my_consumer) + render_db = RenderDb(ioc) + templates = render_db.render_database() + + assert templates == { + "test.db": [ + "name, ip, value", + '"test_consumer", "127.0.0.1", "test_ref_object.127.0.0.1"', + ] + } + def test_environment_variables(objects_classes): ref_cls = find_entity_class(objects_classes, "object_module.RefObject")