diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 74ae18bd3..5e270b08d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,7 +4,6 @@ repos: hooks: - id: check-added-large-files - id: check-yaml - exclude: src/ibek/helm-template/templates - id: check-merge-conflict - repo: local diff --git a/README.rst b/README.rst index 7044b5007..91eb89a43 100644 --- a/README.rst +++ b/README.rst @@ -12,7 +12,7 @@ IOC Builder for EPICS and Kubernetes: image to create a JSON schema of what an IOC using that image can contain - Write an ``ioc.yaml`` file against that schema listing instances of the entities with arguments -- Use ``ibek`` to generate a startup script, database and Helm chart that runs +- Use ``ibek`` to generate a startup script and database that runs up the IOC contained in the image with them ============== ============================================================== diff --git a/tests/samples/helm/bl45p-mo-ioc-03.boot b/bl45p-mo-ioc-03.boot similarity index 100% rename from tests/samples/helm/bl45p-mo-ioc-03.boot rename to bl45p-mo-ioc-03.boot diff --git a/tests/samples/helm/bl45p-mo-ioc-04.boot b/bl45p-mo-ioc-04.boot similarity index 100% rename from tests/samples/helm/bl45p-mo-ioc-04.boot rename to bl45p-mo-ioc-04.boot diff --git a/docs/developer/explanations/entities.rst b/docs/developer/explanations/entities.rst index 54c6817ef..888485ab9 100644 --- a/docs/developer/explanations/entities.rst +++ b/docs/developer/explanations/entities.rst @@ -158,7 +158,7 @@ Click the arrows to reveal the files.
ioc.boot - .. include:: ../../../tests/samples/helm/ioc.boot + .. include:: ../../../tests/samples/boot_scripts/ioc.boot :literal: .. raw:: html @@ -201,9 +201,6 @@ Thus, the sequence of files is as follows: - .ibek.entities.yaml - Description of Entities for an IOC instance. * - 5 - - Helm Chart files - - The generated files for deploying the described IOC instance - * - 6 - IOC Startup Script ioc.boot - Startup script for booting the IOC @@ -262,12 +259,6 @@ The ibek commands to progress through the file sequence above are as follows - ``.ibek.entities.yaml`` - Hand crafted at IOC instance design time * - 5 - - Helm Chart files - - ``ibek build-helm .ibek.entities.yaml`` - run at IOC helm chart generation time. This generates a helm chart - with ``.ibek.entities.yaml`` in its config folder and validates it - against the schema defined at the top of the YAML file. - * - 6 - IOC startup script - ``ibek build-startup .ibek.entities.yaml ...``. Run at IOC startup time in the container. ``...`` == all ``.ibek.defs.yaml`` within the container. diff --git a/docs/user/reference/api.rst b/docs/user/reference/api.rst index f01854637..27b001582 100644 --- a/docs/user/reference/api.rst +++ b/docs/user/reference/api.rst @@ -38,12 +38,6 @@ This is the internal API reference for ibek .. automodule:: ibek.modules :members: -``ibek.helm`` -------------- - -.. automodule:: ibek.helm - :members: - ``ibek.render`` --------------- diff --git a/tests/samples/helm/ioc.boot b/ioc.boot similarity index 100% rename from tests/samples/helm/ioc.boot rename to ioc.boot diff --git a/setup.cfg b/setup.cfg index 0ffdafa4a..3de57e5e6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -65,8 +65,6 @@ where = src # Specify any package data to be included in the wheel below. [options.package_data] ibek = - helm-template/*/* - helm-template/* templates/* [options.entry_points] @@ -86,9 +84,12 @@ float_to_top=true # Make flake8 respect black's line length (default 88), max-line-length = 88 extend-ignore = - E203, # See https://github.com/PyCQA/pycodestyle/issues/373 - F811, # support typing.overload decorator - F722, # allow Annotated[typ, some_func("some string")] + # See https://github.com/PyCQA/pycodestyle/issues/373 + E203, + # support typing.overload decorator + F811, + # allow Annotated[typ, some_func("some string")] + F722, exclude = .tox venv diff --git a/src/ibek/__main__.py b/src/ibek/__main__.py index 8deb750a2..1b0d09e01 100644 --- a/src/ibek/__main__.py +++ b/src/ibek/__main__.py @@ -9,13 +9,7 @@ from ibek import __version__ -from .helm import ( - create_boot_script, - create_db_script, - create_helm, - ioc_deserialize, - load_ioc_yaml, -) +from .gen_scripts import create_boot_script, create_db_script, ioc_deserialize from .ioc import IOC, make_entity_classes from .support import Support @@ -82,28 +76,6 @@ def ioc_schema( output.write_text(schema) -@cli.command() -def build_helm( - entity: Path = typer.Argument( - ..., help="The filepath to the ioc instance entity file" - ), - out: Path = typer.Argument( - default="iocs", help="Path in which to build the helm chart" - ), - no_schema: bool = typer.Option(False, help="disable schema checking"), -): - """ - Build a startup script, database and Helm chart from .yaml - """ - - ioc_dict = load_ioc_yaml(ioc_instance_yaml=entity, no_schema=no_schema) - - with entity.open("r") as stream: - entity_yaml = stream.read() - - create_helm(ioc_dict=ioc_dict, entity_yaml=entity_yaml, path=out) - - @cli.command() def build_startup( instance: Path = typer.Argument( diff --git a/src/ibek/gen_scripts.py b/src/ibek/gen_scripts.py new file mode 100644 index 000000000..2f9a4b5a8 --- /dev/null +++ b/src/ibek/gen_scripts.py @@ -0,0 +1,65 @@ +""" +Functions for building the db and boot scripts +""" +import logging +import re +from pathlib import Path +from typing import List + +from jinja2 import Template +from ruamel.yaml.main import YAML + +from .ioc import IOC, make_entity_classes +from .render import ( + render_database_elements, + render_environment_variable_elements, + render_post_ioc_init_elements, + render_script_elements, +) +from .support import Support + +log = logging.getLogger(__name__) + +TEMPLATES = Path(__file__).parent / "templates" + +schema_modeline = re.compile(r"# *yaml-language-server *: *\$schema=([^ ]*)") +url_f = r"file://" + + +def ioc_deserialize(ioc_instance_yaml: Path, definition_yaml: List[Path]) -> IOC: + """ + Takes an ioc instance entities file, list of generic ioc definitions files. + + Returns an in memory object graph of the resulting ioc instance + """ + # Read and load the support module definitions + for yaml in definition_yaml: + support = Support.deserialize(YAML(typ="safe").load(yaml)) + make_entity_classes(support) + + # Create an IOC instance from it + return IOC.deserialize(YAML(typ="safe").load(ioc_instance_yaml)) + + +def create_db_script(ioc_instance: IOC) -> str: + """ + Create make_db.sh script for expanding the database templates + """ + with open(TEMPLATES / "make_db.jinja", "r") as f: + template = Template(f.read()) + + return template.render(database_elements=render_database_elements(ioc_instance)) + + +def create_boot_script(ioc_instance: IOC) -> str: + """ + Create the boot script for an IOC + """ + with open(TEMPLATES / "ioc.boot.jinja", "r") as f: + template = Template(f.read()) + + return template.render( + env_var_elements=render_environment_variable_elements(ioc_instance), + script_elements=render_script_elements(ioc_instance), + post_ioc_init_elements=render_post_ioc_init_elements(ioc_instance), + ) diff --git a/src/ibek/helm-template/Chart.yaml.jinja b/src/ibek/helm-template/Chart.yaml.jinja deleted file mode 100644 index 8ea46b1e5..000000000 --- a/src/ibek/helm-template/Chart.yaml.jinja +++ /dev/null @@ -1,22 +0,0 @@ -apiVersion: v2 -name: {{ ioc_name }} -description: | - {{ description }} - -type: application - -# chart and app version - these are updated automatically by CI -version: 0.0.0-b0 -appVersion: 0.0.0-b0 - -dependencies: - - name: beamline-chart - version: "1.0.0" - repository: file://../../beamline-chart - import-values: - - beamline_defaults - - name: helm-ioc-lib - version: "1.6.2" - repository: oci://ghcr.io/epics-containers - import-values: - - defaults diff --git a/src/ibek/helm-template/config/start.sh b/src/ibek/helm-template/config/start.sh deleted file mode 100644 index fc0c8cfcb..000000000 --- a/src/ibek/helm-template/config/start.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash - -# -# generic kubernetes IOC startup script -# -this_dir=$(realpath $(dirname $0)) -TOP=$(realpath ${this_dir}/..) -cd ${this_dir} - -set -x -e - -# add module paths to environment for use in ioc startup script -source ${SUPPORT}/configure/RELEASE.shell - -# if there is a non-zero length config.tz then decompress into config_untar -if [ -s config.tz ] -then - # decompress the configuration files into config_untar (/tmp is writeable) - config_dir=/tmp/config_untar - - mkdir -p ${config_dir} - tar -zxvf config.tz -C ${config_dir} -else - config_dir=${TOP}/config -fi - -# setup filenames for boot scripts -startup_src=${config_dir}/ioc.boot.yaml -boot_src=${config_dir}/ioc.boot -db_src=/tmp/make_db.sh -boot=/tmp/$(basename ${boot_src}) -db=/tmp/ioc.db - -# If there is a yaml ioc description then generate the startup and DB. -# Otherwise assume the starup script ioc.boot is provided in the config folder. -if [ -f ${startup_src} ] ; then - # get ibek defs files from this ioc and all support modules and this IOC - # defs="${SUPPORT}/*/ibek/*.ibek.defs.yaml ${TOP}/ibek/*.ibek.defs.yaml" - - # TODO - for early development we ship all latest def files in ioc/ibek - # These Should Assets published by epics-modules and ioc-pmac - defs=${TOP}/ibek/*.ibek.defs.yaml - ibek build-startup ${startup_src} ${defs} --out ${boot} --db-out ${db_src} -fi - -# build expanded database using the db_src shell script -if [ -f ${db_src} ]; then - bash ${db_src} > ${db} -fi -exec ${IOC}/bin/linux-x86_64/ioc ${boot} diff --git a/src/ibek/helm-template/templates/ioc.yaml b/src/ibek/helm-template/templates/ioc.yaml deleted file mode 100644 index b695a57fb..000000000 --- a/src/ibek/helm-template/templates/ioc.yaml +++ /dev/null @@ -1,10 +0,0 @@ -{{ include "ioc-chart.ioc" . }} - -{{- /* - add the contents of every file in the config folder of this IOC helm chart - into the config map - this must include start.sh the startup script. -*/ -}} - -{{ (.Files.Glob "config/*").AsConfig | indent 2 }} - version.txt: | - IOC {{ .Release.Name }} version {{ .Chart.AppVersion }} diff --git a/src/ibek/helm-template/values.yaml.jinja b/src/ibek/helm-template/values.yaml.jinja deleted file mode 100644 index 750a0fb8d..000000000 --- a/src/ibek/helm-template/values.yaml.jinja +++ /dev/null @@ -1 +0,0 @@ -base_image: {{ base_image }} diff --git a/src/ibek/helm.py b/src/ibek/helm.py deleted file mode 100644 index 79cac2f2b..000000000 --- a/src/ibek/helm.py +++ /dev/null @@ -1,152 +0,0 @@ -""" -Functions for building the helm chart -""" - -import json -import logging -import re -import shutil -import urllib.request -from pathlib import Path -from typing import Dict, List, Match, cast - -from jinja2 import Template -from jsonschema import validate -from ruamel.yaml.main import YAML - -from .ioc import IOC, make_entity_classes -from .render import ( - render_database_elements, - render_environment_variable_elements, - render_post_ioc_init_elements, - render_script_elements, -) -from .support import Support - -log = logging.getLogger(__name__) - -HELM_TEMPLATE = Path(__file__).parent / "helm-template" -TEMPLATES = Path(__file__).parent / "templates" - -schema_modeline = re.compile(r"# *yaml-language-server *: *\$schema=([^ ]*)") -url_f = r"file://" - - -def load_ioc_yaml(ioc_instance_yaml: Path, no_schema: bool = False) -> Dict: - """ - Read in an ioc instance entity YAML and convert to dictionary - and validate against its declared schema - """ - entity_dict = YAML().load(ioc_instance_yaml) - - if not no_schema: - try: - comment1 = entity_dict.ca.comment[1][0].value - matches = schema_modeline.match(comment1) - schema_url = cast(Match, matches).group(1) - - # allow relative file paths so that tests can use this - parts = schema_url.split(url_f) - if len(parts) > 1: - schema_url = url_f + str(Path(parts[1]).absolute()).strip() - - with urllib.request.urlopen(schema_url) as url: - entity_schema = json.loads(url.read().decode()) - - except Exception: - log.error( - f"Error getting schema for {ioc_instance_yaml}. " - "make sure it has '# yaml-language-server: $schema='" - ) - raise - - validate(entity_dict, entity_schema) - - return entity_dict - - -def ioc_deserialize(ioc_instance_yaml: Path, definition_yaml: List[Path]) -> IOC: - """ - Takes an ioc instance entities file, list of generic ioc definitions files. - - Returns an in memory object graph of the resulting ioc instance - """ - # Read and load the support module definitions - for yaml in definition_yaml: - support = Support.deserialize(YAML(typ="safe").load(yaml)) - make_entity_classes(support) - - # Create an IOC instance from it - return IOC.deserialize(YAML(typ="safe").load(ioc_instance_yaml)) - - -def create_db_script(ioc_instance: IOC) -> str: - """ - Create make_db.sh script for expanding the database templates - """ - with open(TEMPLATES / "make_db.jinja", "r") as f: - template = Template(f.read()) - - return template.render(database_elements=render_database_elements(ioc_instance)) - - -def create_boot_script(ioc_instance: IOC) -> str: - """ - Create the boot script for an IOC - """ - with open(TEMPLATES / "ioc.boot.jinja", "r") as f: - template = Template(f.read()) - - return template.render( - env_var_elements=render_environment_variable_elements(ioc_instance), - script_elements=render_script_elements(ioc_instance), - post_ioc_init_elements=render_post_ioc_init_elements(ioc_instance), - ) - - -def render_file(file_template: Path, **kwargs): - """ - replace a jinja templated file with its rendered equivalent by passing - kwargs parameters to jinja - """ - template = file_template.read_text() - text = Template(template).render(kwargs) - - file = Path(str(file_template).replace(".jinja", "")) - file.write_text(text) - file_template.unlink() - - -def create_helm(ioc_dict: Dict, entity_yaml: str, path: Path): - """ - create a boilerplate helm chart with name str in folder path - - update the values.yml and Chart.yml by rendering their jinja templates - and insert the boot script whose text is supplied in script_txt - """ - helm_folder = path / ioc_dict["ioc_name"] - - if path.exists(): - if helm_folder.exists(): - shutil.rmtree(helm_folder) - else: - # fail if parent does not exist (usually the iocs folder) - path.mkdir() - - shutil.copytree(HELM_TEMPLATE, helm_folder) - - render_file( - helm_folder / "Chart.yaml.jinja", - ioc_name=ioc_dict["ioc_name"], - description=ioc_dict["description"], - ) - render_file( - helm_folder / "values.yaml.jinja", - base_image=ioc_dict["generic_ioc_image"], - ) - - boot_script_path = helm_folder / "config" / "ioc.boot.yaml" - - # Saves rendered boot script - with open(boot_script_path, "w") as f: - f.write(entity_yaml) diff --git a/tests/samples/boot_scripts/bl45p-mo-ioc-03.boot b/tests/samples/boot_scripts/bl45p-mo-ioc-03.boot new file mode 100644 index 000000000..b1b8f4883 --- /dev/null +++ b/tests/samples/boot_scripts/bl45p-mo-ioc-03.boot @@ -0,0 +1,12 @@ +cd "/repos/epics/ioc" + +dbLoadDatabase "dbd/ioc.dbd" +ioc_registerRecordDeviceDriver(pdbbase) + +pmacAsynIPConfigure(BRICK1port, 192.168.0.12:1112) +pmacCreateController(BL45P-MO-BRICK-01, BRICK1port, 0, 8, 500, 100) +pmacCreateAxes(BL45P-MO-BRICK-01, 8) +none + +dbLoadRecords("/tmp/ioc.db") +iocInit() diff --git a/tests/samples/boot_scripts/bl45p-mo-ioc-04.boot b/tests/samples/boot_scripts/bl45p-mo-ioc-04.boot new file mode 100644 index 000000000..8f9b1367c --- /dev/null +++ b/tests/samples/boot_scripts/bl45p-mo-ioc-04.boot @@ -0,0 +1,10 @@ +cd "/repos/epics/ioc" + +dbLoadDatabase "dbd/ioc.dbd" +ioc_registerRecordDeviceDriver(pdbbase) + + +dbLoadRecords("/tmp/ioc.db") +iocInit() + +dbpf "BL45P-MO-THIN-01:Y1.TWV", "0.5" diff --git a/tests/samples/boot_scripts/ioc.boot b/tests/samples/boot_scripts/ioc.boot new file mode 100644 index 000000000..d97986112 --- /dev/null +++ b/tests/samples/boot_scripts/ioc.boot @@ -0,0 +1,11 @@ +cd "/repos/epics/ioc" + +dbLoadDatabase "dbd/ioc.dbd" +ioc_registerRecordDeviceDriver(pdbbase) + +pmacAsynIPConfigure(BRICK1port, 192.168.0.12:1112) +pmacCreateController(BL45P-MO-BRICK-01, BRICK1port, 0, 8, 500, 100) +pmacCreateAxes(BL45P-MO-BRICK-01, 8) + +dbLoadRecords("/tmp/ioc.db") +iocInit() diff --git a/tests/samples/generate_samples.sh b/tests/samples/generate_samples.sh index 47a6f5eba..f3df5e135 100755 --- a/tests/samples/generate_samples.sh +++ b/tests/samples/generate_samples.sh @@ -26,18 +26,12 @@ ibek ioc-schema ${SAMPLES_DIR}/yaml/asyn.ibek.defs.yaml ${SAMPLES_DIR}/yaml/pmac echo making a schema for bl45p-mo-ioc-04 ibek ioc-schema ${SAMPLES_DIR}/yaml/{epics,pmac}.ibek.defs.yaml $SAMPLES_DIR/schemas/bl45p-mo-ioc-04.ibek.entities.schema.json -# add --no-schema if needed (but above line should ensure that it is correct) -echo making helm files -ibek build-helm ${SAMPLES_DIR}/yaml/bl45p-mo-ioc-02.ibek.entities.yaml /tmp/ioc -cp /tmp/ioc/bl45p-mo-ioc-02/values.yaml ${SAMPLES_DIR}/helm/ -cp /tmp/ioc/bl45p-mo-ioc-02/Chart.yaml ${SAMPLES_DIR}/helm/ - echo making startup script ibek build-startup ${SAMPLES_DIR}/yaml/bl45p-mo-ioc-02.ibek.entities.yaml ${SAMPLES_DIR}/yaml/pmac.ibek.defs.yaml --out /tmp/ioc/ioc.boot --db-out /tmp/ioc/make_db.sh -cp /tmp/ioc/ioc.boot ${SAMPLES_DIR}/helm/ +cp /tmp/ioc/ioc.boot ${SAMPLES_DIR}/boot_scripts/ echo making bl45p-mo-ioc-03.boot ibek build-startup ${SAMPLES_DIR}/yaml/bl45p-mo-ioc-03.ibek.entities.yaml ${SAMPLES_DIR}/yaml/pmac.ibek.defs.yaml ${SAMPLES_DIR}/yaml/asyn.ibek.defs.yaml --out /tmp/ioc/ioc.boot --db-out /tmp/ioc/make_db.sh -cp /tmp/ioc/ioc.boot ${SAMPLES_DIR}/helm/bl45p-mo-ioc-03.boot +cp /tmp/ioc/ioc.boot ${SAMPLES_DIR}/boot_scripts/bl45p-mo-ioc-03.boot echo making bl45p-mo-ioc-04.boot ibek build-startup ${SAMPLES_DIR}/yaml/bl45p-mo-ioc-04.ibek.entities.yaml ${SAMPLES_DIR}/yaml/{epics,pmac}.ibek.defs.yaml --out /tmp/ioc/ioc.boot --db-out /tmp/ioc/make_db.sh -cp /tmp/ioc/ioc.boot ${SAMPLES_DIR}/helm/bl45p-mo-ioc-04.boot +cp /tmp/ioc/ioc.boot ${SAMPLES_DIR}/boot_scripts/bl45p-mo-ioc-04.boot diff --git a/tests/samples/helm/Chart.yaml b/tests/samples/helm/Chart.yaml deleted file mode 100644 index a0c1130a3..000000000 --- a/tests/samples/helm/Chart.yaml +++ /dev/null @@ -1,22 +0,0 @@ -apiVersion: v2 -name: bl45p-mo-ioc-02 -description: | - an example motion ioc for ibek testing - -type: application - -# chart and app version - these are updated automatically by CI -version: 0.0.0-b0 -appVersion: 0.0.0-b0 - -dependencies: - - name: beamline-chart - version: "1.0.0" - repository: file://../../beamline-chart - import-values: - - beamline_defaults - - name: helm-ioc-lib - version: "1.6.2" - repository: oci://ghcr.io/epics-containers - import-values: - - defaults \ No newline at end of file diff --git a/tests/samples/helm/values.yaml b/tests/samples/helm/values.yaml deleted file mode 100644 index eb4da9b54..000000000 --- a/tests/samples/helm/values.yaml +++ /dev/null @@ -1 +0,0 @@ -base_image: ghcr.io/epics-containers/ioc-pmac:main.run \ No newline at end of file diff --git a/tests/test_cli.py b/tests/test_cli.py index bc3563f91..25058cc58 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,5 +1,4 @@ import json -import os import subprocess import sys from pathlib import Path @@ -89,28 +88,6 @@ def test_container_schema(tmp_path: Path, samples: Path): assert expected == actual -def test_build_helm(tmp_path: Path, samples: Path): - """build an ioc helm chart from an IOC instance entity file""" - clear_entity_classes() - entity_file = samples / "yaml" / "bl45p-mo-ioc-02.ibek.entities.yaml" - - os.chdir(str(samples)) - run_cli("build-helm", entity_file, tmp_path) - - example_entity = entity_file.read_text() - actual_file = tmp_path / "bl45p-mo-ioc-02" / "config" / "ioc.boot.yaml" - actual_entity = actual_file.read_text() - - assert example_entity == actual_entity - - for test_file in ["Chart.yaml", "values.yaml"]: - example = (samples / "helm" / test_file).read_text() - actual_file = tmp_path / "bl45p-mo-ioc-02" / test_file - actual = actual_file.read_text() - - assert example == actual - - def test_build_startup_single(tmp_path: Path, samples: Path): """ build an ioc startup script from an IOC instance entity file @@ -132,7 +109,7 @@ def test_build_startup_single(tmp_path: Path, samples: Path): out_db, ) - example_boot = (samples / "helm" / "ioc.boot").read_text() + example_boot = (samples / "boot_scripts" / "ioc.boot").read_text() actual_boot = out_file.read_text() assert example_boot == actual_boot @@ -161,7 +138,7 @@ def test_build_startup_multiple(tmp_path: Path, samples: Path): out_db, ) - example_boot = (samples / "helm" / "bl45p-mo-ioc-03.boot").read_text() + example_boot = (samples / "boot_scripts" / "bl45p-mo-ioc-03.boot").read_text() actual_boot = out_file.read_text() assert example_boot == actual_boot @@ -191,7 +168,7 @@ def test_build_startup_env_vars_and_post_ioc_init(tmp_path: Path, samples: Path) out_db, ) - example_boot = (samples / "helm" / "bl45p-mo-ioc-04.boot").read_text() + example_boot = (samples / "boot_scripts" / "bl45p-mo-ioc-04.boot").read_text() actual_boot = out_file.read_text() assert example_boot == actual_boot