Add ExtraCode declarationFile and implementationFile directives

.pre-commit-config.yaml

@@ -11,12 +11,12 @@ repos:
         name: clang-tidy
         entry: clang-tidy -warnings-as-errors='*,-clang-diagnostic-deprecated-declarations' -p compile_commands.json
         types: [c++]
-        exclude: (tests/(datamodel|src)/.*(h|cc)|podioVersion.in.h|SIOFrame.*h)
+        exclude: (tests/(datamodel|src|extra_code)/.*(h|cc)|podioVersion.in.h|SIOFrame.*h)
         language: system
       - id: clang-format
         name: clang-format
         entry: .github/scripts/clang-format-hook
-        exclude: (tests/(datamodel|src)/.*(h|cc)$|podioVersion.in.h)
+        exclude: (tests/(datamodel|src|extra_code)/.*(h|cc)$|podioVersion.in.h)
         types: [c++]
         language: system
       - id: pylint

doc/datamodel_syntax.md

@@ -119,14 +119,14 @@ The `includes` will be add to the header files of the generated classes.
       includes: <newline separated list of strings (optional)>
       declaration: <string>
       implementation : <string>
-      declarationFile: <string> (to be implemented!)
-      implementationFile: <string> (to be implemented!)
+      declarationFile: <string>
+      implementationFile: <string>
     MutableExtraCode:
       includes: <newline separated list of strings (optional)>
       declaration: <string>
       implementation : <string>
-      declarationFile: <string> (to be implemented!)
-      implementationFile: <string> (to be implemented!)
+      declarationFile: <string>
+      implementationFile: <string>
 ```
 
 The code being provided has to use the macro `{name}` in place of the concrete name of the class.

python/podio_gen/podio_config_reader.py

@@ -2,6 +2,7 @@
 
 import copy
 import re
+import os
 import yaml
 
 from podio_gen.generator_utils import (
@@ -171,9 +172,15 @@ class ClassDefinitionValidator:
     # it applies and also which accessor functions to generate
     required_interface_keys = required_datatype_keys + ("Members", "Types")
 
-    valid_extra_code_keys = ("declaration", "implementation", "includes")
+    valid_extra_code_keys = (
+        "declaration",
+        "implementation",
+        "includes",
+        "declarationFile",
+        "implementationFile",
+    )
     # documented but not yet implemented
-    not_yet_implemented_extra_code = ("declarationFile", "implementationFile")
+    not_yet_implemented_extra_code = ()
 
     @classmethod
     def validate(cls, datamodel, upstream_edm=None):
@@ -205,10 +212,10 @@ def _check_components(cls, datamodel, upstream_edm):
 
             if "ExtraCode" in component:
                 for key in component["ExtraCode"]:
-                    if key not in ("declaration", "includes"):
+                    if key not in ("declaration", "declarationFile", "includes"):
                         raise DefinitionError(
                             f"'{key}' field found in 'ExtraCode' of component '{name}'."
-                            " Only 'declaration' and 'includes' are allowed here"
+                            "Only 'declaration', 'declarationFile' and 'includes' are allowed here"
                         )
 
             for member in component["Members"]:
@@ -442,8 +449,20 @@ def _handle_extracode(definition):
         """Handle the extra code definition. Currently simply returning a copy"""
         return copy.deepcopy(definition)
 
+    @staticmethod
+    def _expand_extracode(definitions, parent_path, directive):
+        """Expand extra code directives by including files from 'File' directives
+        Relative paths to files are extended with parent_path. Mutates definitions"""
+        if directive + "File" in definitions.keys():
+            filename = definitions.pop(directive + "File")
+            if not os.path.isabs(filename):
+                filename = os.path.join(parent_path, filename)
+            with open(filename, "r", encoding="utf-8") as stream:
+                contents = stream.read()
+            definitions[directive] = definitions.get(directive, "") + "\n" + contents
+
     @classmethod
-    def _read_component(cls, definition):
+    def _read_component(cls, definition, parent_path):
         """Read the component and put it into an easily digestible format."""
         component = {}
         for name, category in definition.items():
@@ -452,13 +471,17 @@ def _read_component(cls, definition):
                 for member in definition[name]:
                     # for components we do not require a description in the members
                     component["Members"].append(cls.member_parser.parse(member, False))
+            elif name == "ExtraCode":
+                extra_code = copy.deepcopy(category)
+                cls._expand_extracode(extra_code, parent_path, "declaration")
+                component[name] = extra_code
             else:
                 component[name] = copy.deepcopy(category)
 
         return component
 
     @classmethod
-    def _read_datatype(cls, value):
+    def _read_datatype(cls, value, parent_path):
         """Read the datatype and put it into an easily digestible format"""
         datatype = {}
         for category, definition in value.items():
@@ -468,6 +491,11 @@ def _read_datatype(cls, value):
                 for member in definition:
                     members.append(cls.member_parser.parse(member))
                 datatype[category] = members
+            elif category in ("ExtraCode", "MutableExtraCode"):
+                extra_code = copy.deepcopy(definition)
+                cls._expand_extracode(extra_code, parent_path, "implementation")
+                cls._expand_extracode(extra_code, parent_path, "declaration")
+                datatype[category] = extra_code
             else:
                 datatype[category] = copy.deepcopy(definition)
 
@@ -494,7 +522,7 @@ def _read_interface(cls, value):
         return interface
 
     @classmethod
-    def parse_model(cls, model_dict, package_name, upstream_edm=None):
+    def parse_model(cls, model_dict, package_name, upstream_edm=None, parent_path=None):
         """Parse a model from the dictionary, e.g. read from a yaml file."""
 
         try:
@@ -515,12 +543,12 @@ def parse_model(cls, model_dict, package_name, upstream_edm=None):
         components = {}
         if "components" in model_dict:
             for klassname, value in model_dict["components"].items():
-                components[klassname] = cls._read_component(value)
+                components[klassname] = cls._read_component(value, parent_path)
 
         datatypes = {}
         if "datatypes" in model_dict:
             for klassname, value in model_dict["datatypes"].items():
-                datatypes[klassname] = cls._read_datatype(value)
+                datatypes[klassname] = cls._read_datatype(value, parent_path)
 
         interfaces = {}
         if "interfaces" in model_dict:
@@ -549,5 +577,6 @@ def read(cls, yamlfile, package_name, upstream_edm=None):
         """Read the datamodel definition from the yamlfile."""
         with open(yamlfile, "r", encoding="utf-8") as stream:
             content = yaml.load(stream, yaml.SafeLoader)
+        parent_path = os.path.dirname(yamlfile)
 
-        return cls.parse_model(content, package_name, upstream_edm)
+        return cls.parse_model(content, package_name, upstream_edm, parent_path)

tests/datalayout.yaml

@@ -53,6 +53,15 @@ components :
       - int i{42} // is there even another value to initialize ints to?
       - std::array<double, 10> arr {1.2, 3.4} // half initialized double array
 
+  StructWithExtraCode:
+    Members:
+      - int x
+    ExtraCode:
+      declaration: "
+        int negate() { x = -x; return x; }\n
+      "
+      declarationFile: "extra_code/component_declarations.cc"
+
 datatypes :
 
   EventInfo:
@@ -209,6 +218,26 @@ datatypes :
       - TypeWithEnergy manyEnergies // multiple relations
       - ex42::AnotherTypeWithEnergy moreEnergies // multiple namespace relations
 
+  ExampleWithExternalExtraCode:
+    Description: "Type showing usage of 'declaration', 'implementation', 'declarationFile' and 'implementationFile' directives'"
+    Author: "Mateusz Jakub Fila"
+    Members:
+      - int number // a number
+    ExtraCode:
+      declaration:
+        "int add(int i) const;"
+      declarationFile: "extra_code/declarations.cc"
+      implementation:
+        "int {name}::add(int i) const { return number() + i; }"
+      implementationFile: "extra_code/implementations.cc"
+    MutableExtraCode:
+      declaration:
+        int add_inplace(int i);
+      declarationFile: "extra_code/mutable_declarations.cc"
+      implementation:
+        int {name}::add_inplace(int i) { return number() += i; }
+      implementationFile: "extra_code/mutable_implementations.cc"
+
   nsp::EnergyInNamespace:
     Description: "A type with energy in a namespace"
     Author: "Thomas Madlener"

tests/extra_code/component_declarations.cc

@@ -0,0 +1,4 @@
+int reset() {
+  x = 0;
+  return x;
+}

tests/extra_code/declarations.cc

@@ -0,0 +1,4 @@
+bool gt(int i) const {
+  return number() > i;
+}
+bool lt(int i) const;

tests/extra_code/implementations.cc

@@ -0,0 +1 @@
+bool {name}::lt(int i) const { return number() < i; }

tests/extra_code/mutable_declarations.cc

@@ -0,0 +1 @@
+int reset();

tests/extra_code/mutable_implementations.cc

@@ -0,0 +1,4 @@
+int {name}::reset() {
+  number() = 0;
+  return number();
+}

tests/unittests/unittest.cpp

@@ -43,13 +43,17 @@
 #include "datamodel/ExampleWithArray.h"
 #include "datamodel/ExampleWithArrayComponent.h"
 #include "datamodel/ExampleWithComponent.h"
+#include "datamodel/ExampleWithExternalExtraCode.h"
 #include "datamodel/ExampleWithFixedWidthIntegers.h"
 #include "datamodel/ExampleWithOneRelationCollection.h"
 #include "datamodel/ExampleWithUserInitCollection.h"
 #include "datamodel/ExampleWithVectorMemberCollection.h"
 #include "datamodel/MutableExampleCluster.h"
 #include "datamodel/MutableExampleWithArray.h"
 #include "datamodel/MutableExampleWithComponent.h"
+#include "datamodel/MutableExampleWithExternalExtraCode.h"
+#include "datamodel/StructWithExtraCode.h"
+
 #include "podio/UserDataCollection.h"
 
 TEST_CASE("AutoDelete", "[basics][memory-management]") {
@@ -398,6 +402,26 @@ TEST_CASE("Extracode", "[basics][code-gen]") {
   REQUIRE(simple.z == 3);
 }
 
+TEST_CASE("ExtraCode declarationFile and implementationFile", "[basics][code-gen]") {
+  auto mutable_number = MutableExampleWithExternalExtraCode();
+  REQUIRE(mutable_number.reset() == 0);
+  REQUIRE(mutable_number.add(2) == 2);
+  REQUIRE(mutable_number.add_inplace(1) == 1);
+  REQUIRE(mutable_number.gt(-1));
+  REQUIRE(mutable_number.lt(100));
+  ExampleWithExternalExtraCode number = mutable_number;
+  REQUIRE(number.add(1) == 2);
+  REQUIRE(number.gt(-1));
+  REQUIRE(number.lt(100));
+}
+
+TEST_CASE("ExtraCode declarationFile in component", "[basics][code-gen]") {
+  auto value = StructWithExtraCode();
+  value.x = 1;
+  REQUIRE(value.negate() == -1);
+  REQUIRE(value.reset() == 0);
+}
+
 TEST_CASE("AssociativeContainer", "[basics]") {
   auto clu1 = MutableExampleCluster();
   auto clu2 = MutableExampleCluster();