From db1bfaeb372b92a270e7e5b52b976a2d21959eae Mon Sep 17 00:00:00 2001 From: Lars Asplund Date: Tue, 13 Aug 2024 12:38:09 +0200 Subject: [PATCH] Added support for setting run_all_in_same_sim and fail_on_warning attributes from Python --- docs/news.d/1053.feature.rst | 1 + docs/run/img/tb_fail_on_warning.html | 21 ++++++ docs/run/img/tb_run_all_in_same_sim.html | 5 +- docs/run/img/tb_stopping_failure.html | 2 + docs/run/img/tb_stopping_failure_stdout.html | 62 ++++++++--------- docs/run/src/run.py | 11 ++- docs/run/src/tb_fail_on_warning.vhd | 38 +++++++++++ docs/run/src/tb_run_all_in_same_sim.vhd | 3 +- docs/run/src/tb_stopping_failure.vhd | 2 + docs/run/user_guide.rst | 22 +++++- tests/acceptance/artificial/verilog/run.py | 7 +- .../verilog/tb_fail_on_warning_from_python.sv | 15 ++++ .../tb_same_sim_from_python_all_pass.sv | 41 +++++++++++ .../tb_same_sim_from_python_some_fail.sv | 28 ++++++++ tests/acceptance/artificial/vhdl/run.py | 12 +++- .../vhdl/tb_fail_on_warning_from_python.vhd | 23 +++++++ .../vhdl/tb_same_sim_from_python_all_pass.vhd | 51 ++++++++++++++ .../tb_same_sim_from_python_some_fail.vhd | 38 +++++++++++ tests/acceptance/test_artificial.py | 17 ++++- tests/unit/test_test_bench.py | 34 ++++++++++ vunit/test/bench.py | 68 ++++++++++++++++--- 21 files changed, 446 insertions(+), 55 deletions(-) create mode 100644 docs/news.d/1053.feature.rst create mode 100644 docs/run/img/tb_fail_on_warning.html create mode 100644 docs/run/src/tb_fail_on_warning.vhd create mode 100644 tests/acceptance/artificial/verilog/tb_fail_on_warning_from_python.sv create mode 100644 tests/acceptance/artificial/verilog/tb_same_sim_from_python_all_pass.sv create mode 100644 tests/acceptance/artificial/verilog/tb_same_sim_from_python_some_fail.sv create mode 100644 tests/acceptance/artificial/vhdl/tb_fail_on_warning_from_python.vhd create mode 100644 tests/acceptance/artificial/vhdl/tb_same_sim_from_python_all_pass.vhd create mode 100644 tests/acceptance/artificial/vhdl/tb_same_sim_from_python_some_fail.vhd diff --git a/docs/news.d/1053.feature.rst b/docs/news.d/1053.feature.rst new file mode 100644 index 000000000..b2dce6176 --- /dev/null +++ b/docs/news.d/1053.feature.rst @@ -0,0 +1 @@ +Added support for setting ``run_all_in_same_sim`` and ``fail_on_warning attributes`` from Python. diff --git a/docs/run/img/tb_fail_on_warning.html b/docs/run/img/tb_fail_on_warning.html new file mode 100644 index 000000000..285784250 --- /dev/null +++ b/docs/run/img/tb_fail_on_warning.html @@ -0,0 +1,21 @@ +
test_runner : process
+  variable my_vector : integer_vector(1 to 17);
+begin
+  test_runner_setup(runner, runner_cfg);
+
+  -- vunit: fail_on_warning
+  while test_suite loop
+    if run("Test that fails on an assert") then
+      assert false;
+    elsif run("Test that crashes on boundary problems") then
+      report to_string(my_vector(runner_cfg'length));
+    elsif run("Test that fails on VUnit check procedure") then
+      check_equal(17, 18);
+    elsif run("Test that a warning passes") then
+      assert false severity warning;
+    end if;
+  end loop;
+
+  test_runner_cleanup(runner);
+end process;
+
diff --git a/docs/run/img/tb_run_all_in_same_sim.html b/docs/run/img/tb_run_all_in_same_sim.html index 658f71aba..1e82f4e0c 100644 --- a/docs/run/img/tb_run_all_in_same_sim.html +++ b/docs/run/img/tb_run_all_in_same_sim.html @@ -1,6 +1,4 @@ -
-- vunit: run_all_in_same_sim
-
-library vunit_lib;
+
library vunit_lib;
 context vunit_lib.vunit_context;
 
 entity tb_run_all_in_same_sim is
@@ -13,6 +11,7 @@
   begin
     test_runner_setup(runner, runner_cfg);
 
+    -- vunit: run_all_in_same_sim
     while test_suite loop
       if run("Test to_string for integer again") then
         check_equal(to_string(17), "17");
diff --git a/docs/run/img/tb_stopping_failure.html b/docs/run/img/tb_stopping_failure.html
index 889a2e458..98152459a 100644
--- a/docs/run/img/tb_stopping_failure.html
+++ b/docs/run/img/tb_stopping_failure.html
@@ -10,6 +10,8 @@
       report to_string(my_vector(runner_cfg'length));
     elsif run("Test that fails on VUnit check procedure") then
       check_equal(17, 18);
+    elsif run("Test that a warning passes") then
+      assert false severity warning;
     end if;
   end loop;
 
diff --git a/docs/run/img/tb_stopping_failure_stdout.html b/docs/run/img/tb_stopping_failure_stdout.html
index 0bba045c6..63d7964a7 100644
--- a/docs/run/img/tb_stopping_failure_stdout.html
+++ b/docs/run/img/tb_stopping_failure_stdout.html
@@ -1,46 +1,40 @@
 
> python run.py
 Starting lib.tb_stopping_failure.Test that fails on an assert
-Output file: C:\github\vunit\docs\run\src\vunit_out\test_output\lib.tb_stopping_failure.Test_that_fails_on_an_assert_f53b930e2c7649bc33253af52f8ea89a9c05f07b\output.txt
-C:\github\vunit\docs\run\src\tb_stopping_failure.vhd:24:9:@0ms:(assertion error): Assertion violation
-C:\ghdl\bin\ghdl.exe:error: assertion failed
-in process .tb_stopping_failure(tb).test_runner
-C:\ghdl\bin\ghdl.exe:error: simulation failed
-fail (P=0 S=0 F=1 T=3) lib.tb_stopping_failure.Test that fails on an assert (0.5 seconds)
+Output file: C:\repos\vunit\docs\run\src\vunit_out\test_output\lib.tb_stopping_failure.Test_that_fails_on_an_assert_f53b930e2c7649bc33253af52f8ea89a9c05f07b\output.txt
+C:\repos\vunit\docs\run\src\tb_stopping_failure.vhd:24:9:@0ms:(assertion error): Assertion violation
+ghdl:error: assertion failed
+ghdl:error: simulation failed
+fail (P=0 S=0 F=1 T=4) lib.tb_stopping_failure.Test that fails on an assert (0.5 s)
 
-Starting lib.tb_stopping_failure.Test that crashes on boundary problems
-Output file: C:\github\vunit\docs\run\src\vunit_out\test_output\lib.tb_stopping_failure.Test_that_crashes_on_boundary_problems_b53105615efefaa16d0cf9ee1bad37b5d3369e95\output.txt
-C:\ghdl\bin\ghdl.exe:error: index (316) out of bounds (1 to 17) at C:\github\vunit\docs\run\src\tb_stopping_failure.vhd:26
-in process .tb_stopping_failure(tb).test_runner
-C:\ghdl\bin\ghdl.exe:error: simulation failed
-fail (P=0 S=0 F=2 T=3) lib.tb_stopping_failure.Test that crashes on boundary problems (0.5 seconds)
+(11:35:31) Starting lib.tb_stopping_failure.Test that crashes on boundary problems
+Output file: C:\repos\vunit\docs\run\src\vunit_out\test_output\lib.tb_stopping_failure.Test_that_crashes_on_boundary_problems_b53105615efefaa16d0cf9ee1bad37b5d3369e95\output.txt
+ghdl:error: index (314) out of bounds (1 to 17) at C:\repos\vunit\docs\run\src\tb_stopping_failure.vhd:26
+ghdl:error: simulation failed
+fail (P=0 S=0 F=2 T=4) lib.tb_stopping_failure.Test that crashes on boundary problems (0.5 s)
 
-Starting lib.tb_stopping_failure.Test that fails on VUnit check procedure
-Output file: C:\github\vunit\docs\run\src\vunit_out\test_output\lib.tb_stopping_failure.Test_that_fails_on_VUnit_check_procedure_717a6f8ff044e3d5fa7d7d3ec5a32971d74864dd\output.txt
+(11:35:31) Starting lib.tb_stopping_failure.Test that fails on VUnit check procedure
+Output file: C:\repos\vunit\docs\run\src\vunit_out\test_output\lib.tb_stopping_failure.Test_that_fails_on_VUnit_check_procedure_717a6f8ff044e3d5fa7d7d3ec5a32971d74864dd\output.txt
                0 fs - check                -   ERROR - Equality check failed - Got 17. Expected 18.
-C:\github\vunit\vunit\vhdl\core\src\core_pkg.vhd:84:7:@0ms:(report failure): Stop simulation on log level error
-C:\ghdl\bin\ghdl.exe:error: report failed
-in process .tb_stopping_failure(tb).test_runner
-  from: vunit_lib.logger_pkg.decrease_stop_count at logger_pkg-body.vhd:736
-  from: vunit_lib.logger_pkg.decrease_stop_count at logger_pkg-body.vhd:731
-  from: vunit_lib.logger_pkg.log at logger_pkg-body.vhd:910
-  from: vunit_lib.checker_pkg.log_failing_check at checker_pkg-body.vhd:258
-  from: vunit_lib.checker_pkg.failing_check at checker_pkg-body.vhd:275
-  from: vunit_lib.check_pkg.check_equal at check.vhd:4358
-  from: vunit_lib.check_pkg.check_equal at check.vhd:4313
-  from: process lib.tb_stopping_failure(tb).test_runner at tb_stopping_failure.vhd:28
-C:\ghdl\bin\ghdl.exe:error: simulation failed
-fail (P=0 S=0 F=3 T=3) lib.tb_stopping_failure.Test that fails on VUnit check procedure (0.5 seconds)
+C:\repos\vunit\vunit\vhdl\core\src\core_pkg.vhd:85:7:@0ms:(report failure): Stop simulation on log level error
+ghdl:error: report failed
+ghdl:error: simulation failed
+fail (P=0 S=0 F=3 T=4) lib.tb_stopping_failure.Test that fails on VUnit check procedure (0.5 s)
+
+(11:35:32) Starting lib.tb_stopping_failure.Test that a warning passes
+Output file: C:\repos\vunit\docs\run\src\vunit_out\test_output\lib.tb_stopping_failure.Test_that_a_warning_passes_7db91f3b27aea5f89e74e39ea51ce6d61558674e\output.txt
+pass (P=1 S=0 F=3 T=4) lib.tb_stopping_failure.Test that a warning passes (0.4 s)
 
 ==== Summary ============================================================================
-fail lib.tb_stopping_failure.Test that fails on an assert             (0.5 seconds)
-fail lib.tb_stopping_failure.Test that crashes on boundary problems   (0.5 seconds)
-fail lib.tb_stopping_failure.Test that fails on VUnit check procedure (0.5 seconds)
+pass lib.tb_stopping_failure.Test that a warning passes               (0.4 s)
+fail lib.tb_stopping_failure.Test that fails on an assert             (0.5 s)
+fail lib.tb_stopping_failure.Test that crashes on boundary problems   (0.5 s)
+fail lib.tb_stopping_failure.Test that fails on VUnit check procedure (0.5 s)
 =========================================================================================
-pass 0 of 3
-fail 3 of 3
+pass 1 of 4
+fail 3 of 4
 =========================================================================================
-Total time was 1.6 seconds
-Elapsed time was 1.6 seconds
+Total time was 1.8 s
+Elapsed time was 1.8 s
 =========================================================================================
 Some failed!
 

diff --git a/docs/run/src/run.py b/docs/run/src/run.py
index ba58bbe86..f048e5736 100644
--- a/docs/run/src/run.py
+++ b/docs/run/src/run.py
@@ -90,6 +90,15 @@ def extract_snippets():
             snippet,
         )
 
+    for snippet in [
+        "tb_fail_on_warning",
+    ]:
+        highlight_code(
+            root / "tb_fail_on_warning.vhd",
+            root / ".." / "img" / f"{snippet}.html",
+            snippet,
+        )
+
     for snippet in [
         "tb_stop_level",
     ]:
@@ -164,7 +173,7 @@ def _post_run(results):
         args.verbose = True
         options += " -v"
 
-    vu = VUnit.from_args(args=args, compile_builtins=False)
+    vu = VUnit.from_args(args=args)
     vu.add_vhdl_builtins()
 
     lib = vu.add_library("lib")
diff --git a/docs/run/src/tb_fail_on_warning.vhd b/docs/run/src/tb_fail_on_warning.vhd
new file mode 100644
index 000000000..1e56fc822
--- /dev/null
+++ b/docs/run/src/tb_fail_on_warning.vhd
@@ -0,0 +1,38 @@
+-- This Source Code Form is subject to the terms of the Mozilla Public
+-- License, v. 2.0. If a copy of the MPL was not distributed with this file,
+-- You can obtain one at http://mozilla.org/MPL/2.0/.
+--
+-- Copyright (c) 2014-2024, Lars Asplund lars.anders.asplund@gmail.com
+
+library vunit_lib;
+context vunit_lib.vunit_context;
+
+entity tb_fail_on_warning is
+  generic(runner_cfg : string);
+end entity;
+
+architecture tb of tb_fail_on_warning is
+begin
+  -- start_snippet tb_fail_on_warning
+  test_runner : process
+    variable my_vector : integer_vector(1 to 17);
+  begin
+    test_runner_setup(runner, runner_cfg);
+
+    -- vunit: fail_on_warning
+    while test_suite loop
+      if run("Test that fails on an assert") then
+        assert false;
+      elsif run("Test that crashes on boundary problems") then
+        report to_string(my_vector(runner_cfg'length));
+      elsif run("Test that fails on VUnit check procedure") then
+        check_equal(17, 18);
+      elsif run("Test that a warning passes") then
+        assert false severity warning;
+      end if;
+    end loop;
+
+    test_runner_cleanup(runner);
+  end process;
+  -- end_snippet tb_fail_on_warning
+end;
diff --git a/docs/run/src/tb_run_all_in_same_sim.vhd b/docs/run/src/tb_run_all_in_same_sim.vhd
index c280ac903..6250df05f 100644
--- a/docs/run/src/tb_run_all_in_same_sim.vhd
+++ b/docs/run/src/tb_run_all_in_same_sim.vhd
@@ -5,8 +5,6 @@
 -- Copyright (c) 2014-2024, Lars Asplund lars.anders.asplund@gmail.com
 
 -- start_snippet tb_run_all_in_same_sim
--- vunit: run_all_in_same_sim
-
 library vunit_lib;
 context vunit_lib.vunit_context;
 
@@ -20,6 +18,7 @@ begin
   begin
     test_runner_setup(runner, runner_cfg);
 
+    -- vunit: run_all_in_same_sim
     while test_suite loop
       if run("Test to_string for integer again") then
         check_equal(to_string(17), "17");
diff --git a/docs/run/src/tb_stopping_failure.vhd b/docs/run/src/tb_stopping_failure.vhd
index 4120d713b..15cdd7771 100644
--- a/docs/run/src/tb_stopping_failure.vhd
+++ b/docs/run/src/tb_stopping_failure.vhd
@@ -26,6 +26,8 @@ begin
         report to_string(my_vector(runner_cfg'length));
       elsif run("Test that fails on VUnit check procedure") then
         check_equal(17, 18);
+      elsif run("Test that a warning passes") then
+        assert false severity warning;
       end if;
     end loop;
 
diff --git a/docs/run/user_guide.rst b/docs/run/user_guide.rst
index 2499b1cff..be74a1979 100644
--- a/docs/run/user_guide.rst
+++ b/docs/run/user_guide.rst
@@ -163,12 +163,18 @@ good reasons for this
 Possible drawbacks to this approach are that test cases have to be independent and the overhead
 of starting a new simulation for each test case (this is typically less than one second per test case). If that
 is the case you can force all test cases of a testbench to be run in the same simulation. This is done by adding
-the ``run_all_in_same_sim`` attribute.
+the ``run_all_in_same_sim`` attribute (``-- vunit: run_all_in_same_sim``) before the test suite.
 
 .. raw:: html
     :file: img/tb_run_all_in_same_sim.html
 
 
+The ``run_all_in_same_sim`` attribute can also be set from the run script, see :class:`vunit.ui.testbench.TestBench`.
+
+.. important::
+   When setting ``run_all_in_same_sim`` from the run script, the setting must be identical for all configurations
+   of the testbench.
+	   
 The VUnit Watchdog
 ------------------
 
@@ -196,11 +202,23 @@ test.
 .. raw:: html
     :file: img/tb_stopping_failure.html
 
-All these test cases will fail
+All but the last of these test cases will fail
 
 .. raw:: html
     :file: img/tb_stopping_failure_stdout.html
 
+By setting the VUnit ``fail_on_warning`` attribute (``-- vunit: fail_on_warning``) before the test suite,
+the last test case will also fail.
+
+.. raw:: html
+    :file: img/tb_fail_on_warning.html
+
+The ``fail_on_warning`` attribute can also be set from the run script, see :class:`vunit.ui.testbench.TestBench`.
+
+.. important::
+   When setting ``fail_on_warning`` from the run script, the setting must be identical for all configurations
+   of the testbench.
+
 Counting Errors with VUnit Logging/Check Libraries
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/tests/acceptance/artificial/verilog/run.py b/tests/acceptance/artificial/verilog/run.py
index a7f1556d5..73cd289c4 100644
--- a/tests/acceptance/artificial/verilog/run.py
+++ b/tests/acceptance/artificial/verilog/run.py
@@ -63,10 +63,13 @@ def post_check(output_path):
 
     module = ui.library("lib").module("tb_same_sim_all_pass")
     module.add_config("cfg", post_check=post_check)
-
-
+    module = ui.library("lib").module("tb_same_sim_from_python_all_pass")
+    module.add_config("cfg", post_check=post_check, attributes=dict(run_all_in_same_sim=True))
+   
 configure_tb_with_parameter_config()
 configure_tb_same_sim_all_pass(vu)
+lib.module("tb_same_sim_from_python_some_fail").set_attribute("run_all_in_same_sim", True)
 lib.module("tb_other_file_tests").scan_tests_from_file(str(root / "other_file_tests.sv"))
+lib.module("tb_fail_on_warning_from_python").set_attribute("fail_on_warning", True)
 
 vu.main()
diff --git a/tests/acceptance/artificial/verilog/tb_fail_on_warning_from_python.sv b/tests/acceptance/artificial/verilog/tb_fail_on_warning_from_python.sv
new file mode 100644
index 000000000..90a3a98db
--- /dev/null
+++ b/tests/acceptance/artificial/verilog/tb_fail_on_warning_from_python.sv
@@ -0,0 +1,15 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// Copyright (c) 2014-2024, Lars Asplund lars.anders.asplund@gmail.com
+
+`include "vunit_defines.svh"
+
+module tb_fail_on_warning_from_python;
+   `TEST_SUITE begin
+      `TEST_CASE("fail") begin
+         $warning("A warning");
+      end
+   end;
+endmodule
diff --git a/tests/acceptance/artificial/verilog/tb_same_sim_from_python_all_pass.sv b/tests/acceptance/artificial/verilog/tb_same_sim_from_python_all_pass.sv
new file mode 100644
index 000000000..09f69dc33
--- /dev/null
+++ b/tests/acceptance/artificial/verilog/tb_same_sim_from_python_all_pass.sv
@@ -0,0 +1,41 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// Copyright (c) 2014-2024, Lars Asplund lars.anders.asplund@gmail.com
+
+`include "vunit_defines.svh"
+
+module tb_same_sim_from_python_all_pass;
+
+   parameter string output_path = "";
+
+   integer counter = 1;
+
+   `TEST_SUITE begin
+
+      `TEST_CASE("Test 1") begin
+         $info("Test 1");
+         `CHECK_EQUAL(counter, 1);
+         counter = counter + 1;
+      end
+
+      `TEST_CASE("Test 2") begin
+         $info("Test 2");
+         `CHECK_EQUAL(counter, 2);
+         counter = counter + 1;
+      end
+
+      `TEST_CASE("Test 3") begin
+         int fd;
+         $info("Test 3");
+         `CHECK_EQUAL(counter, 3);
+         counter = counter + 1;
+         fd = $fopen({output_path, "post_check.txt"});
+         $fwrite(fd, "Test 3 was here");
+         $fclose(fd);
+      end
+   end;
+
+   `WATCHDOG(1ns);
+endmodule
diff --git a/tests/acceptance/artificial/verilog/tb_same_sim_from_python_some_fail.sv b/tests/acceptance/artificial/verilog/tb_same_sim_from_python_some_fail.sv
new file mode 100644
index 000000000..f207c1f6b
--- /dev/null
+++ b/tests/acceptance/artificial/verilog/tb_same_sim_from_python_some_fail.sv
@@ -0,0 +1,28 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// Copyright (c) 2014-2024, Lars Asplund lars.anders.asplund@gmail.com
+
+`include "vunit_defines.svh"
+
+module tb_same_sim_from_python_some_fail;
+
+   `TEST_SUITE begin
+
+      `TEST_CASE("Test 1") begin
+         $info("Test 1");
+      end
+
+      `TEST_CASE("Test 2") begin
+         $info("Test 2");
+         $error("");
+      end
+
+      `TEST_CASE("Test 3") begin
+         $info("Test 3");
+      end
+   end;
+
+   `WATCHDOG(1ns);
+endmodule
diff --git a/tests/acceptance/artificial/vhdl/run.py b/tests/acceptance/artificial/vhdl/run.py
index 6e1476e33..e2d489d5a 100644
--- a/tests/acceptance/artificial/vhdl/run.py
+++ b/tests/acceptance/artificial/vhdl/run.py
@@ -60,7 +60,9 @@ def post_check(output_path):
             return "Test 3 was here" in fptr.read()
 
     ent = ui.library("lib").entity("tb_same_sim_all_pass")
-    ent.add_config("cfg", generics=dict(), post_check=post_check)
+    ent.add_config("cfg", post_check=post_check)
+    ent = ui.library("lib").entity("tb_same_sim_from_python_all_pass")
+    ent.add_config("cfg", post_check=post_check, attributes=dict(run_all_in_same_sim=True))
 
 
 def configure_tb_set_generic(ui):
@@ -115,13 +117,21 @@ def post_check(output_path):
     test_2.add_config(name="cfg2", post_check=make_post_check("arch2"))
     test_3.add_config(name="cfg3", post_check=make_post_check("arch3"), vhdl_configuration_name="cfg3")
 
+def configure_tb_no_fail_on_warning(ui):
+    tb = ui.library("lib").test_bench("tb_no_fail_on_warning")
+    tb.add_config(name="cfg1", attributes=dict(fail_on_warning=False))
+    tb.add_config(name="cfg2")
 
 configure_tb_with_generic_config()
 configure_tb_same_sim_all_pass(vu)
 configure_tb_set_generic(vu)
 configure_tb_assert_stop_level(vu)
 configure_tb_with_vhdl_configuration(vu)
+configure_tb_no_fail_on_warning(vu)
 lib.entity("tb_no_generic_override").set_generic("g_val", False)
 lib.entity("tb_ieee_warning").test("pass").set_sim_option("disable_ieee_warnings", True)
 lib.entity("tb_other_file_tests").scan_tests_from_file(str(root / "other_file_tests.vhd"))
+lib.entity("tb_same_sim_from_python_some_fail").set_attribute("run_all_in_same_sim", True)
+lib.entity("tb_fail_on_warning_from_python").set_attribute("fail_on_warning", True)
+
 vu.main()
diff --git a/tests/acceptance/artificial/vhdl/tb_fail_on_warning_from_python.vhd b/tests/acceptance/artificial/vhdl/tb_fail_on_warning_from_python.vhd
new file mode 100644
index 000000000..76af047e5
--- /dev/null
+++ b/tests/acceptance/artificial/vhdl/tb_fail_on_warning_from_python.vhd
@@ -0,0 +1,23 @@
+-- This Source Code Form is subject to the terms of the Mozilla Public
+-- License, v. 2.0. If a copy of the MPL was not distributed with this file,
+-- You can obtain one at http://mozilla.org/MPL/2.0/.
+--
+-- Copyright (c) 2014-2024, Lars Asplund lars.anders.asplund@gmail.com
+
+library vunit_lib;
+context vunit_lib.vunit_context;
+
+entity tb_fail_on_warning_from_python is
+  generic (runner_cfg : string);
+end entity;
+
+architecture vunit_test_bench of tb_fail_on_warning_from_python is
+begin
+  test_runner : process
+  begin
+    test_runner_setup(runner, runner_cfg);
+    report "A warning" severity warning;
+    test_runner_cleanup(runner);
+    wait;
+  end process;
+end architecture;
diff --git a/tests/acceptance/artificial/vhdl/tb_same_sim_from_python_all_pass.vhd b/tests/acceptance/artificial/vhdl/tb_same_sim_from_python_all_pass.vhd
new file mode 100644
index 000000000..8b9282571
--- /dev/null
+++ b/tests/acceptance/artificial/vhdl/tb_same_sim_from_python_all_pass.vhd
@@ -0,0 +1,51 @@
+-- This Source Code Form is subject to the terms of the Mozilla Public
+-- License, v. 2.0. If a copy of the MPL was not distributed with this file,
+-- You can obtain one at http://mozilla.org/MPL/2.0/.
+--
+-- Copyright (c) 2014-2024, Lars Asplund lars.anders.asplund@gmail.com
+
+use std.textio.all;
+
+library vunit_lib;
+context vunit_lib.vunit_context;
+
+entity tb_same_sim_from_python_all_pass is
+  generic (
+    output_path : string;
+    runner_cfg : string);
+end entity;
+
+architecture vunit_test_bench of tb_same_sim_from_python_all_pass is
+begin
+  test_runner : process
+    file fptr : text;
+    variable counter : integer := 1;
+  begin
+    test_runner_setup(runner, runner_cfg);
+
+    while test_suite loop
+      if run("Test 1") then
+        wait for 10 ns;
+        report "Test 1";
+        assert counter = 1;
+        counter := counter + 1;
+      elsif run("Test 2") then
+        wait for 10 ns;
+        report "Test 2";
+        assert counter = 2;
+        counter := counter + 1;
+      elsif run("Test 3") then
+        wait for 10 ns;
+        report "Test 3";
+        assert counter = 3;
+        counter := counter + 1;
+        file_open(fptr, output_path & "post_check.txt", write_mode);
+        write(fptr, string'("Test 3 was here"));
+        file_close(fptr);
+      end if;
+    end loop;
+
+    test_runner_cleanup(runner);
+    wait;
+  end process;
+end architecture;
diff --git a/tests/acceptance/artificial/vhdl/tb_same_sim_from_python_some_fail.vhd b/tests/acceptance/artificial/vhdl/tb_same_sim_from_python_some_fail.vhd
new file mode 100644
index 000000000..41da715a5
--- /dev/null
+++ b/tests/acceptance/artificial/vhdl/tb_same_sim_from_python_some_fail.vhd
@@ -0,0 +1,38 @@
+-- This Source Code Form is subject to the terms of the Mozilla Public
+-- License, v. 2.0. If a copy of the MPL was not distributed with this file,
+-- You can obtain one at http://mozilla.org/MPL/2.0/.
+--
+-- Copyright (c) 2014-2024, Lars Asplund lars.anders.asplund@gmail.com
+
+library vunit_lib;
+context vunit_lib.vunit_context;
+
+entity tb_same_sim_from_python_some_fail is
+  generic (
+    runner_cfg : string);
+end entity;
+
+architecture vunit_test_bench of tb_same_sim_from_python_some_fail is
+begin
+  test_runner : process
+  begin
+    test_runner_setup(runner, runner_cfg);
+
+    while test_suite loop
+      if run("Test 1") then
+        wait for 10 ns;
+        report "Test 1";
+      elsif run("Test 2") then
+        wait for 10 ns;
+        report "Test 2";
+        assert false;
+      elsif run("Test 3") then
+        wait for 10 ns;
+        report "Test 3";
+      end if;
+    end loop;
+
+    test_runner_cleanup(runner);
+    wait;
+  end process;
+end architecture;
diff --git a/tests/acceptance/test_artificial.py b/tests/acceptance/test_artificial.py
index 546f68759..2947ec4db 100644
--- a/tests/acceptance/test_artificial.py
+++ b/tests/acceptance/test_artificial.py
@@ -129,6 +129,7 @@ def test_artificial_verilog(self):
                 ("passed", "lib.tb_magic_paths.Test magic paths are correct"),
                 ("passed", "lib.tb_with_define.test 1"),
                 ("failed", "lib.tb_fail_on_warning.fail"),
+                ("failed", "lib.tb_fail_on_warning_from_python.fail"),
                 ("failed", "lib.tb_fail_on_fatal_and_early_finish.fatal0"),
                 ("failed", "lib.tb_fail_on_fatal_and_early_finish.fatal1"),
                 ("failed", "lib.tb_fail_on_fatal_and_early_finish.finish0"),
@@ -144,6 +145,12 @@ def test_artificial_verilog(self):
                 ("passed", "lib.tb_same_sim_some_fail.Test 1"),
                 ("failed", "lib.tb_same_sim_some_fail.Test 2"),
                 ("skipped", "lib.tb_same_sim_some_fail.Test 3"),
+                ("passed", "lib.tb_same_sim_from_python_all_pass.cfg.Test 1"),
+                ("passed", "lib.tb_same_sim_from_python_all_pass.cfg.Test 2"),
+                ("passed", "lib.tb_same_sim_from_python_all_pass.cfg.Test 3"),
+                ("passed", "lib.tb_same_sim_from_python_some_fail.Test 1"),
+                ("failed", "lib.tb_same_sim_from_python_some_fail.Test 2"),
+                ("skipped", "lib.tb_same_sim_from_python_some_fail.Test 3"),
                 ("passed", "lib.tb_with_runner.pass"),
                 ("failed", "lib.tb_with_runner.fail"),
             ],
@@ -183,7 +190,9 @@ def test_exit_0_flag(self):
     ("failed", "lib.tb_fail.all"),
     ("passed", "lib.tb_infinite_events.all"),
     ("failed", "lib.tb_fail_on_warning.all"),
-    ("passed", "lib.tb_no_fail_on_warning.all"),
+    ("failed", "lib.tb_fail_on_warning_from_python.all"),
+    ("passed", "lib.tb_no_fail_on_warning.cfg1"),
+    ("passed", "lib.tb_no_fail_on_warning.cfg2"),
     ("passed", "lib.tb_with_vhdl_runner.pass"),
     ("passed", "lib.tb_with_vhdl_runner.Test with spaces"),
     ("failed", "lib.tb_with_vhdl_runner.fail"),
@@ -197,6 +206,12 @@ def test_exit_0_flag(self):
     ("passed", "lib.tb_same_sim_some_fail.Test 1"),
     ("failed", "lib.tb_same_sim_some_fail.Test 2"),
     ("skipped", "lib.tb_same_sim_some_fail.Test 3"),
+    ("passed", "lib.tb_same_sim_from_python_all_pass.cfg.Test 1"),
+    ("passed", "lib.tb_same_sim_from_python_all_pass.cfg.Test 2"),
+    ("passed", "lib.tb_same_sim_from_python_all_pass.cfg.Test 3"),
+    ("passed", "lib.tb_same_sim_from_python_some_fail.Test 1"),
+    ("failed", "lib.tb_same_sim_from_python_some_fail.Test 2"),
+    ("skipped", "lib.tb_same_sim_from_python_some_fail.Test 3"),
     ("passed", "lib.tb_with_checks.Test passing check"),
     ("failed", "lib.tb_with_checks.Test failing check"),
     ("failed", "lib.tb_with_checks.Test non-stopping failing check"),
diff --git a/tests/unit/test_test_bench.py b/tests/unit/test_test_bench.py
index 19d7071a5..994fd2481 100644
--- a/tests/unit/test_test_bench.py
+++ b/tests/unit/test_test_bench.py
@@ -273,6 +273,24 @@ def test_that_run_in_same_simulation_attribute_works(self, tempdir):
         tests = self.create_tests(test_bench)
         self.assert_has_tests(tests, [("lib.tb_entity", ("lib.tb_entity.Test_1", "lib.tb_entity.Test_2"))])
 
+    @with_tempdir
+    def test_that_run_in_same_simulation_attribute_from_python_works(self, tempdir):
+        for value in [None, True]:
+            design_unit = Entity(
+                "tb_entity",
+                file_name=str(Path(tempdir) / "file.vhd"),
+                contents="""\
+if run("Test_1")
+if run("Test_2")
+--if run("Test_3")
+""",
+            )
+            design_unit.generic_names = ["runner_cfg"]
+            test_bench = TestBench(design_unit)
+            test_bench.set_attribute("run_all_in_same_sim", value)
+            tests = self.create_tests(test_bench)
+            self.assert_has_tests(tests, [("lib.tb_entity", ("lib.tb_entity.Test_1", "lib.tb_entity.Test_2"))])
+
     @with_tempdir
     def test_add_config(self, tempdir):
         design_unit = Entity("tb_entity", file_name=str(Path(tempdir) / "file.vhd"))
@@ -451,6 +469,22 @@ def test_error_on_global_attributes_on_tests(self, tempdir):
         else:
             assert False, "RuntimeError not raised"
 
+    @with_tempdir
+    def test_error_on_global_attributes_from_python_on_tests(self, tempdir):
+        design_unit = Entity(
+            "tb_entity",
+            file_name=str(Path(tempdir) / "file.vhd"),
+            contents="""\
+if run("Test 1")
+if run("Test 2")
+""",
+        )
+        design_unit.generic_names = ["runner_cfg"]
+
+        test_bench = TestBench(design_unit)
+        test = test_bench.get_test_case("Test 1")
+        self.assertRaises(AttributeException, test.set_attribute, "run_all_in_same_sim", True)    
+
     @with_tempdir
     def test_test_information(self, tempdir):
         file_name = str(Path(tempdir) / "file.vhd")
diff --git a/vunit/test/bench.py b/vunit/test/bench.py
index 874ff40a8..e90a031a5 100644
--- a/vunit/test/bench.py
+++ b/vunit/test/bench.py
@@ -184,13 +184,7 @@ def parse(content):
                         f"{file_name!s} line {attr.location.lineno:d}"
                     )
 
-        attribute_names = [attr.name for attr in attributes]
-
         default_config = Configuration(DEFAULT_NAME, self.design_unit)
-
-        if "fail_on_warning" in attribute_names:
-            default_config.set_sim_option("vhdl_assert_stop_level", "warning")
-
         self._configs = OrderedDict({default_config.name: default_config})
 
         explicit_tests = [test for test in tests if test.is_explicit]
@@ -203,12 +197,68 @@ def parse(content):
             assert len(tests) == 1
             self._implicit_test = tests[0]
 
-        self._individual_tests = "run_all_in_same_sim" not in attribute_names and len(explicit_tests) > 0
+        self._individual_tests = len(explicit_tests) > 0
         self._test_cases = [
             TestConfigurationVisitor(test, self.design_unit, self._individual_tests, default_config.copy())
             for test in explicit_tests
         ]
 
+        # This must be done after self._test_cases have been created such that run_all_in_same_sim can disable them
+        for attr in attributes:
+            self.set_attribute(attr.name, attr.value)
+
+    def set_attribute(self, name, value):
+        """
+        Set attributes except fail_on_warning and run_all_in_same_sim which have special meanings
+        """
+        if name == "fail_on_warning":
+            self.set_sim_option("vhdl_assert_stop_level", "warning" if value or value is None else "error")
+            return
+
+        if name == "run_all_in_same_sim":
+            run_all_in_same_sim = value or value is None
+            for test_case in self._test_cases:
+                test_case.enable_configuration = not run_all_in_same_sim
+
+            self._individual_tests = not run_all_in_same_sim and len(self._test_cases) > 0
+
+            return
+
+        super().set_attribute(name, value)
+
+    def add_config(  # pylint: disable=too-many-arguments
+        self,
+        name,
+        generics=None,
+        pre_config=None,
+        post_check=None,
+        sim_options=None,
+        attributes=None,
+        vhdl_configuration_name=None,
+    ):
+        """
+        Add a configuration copying unset fields from the default configuration.
+
+        fail_on_warning and run_all_in_same_sim have special meaning and are handled
+        separately.
+        """
+        if attributes:
+            if "fail_on_warning" in attributes:
+                value = attributes["fail_on_warning"]
+                self.set_sim_option("vhdl_assert_stop_level", "warning" if value or value is None else "error")
+                del attributes["fail_on_warning"]
+
+            if "run_all_in_same_sim" in attributes:
+                value = attributes["run_all_in_same_sim"]
+                run_all_in_same_sim = value or value is None
+                for test_case in self._test_cases:
+                    test_case.enable_configuration = not run_all_in_same_sim
+
+                self._individual_tests = not run_all_in_same_sim and len(self._test_cases) > 0
+                del attributes["run_all_in_same_sim"]
+
+        super().add_config(name, generics, pre_config, post_check, sim_options, attributes, vhdl_configuration_name)
+
 
 class FileLocation(object):
     """
@@ -315,7 +365,7 @@ def __init__(self, test, design_unit, enable_configuration, default_config):
         ConfigurationVisitor.__init__(self, design_unit)
         self._test = test
         assert test.is_explicit
-        self._enable_configuration = enable_configuration
+        self.enable_configuration = enable_configuration
         self._configs = OrderedDict({default_config.name: default_config})
 
     @property
@@ -334,7 +384,7 @@ def get_default_config(self):
         return self._configs[DEFAULT_NAME]
 
     def _check_enabled(self):
-        if not self._enable_configuration:
+        if not self.enable_configuration:
             raise RuntimeError("Individual test configuration is not possible with run_all_in_same_sim")
 
     def get_configuration_dicts(self):  # pylint: disable=arguments-differ