From e1bafb4385d5a3a06a780df54ab7c524d640aa06 Mon Sep 17 00:00:00 2001 From: Oleg Jukovec Date: Mon, 11 Jul 2022 10:01:29 +0300 Subject: [PATCH 1/5] test: reset expirationd.cfg to default Tests that change expirationd.cfg did not perform a teardown correctly. The tests broke other tests after shuffling. The patch resets expirationd to the default configuration after execution of such tests. Related to #118 --- test/unit/cfg_test.lua | 8 ++++++++ test/unit/metrics_test.lua | 7 ++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/test/unit/cfg_test.lua b/test/unit/cfg_test.lua index b2ea044..0ae362a 100644 --- a/test/unit/cfg_test.lua +++ b/test/unit/cfg_test.lua @@ -6,6 +6,14 @@ local g = t.group('expirationd_cfg') local metrics_required_msg = "metrics >= 0.11.0 is not installed" local metrics_not_required_msg = "metrics >= 0.11.0 is installed" +g.before_all(function() + g.default_cfg = { metrics = expirationd.cfg.metrics } +end) + +g.after_each(function() + expirationd.cfg(g.default_cfg) +end) + function g.test_cfg_default_if_installed() t.skip_if(not helpers.is_metrics_supported(), metrics_required_msg) t.assert_equals(expirationd.cfg.metrics, true) diff --git a/test/unit/metrics_test.lua b/test/unit/metrics_test.lua index 3c815f6..afd727b 100644 --- a/test/unit/metrics_test.lua +++ b/test/unit/metrics_test.lua @@ -3,6 +3,10 @@ local t = require("luatest") local helpers = require("test.helper") local g = t.group('expirationd_metrics') +g.before_all(function() + g.default_cfg = { metrics = expirationd.cfg.metrics } +end) + g.before_each(function() t.skip_if(not helpers.is_metrics_supported(), "metrics >= 0.11.0 is not installed") @@ -11,8 +15,9 @@ end) local task = nil g.after_each(function(g) - expirationd.cfg({metrics = false}) + expirationd.cfg({metrics = false}) -- reset metrics stats require('metrics').clear() + expirationd.cfg(g.default_cfg) g.space:drop() if task ~= nil then task:kill() From 4eb01e4c04f34d730393476939d83e9ef3186297 Mon Sep 17 00:00:00 2001 From: Oleg Jukovec Date: Mon, 11 Jul 2022 10:49:54 +0300 Subject: [PATCH 2/5] test: kill task after test_cancel_on_pcall The test just stop the task. It must to kill the task to perform a cleanup. Related to #118 --- test/unit/task_stop_test.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/test/unit/task_stop_test.lua b/test/unit/task_stop_test.lua index af7a981..13b2147 100644 --- a/test/unit/task_stop_test.lua +++ b/test/unit/task_stop_test.lua @@ -37,4 +37,5 @@ function g.test_cancel_on_pcall(cg) helpers.retrying({timeout = 5}, function() t.assert_equals(f:status(), "dead") end) + task:kill() end From d2684969b069f0a18f99221ad871e9680950e65a Mon Sep 17 00:00:00 2001 From: Oleg Jukovec Date: Mon, 11 Jul 2022 10:23:39 +0300 Subject: [PATCH 3/5] test: clean metrics before execution of metrics tests Metrics is now enabled by default. We need to clear it state before running the appropriate tests to make sure there is no data from previous tests. Related to #118 --- test/unit/metrics_test.lua | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/test/unit/metrics_test.lua b/test/unit/metrics_test.lua index afd727b..0c715b6 100644 --- a/test/unit/metrics_test.lua +++ b/test/unit/metrics_test.lua @@ -11,12 +11,17 @@ g.before_each(function() t.skip_if(not helpers.is_metrics_supported(), "metrics >= 0.11.0 is not installed") g.space = helpers.create_space_with_tree_index('memtx') + -- kill live tasks (it can still live after failed tests) + for _, t in ipairs(expirationd.tasks()) do + expirationd.kill(t) + end + -- disable and clean metrics by default + expirationd.cfg({metrics = false}) + require('metrics').clear() end) local task = nil g.after_each(function(g) - expirationd.cfg({metrics = false}) -- reset metrics stats - require('metrics').clear() expirationd.cfg(g.default_cfg) g.space:drop() if task ~= nil then From 85146431201e295b7e9de8906a21407baa88a97e Mon Sep 17 00:00:00 2001 From: Sergey Bronnikov Date: Sat, 9 Jul 2022 11:08:13 +0300 Subject: [PATCH 4/5] test: bump luatest version to 0.5.7 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f6121d5..f5dbf36 100644 --- a/Makefile +++ b/Makefile @@ -53,7 +53,7 @@ coveralls: $(LUACOV_STATS) luacov-coveralls --include ^expirationd --verbose --repo-token ${GITHUB_TOKEN} deps: - tarantoolctl rocks install luatest 0.5.6 + tarantoolctl rocks install luatest 0.5.7 tarantoolctl rocks install luacheck 0.26.0 tarantoolctl rocks install luacov 0.13.0-1 tarantoolctl rocks install ldoc --server=https://tarantool.github.io/LDoc/ From f1ad823590aee72b65c05e10a4a1e61ce7114af2 Mon Sep 17 00:00:00 2001 From: Sergey Bronnikov Date: Sat, 9 Jul 2022 11:08:41 +0300 Subject: [PATCH 5/5] test: shuffle tests luatest has an option --shuffle that allows to randomise the order of tests. This can be useful to detect a test that passes just because it happens to run after an unrelated test that leaves the system in a favourable state. Patch enable tests shuffling with predefined random seed specified by $RANDOM. Seed is useful for reproducing a problems related to the specific test order. --- Makefile | 3 ++- README.md | 11 +++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f5dbf36..5c8d511 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,7 @@ LUACOV_REPORT := $(PROJECT_DIR)/luacov.report.out LUACOV_STATS := $(PROJECT_DIR)/luacov.stats.out SHELL := $(shell which bash) # Required for brace expansion used in a clean target. +SEED ?= $(shell /bin/bash -c "echo $$RANDOM") CLEANUP_FILES = tarantool.log CLEANUP_FILES += *.index @@ -32,7 +33,7 @@ luacheck: .PHONY: test test: - luatest -v --coverage + luatest -v --coverage --shuffle all:${SEED} rm -rf ${CLEANUP_FILES} INDEX_TYPE='TREE' SPACE_TYPE='vinyl' ./test.lua rm -rf ${CLEANUP_FILES} diff --git a/README.md b/README.md index d7346e5..d1bbcd6 100644 --- a/README.md +++ b/README.md @@ -134,3 +134,14 @@ expirationd.start(job_name, space.id, is_expired, { $ make deps-full $ make test ``` + +Regression tests running in continuous integration that uses luatest are +executed in shuffle mode. It means that every time order of tests is +pseudorandom with predefined seed. If tests in CI are failed it is better to +reproduce these failures with the same seed: + +```sh +$ make SEED=1334 test +luatest -v --coverage --shuffle all:1334 +... +```