Shuffle tests

Makefile

@@ -7,6 +7,7 @@ LUACOV_REPORT := $(PROJECT_DIR)/luacov.report.out
 LUACOV_STATS := $(PROJECT_DIR)/luacov.stats.out
 
 SHELL := $(shell which bash) # Required for brace expansion used in a clean target.
+SEED ?= $(shell /bin/bash -c "echo $$RANDOM")
 
 CLEANUP_FILES  = tarantool.log
 CLEANUP_FILES += *.index
@@ -32,7 +33,7 @@ luacheck:
 
 .PHONY: test
 test:
-	luatest -v --coverage
+	luatest -v --coverage --shuffle all:${SEED}
 	rm -rf ${CLEANUP_FILES}
 	INDEX_TYPE='TREE' SPACE_TYPE='vinyl' ./test.lua
 	rm -rf ${CLEANUP_FILES}
@@ -53,7 +54,7 @@ coveralls: $(LUACOV_STATS)
 	luacov-coveralls --include ^expirationd --verbose --repo-token ${GITHUB_TOKEN}
 
 deps:
-	tarantoolctl rocks install luatest 0.5.6
+	tarantoolctl rocks install luatest 0.5.7
 	tarantoolctl rocks install luacheck 0.26.0
 	tarantoolctl rocks install luacov 0.13.0-1
 	tarantoolctl rocks install ldoc --server=https://tarantool.github.io/LDoc/

README.md

@@ -134,3 +134,14 @@ expirationd.start(job_name, space.id, is_expired, {
 $ make deps-full
 $ make test
 ```
+
+Regression tests running in continuous integration that uses luatest are
+executed in shuffle mode. It means that every time order of tests is
+pseudorandom with predefined seed. If tests in CI are failed it is better to
+reproduce these failures with the same seed:
+
+```sh
+$ make SEED=1334 test
+luatest -v --coverage --shuffle all:1334
+...
+```

test/unit/cfg_test.lua

@@ -6,6 +6,14 @@ local g = t.group('expirationd_cfg')
 local metrics_required_msg = "metrics >= 0.11.0 is not installed"
 local metrics_not_required_msg = "metrics >= 0.11.0 is installed"
 
+g.before_all(function()
+    g.default_cfg = { metrics = expirationd.cfg.metrics }
+end)
+
+g.after_each(function()
+    expirationd.cfg(g.default_cfg)
+end)
+
 function g.test_cfg_default_if_installed()
     t.skip_if(not helpers.is_metrics_supported(), metrics_required_msg)
     t.assert_equals(expirationd.cfg.metrics, true)

test/unit/metrics_test.lua

@@ -3,16 +3,26 @@ local t = require("luatest")
 local helpers = require("test.helper")
 local g = t.group('expirationd_metrics')
 
+g.before_all(function()
+    g.default_cfg = { metrics = expirationd.cfg.metrics }
+end)
+
 g.before_each(function()
     t.skip_if(not helpers.is_metrics_supported(),
               "metrics >= 0.11.0 is not installed")
     g.space = helpers.create_space_with_tree_index('memtx')
+    -- kill live tasks (it can still live after failed tests)
+    for _, t in ipairs(expirationd.tasks()) do
+        expirationd.kill(t)
+    end
+    -- disable and clean metrics by default
+    expirationd.cfg({metrics = false})
+    require('metrics').clear()
 end)
 
 local task = nil
 g.after_each(function(g)
-    expirationd.cfg({metrics = false})
-    require('metrics').clear()
+    expirationd.cfg(g.default_cfg)
     g.space:drop()
     if task ~= nil then
         task:kill()

test/unit/task_stop_test.lua

@@ -37,4 +37,5 @@ function g.test_cancel_on_pcall(cg)
     helpers.retrying({timeout = 5}, function()
         t.assert_equals(f:status(), "dead")
     end)
+    task:kill()
 end