diff --git a/tests/training/index.js b/tests/training/index.js index 1c1cd3fe5..93818342f 100644 --- a/tests/training/index.js +++ b/tests/training/index.js @@ -18,15 +18,10 @@ const assert = require('assert'); //const Tp = require('thingpedia'); const db = require('../../util/db'); +const sleep = require('../../util/sleep'); const trainingJobModel = require('../../model/training_job'); const TrainingServer = require('../../util/training_server'); -async function delay(ms) { - return new Promise((resolve, reject) => { - setTimeout(resolve, ms); - }); -} - async function waitUntilAllJobsDone() { for (;;) { const row = await db.withClient((dbClient) => { @@ -37,7 +32,7 @@ async function waitUntilAllJobsDone() { if (row.cnt === 0) break; - await delay(10000); + await sleep(10000); } const failed = await db.withClient((dbClient) => { @@ -74,7 +69,7 @@ async function testBasic() { // issue a basic train command await server.queue('en', null, 'train'); - await delay(1000); + await sleep(1000); const queue = await db.withClient((dbClient) => server.getJobQueue(dbClient)); //console.log(queue); @@ -185,7 +180,7 @@ async function testForDevice() { // issue a train command for a device that is not approved await server.queue('en', ['org.thingpedia.builtin.test.adminonly'], 'train'); - await delay(1000); + await sleep(1000); const queue = await db.withClient((dbClient) => server.getJobQueue(dbClient)); //console.log(queue); diff --git a/tests/unit/test_lock.js b/tests/unit/test_lock.js index 69a9aebe1..5d38526e9 100644 --- a/tests/unit/test_lock.js +++ b/tests/unit/test_lock.js @@ -12,12 +12,7 @@ const assert = require('assert'); const Lock = require('../../util/lock'); - -function delay(timeout) { - return new Promise((resolve, reject) => { - setTimeout(resolve, timeout); - }); -} +const sleep = require('../../util/sleep'); async function withTimeout(promise, timeout = 30000) { await Promise.race([ @@ -50,20 +45,20 @@ async function testInterleave() { async function thread1() { const release1 = await lock.acquire(); output.push(1); - await delay(1000); + await sleep(1000); output.push(2); release1(); - await delay(1000); + await sleep(1000); const release2 = await lock.acquire(); output.push(5); release2(); } async function thread2() { - await delay(500); + await sleep(500); const release1 = await lock.acquire(); output.push(3); - await delay(500); + await sleep(500); output.push(4); release1(); } @@ -82,22 +77,22 @@ async function testQueue() { async function thread1() { const release1 = await lock.acquire(); output.push(1); - await delay(5000); + await sleep(5000); output.push(2); release1(); } async function thread2() { - await delay(500); + await sleep(500); const release1 = await lock.acquire(); output.push(3); - await delay(500); + await sleep(500); output.push(4); release1(); } async function thread3() { - await delay(1500); + await sleep(1500); const release1 = await lock.acquire(); output.push(5); output.push(6); diff --git a/tests/website/test_admin.js b/tests/website/test_admin.js index cdb6ad294..009aab7d0 100644 --- a/tests/website/test_admin.js +++ b/tests/website/test_admin.js @@ -15,6 +15,7 @@ const { assertHttpError, assertRedirect, assertLoginRequired, assertBlocked, ses const { login, startSession } = require('../login'); const db = require('../../util/db'); +const sleep = require('../../util/sleep'); const EngineManagerClient = require('../../almond/enginemanagerclient'); const Config = require('../../config'); @@ -44,12 +45,6 @@ async function testAdminUsers(root, bob, nobody) { assert(rootUserPage2.indexOf('root@localhost') >= 0); } -function delay(ms) { - return new Promise((resolve, reject) => { - setTimeout(resolve, ms); - }); -} - async function testAdminKillRestart(root, bob, nobody) { const emc = EngineManagerClient.get(); assert (await emc.isRunning(1)); // root @@ -71,7 +66,7 @@ async function testAdminKillRestart(root, bob, nobody) { assert (!await emc.isRunning(5)); // emma -or- alexa_user // the shared processes will be restarted in 5s - await delay(10000); + await sleep(10000); await assertLoginRequired(sessionRequest('/admin/users/start/1', 'POST', '', nobody)); await assertRedirect(sessionRequest('/admin/users/start/1', 'POST', '', root, { followRedirects: false }), '/admin/users/search?q=1'); diff --git a/training/tasks/train.js b/training/tasks/train.js index a5b187221..cc80dd5a2 100644 --- a/training/tasks/train.js +++ b/training/tasks/train.js @@ -16,9 +16,23 @@ const fs = require('fs'); const Genie = require('genie-toolkit'); const AbstractFS = require('../../util/abstract_fs'); +const sleep = require('../../util/sleep'); const Config = require('../../config'); module.exports = async function main(task, argv) { + // on kubernetes, we might encounter a race when this pod is scheduled on a + // newly started node, where the pod is started before the kube2iam daemonset + // is ready + // in turn, this means we don't have the right credentials to access s3, and die + // + // we work around that problem with an artificial 1 minute delay when training + // we only do it for training, because other jobs are likely scheduled on existing + // general-purpose nodes (where kube2iam is already active) + // we also only do it for the kubernetes backend, because the local backend doesn't + // have that problem, and we don't want our CI to become longer + if (Config.TRAINING_TASK_BACKEND === 'kubernetes') + await sleep(60000); + const jobdir = await AbstractFS.download(task.jobDir + '/'); const datadir = path.resolve(jobdir, 'dataset'); const workdir = path.resolve(jobdir, 'workdir'); diff --git a/util/sleep.js b/util/sleep.js new file mode 100644 index 000000000..b8b2ae42f --- /dev/null +++ b/util/sleep.js @@ -0,0 +1,16 @@ +// -*- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 -*- +// +// This file is part of ThingEngine +// +// Copyright 2019 The Board of Trustees of the Leland Stanford Junior University +// +// Author: Giovanni Campagna +// +// See COPYING for details +"use strict"; + +module.exports = async function sleep(ms) { + return new Promise((resolve, reject) => { + setTimeout(resolve, ms); + }); +};