feat(metrics): Collect healthcheck results

CHANGELOG.md

@@ -1,5 +1,8 @@
 # Changelog
 
+## Unreleased
+ - Collect healthcheck results.
+
 ## 1.23.1
  - Support CPU profiling in Node.js >=7.0.0
 

package.json

@@ -50,6 +50,8 @@
     "v8-profiler": "5.7.0"
   },
   "devDependencies": {
+    "admin": "^1.4.0",
+    "admin-plugin-healthcheck": "^1.1.0",
     "bluebird": "3.4.6",
     "body-parser": "1.15.2",
     "chai": "3.3.0",
@@ -62,6 +64,7 @@
     "lodash": "4.15.0",
     "mocha": "2.3.3",
     "mongodb": "2.2.9",
+    "morgan": "^1.8.1",
     "proxyquire": "1.7.3",
     "request": "2.74.0",
     "request-promise": "4.1.1",

src/index.js

@@ -15,6 +15,7 @@ module.exports = exports = function start(config) {
   require('./agent/opts').init(config);
   require('./actions/profiling/cpu').init(config);
   require('./tracing').init(config);
+  require('./states/agentready').init(config);
 
   var logger = log.getLogger('index');
 

src/metrics/healthchecks.js

@@ -0,0 +1,58 @@
+'use strict';
+
+var requireHook = require('../util/requireHook');
+var logger = require('../logger').getLogger('metrics/healthchecks');
+
+var timeBetweenHealthcheckCalls;
+var healthy = 1;
+var unhealthy = 0;
+
+var adminPluginHealthcheck;
+var timeoutHandle;
+
+exports.payloadPrefix = 'healthchecks';
+exports.currentPayload = {};
+
+requireHook.on('admin-plugin-healthcheck', function onAdminPluginHealthcheckLoaded(_adminPluginHealthcheck) {
+  adminPluginHealthcheck = _adminPluginHealthcheck;
+});
+
+exports.activate = function(config) {
+  timeBetweenHealthcheckCalls = config.timeBetweenHealthcheckCalls || 3000;
+
+  if (adminPluginHealthcheck != null) {
+    gatherHealthcheckResults();
+  }
+};
+
+function gatherHealthcheckResults() {
+  adminPluginHealthcheck.getHealthCheckResult()
+    .then(function onHealthcheckResults(adminHealthcheckResults) {
+      var results = {};
+      var previousResults = exports.currentPayload;
+
+      for (var key in adminHealthcheckResults) {
+        if (adminHealthcheckResults.hasOwnProperty(key)) {
+          var result = adminHealthcheckResults[key];
+          var checkHealthy = result.healthy ? healthy : unhealthy;
+          var changed = previousResults[key] == null || previousResults[key].healthy !== checkHealthy;
+          results[key] = {
+            healthy: checkHealthy,
+            since: changed ? new Date().getTime() : previousResults[key].since
+          };
+        }
+      }
+
+      exports.currentPayload = results;
+      timeoutHandle = setTimeout(gatherHealthcheckResults, timeBetweenHealthcheckCalls);
+    })
+    .catch(function onHealthcheckResultFailure(err) {
+      exports.currentPayload = {};
+      logger.warn('Unexpected error while getting healthcheck results', err);
+      timeoutHandle = setTimeout(gatherHealthcheckResults, timeBetweenHealthcheckCalls);
+    });
+}
+
+exports.deactivate = function() {
+  clearTimeout(timeoutHandle);
+};

src/states/agentready.js

@@ -19,13 +19,14 @@ var modules = fs.readdirSync(metricsBaseDir)
     return require(path.join(metricsBaseDir, moduleName));
   });
 
+var config;
 var resendFullDataEveryXTransmissions = 300; /* about every 5 minutes */
 
 var transmissionsSinceLastFullDataEmit = 0;
 var previousTransmittedValue = undefined;
 
 
-module.exports = {
+module.exports = exports = {
   enter: function(ctx) {
     transmissionsSinceLastFullDataEmit = 0;
 
@@ -72,10 +73,13 @@ module.exports = {
   }
 };
 
+exports.init = function init(_config) {
+  config = _config;
+};
 
 function enableAllSensors() {
   modules.forEach(function(mod) {
-    mod.activate();
+    mod.activate(config);
   });
 }
 

test/apps/agentStub.js

@@ -1,9 +1,11 @@
 /* eslint-disable */
 
-var express = require('express');
 var bodyParser = require('body-parser');
+var express = require('express');
+var morgan = require('morgan');
 var app = express();
 
+var logPrefix = 'Agent Stub (' + process.pid + '):\t';
 var discoveries = {};
 var requests = {};
 var retrievedData = {
@@ -12,6 +14,10 @@ var retrievedData = {
   responses: []
 };
 
+if (process.env.WITH_STDOUT) {
+  app.use(morgan(logPrefix + ':method :url :status'));
+}
+
 app.use(bodyParser.json());
 
 
@@ -103,7 +109,8 @@ app.delete('/retrievedData', function(req, res) {
   log('Clearing retrieved data');
   retrievedData = {
     runtime: [],
-    traces: []
+    traces: [],
+    responses: []
   };
   res.sendStatus(200);
 });
@@ -135,6 +142,6 @@ app.listen(process.env.AGENT_PORT, function() {
 
 function log() {
   var args = Array.prototype.slice.call(arguments);
-  args[0] = 'Agent Stub (' + process.pid + '):\t' + args[0];
+  args[0] = logPrefix + args[0];
   console.log.apply(console, args);
 }

test/apps/express.js

@@ -8,19 +8,53 @@ require('../../')({
   agentPort: process.env.AGENT_PORT,
   level: 'info',
   tracing: {
+    timeBetweenHealthcheckCalls: 1000,
     enabled: process.env.TRACING_ENABLED === 'true',
     forceTransmissionStartingAt: 1,
     stackTraceLength: process.env.STACK_TRACE_LENGTH != null ? parseInt(process.env.STACK_TRACE_LENGTH, 10) : 10
   }
 });
 
 var express = require('express');
+var semver = require('semver');
 var app = express();
 
+var healthcheckFunction = function() {
+  return 'OK!';
+};
+
+if (semver.satisfies(process.versions.node, '>=6.0.0')) {
+  require('admin').configure({
+    plugins: [
+      require('admin-plugin-healthcheck')({
+        checks: {
+          configurable: function() {
+            healthcheckFunction();
+          }
+        }
+      })
+    ]
+  });
+}
+
 app.get('/return-instana-trace-id', function(req, res) {
   res.send(req.get('x-instana-t'));
 });
 
+app.post('/admin/set-to-unhealthy', function(req, res) {
+  healthcheckFunction = function() {
+    throw new Error('Explicit healthcheck failure');
+  };
+  res.send('OK');
+});
+
+app.post('/admin/set-to-healthy', function(req, res) {
+  healthcheckFunction = function() {
+    return 'OK';
+  };
+  res.send('OK');
+});
+
 app.use(function(req, res) {
   log(req.method, req.url);
   var delay = parseInt(req.query.delay || 0, 10);

test/apps/expressControls.js

@@ -76,3 +76,17 @@ exports.sendRequest = function(opts) {
     throw reason;
   });
 };
+
+exports.setHealthy = function() {
+  return request({
+    method: 'POST',
+    url: 'http://127.0.0.1:' + appPort + '/admin/set-to-healthy'
+  });
+};
+
+exports.setUnhealthy = function() {
+  return request({
+    method: 'POST',
+    url: 'http://127.0.0.1:' + appPort + '/admin/set-to-unhealthy'
+  });
+};

test/apps/expressMongodb.js

@@ -14,11 +14,17 @@ require('../../')({
 var MongoClient = require('mongodb').MongoClient;
 var bodyParser = require('body-parser');
 var express = require('express');
+var morgan = require('morgan');
 var assert = require('assert');
 
 var app = express();
 var db;
 var collection;
+var logPrefix = 'Express / MongoDB App (' + process.pid + '):\t';
+
+if (process.env.WITH_STDOUT) {
+  app.use(morgan(logPrefix + ':method :url :status'));
+}
 
 app.use(bodyParser.json());
 
@@ -77,6 +83,6 @@ app.listen(process.env.APP_PORT, function() {
 
 function log() {
   var args = Array.prototype.slice.call(arguments);
-  args[0] = 'Express / MongoDB App (' + process.pid + '):\t' + args[0];
+  args[0] = logPrefix + args[0];
   console.log.apply(console, args);
 }

test/metrics/healthchecks_test.js

@@ -0,0 +1,53 @@
+'use strict';
+
+var expect = require('chai').expect;
+var semver = require('semver');
+
+var agentStubControls = require('../apps/agentStubControls');
+var expressControls = require('../apps/expressControls');
+var config = require('../config');
+var utils = require('../utils');
+
+describe('metrics/healthchecks', function() {
+  // admin uses JavaScript language features which aren't available in all
+  // Node.js versions
+  if (!semver.satisfies(process.versions.node, '>=6.0.0')) {
+    return;
+  }
+  this.timeout(config.getTestTimeout());
+
+  var start = new Date().getTime();
+
+  agentStubControls.registerTestHooks();
+  expressControls.registerTestHooks({
+    enableTracing: false
+  });
+
+  beforeEach(function() {
+    return agentStubControls.waitUntilAppIsCompletelyInitialized(expressControls.getPid());
+  });
+
+  it('must report health status', function() {
+    var healthyTimestamp;
+    return utils.retry(function() {
+      return agentStubControls.getLastMetricValue(expressControls.getPid(), ['healthchecks'])
+      .then(function(healthchecks) {
+        expect(healthchecks.configurable.healthy).to.equal(1);
+        expect(healthchecks.configurable.since).to.be.gte(start);
+        healthyTimestamp = healthchecks.configurable.since;
+      });
+    })
+    .then(function() {
+      return expressControls.setUnhealthy();
+    })
+    .then(function() {
+      return utils.retry(function() {
+        return agentStubControls.getLastMetricValue(expressControls.getPid(), ['healthchecks'])
+        .then(function(healthchecks) {
+          expect(healthchecks.configurable.healthy).to.equal(0);
+          expect(healthchecks.configurable.since).to.be.gt(healthyTimestamp);
+        });
+      });
+    });
+  });
+});