inotify-instances

#!/usr/bin/env python3

"""
Expose Linux inotify(7) instance resource consumption.

Operational properties:

  - This script may be invoked as an unprivileged user; in this case, metrics
    will only be exposed for processes owned by that unprivileged user.

  - No metrics will be exposed for processes that do not hold any inotify fds.

Requires Python 3.5 or later.
"""

import collections
import os
import sys
from prometheus_client import CollectorRegistry, Gauge, generate_latest


class Error(Exception):
    pass


class _PIDGoneError(Error):
    pass


_Process = collections.namedtuple(
    "Process", ["pid", "uid", "command", "inotify_instances"])


def _read_bytes(name):
    with open(name, mode='rb') as f:
        return f.read()


def _pids():
    for n in os.listdir("/proc"):
        if not n.isdigit():
            continue
        yield int(n)


def _pid_uid(pid):
    try:
        s = os.stat("/proc/{}".format(pid))
    except FileNotFoundError:
        raise _PIDGoneError()
    return s.st_uid


def _pid_command(pid):
    # Avoid GNU ps(1) for it truncates comm.
    # https://bugs.launchpad.net/ubuntu/+source/procps/+bug/295876/comments/3
    try:
        cmdline = _read_bytes("/proc/{}/cmdline".format(pid))
    except FileNotFoundError:
        raise _PIDGoneError()

    if not len(cmdline):
        return "<zombie>"

    try:
        prog = cmdline[0:cmdline.index(0x00)]
    except ValueError:
        prog = cmdline
    return os.path.basename(prog).decode(encoding="ascii",
                                         errors="surrogateescape")


def _pid_inotify_instances(pid):
    instances = 0
    try:
        for fd in os.listdir("/proc/{}/fd".format(pid)):
            try:
                target = os.readlink("/proc/{}/fd/{}".format(pid, fd))
            except FileNotFoundError:
                continue
            if target == "anon_inode:inotify":
                instances += 1
    except FileNotFoundError:
        raise _PIDGoneError()
    return instances


def _get_processes():
    for p in _pids():
        try:
            yield _Process(p, _pid_uid(p), _pid_command(p),
                           _pid_inotify_instances(p))
        except (PermissionError, _PIDGoneError):
            continue


def _get_processes_nontrivial():
    return (p for p in _get_processes() if p.inotify_instances > 0)


def main(args_unused=None):
    registry = CollectorRegistry()

    g = Gauge('inotify_instances', 'Total number of inotify instances held open by a process.',
              ['pid', 'uid', 'command'], registry=registry)

    for proc in _get_processes_nontrivial():
        g.labels(proc.pid, proc.uid, proc.command).set(proc.inotify_instances)
    print(generate_latest(registry).decode(), end='')


if __name__ == "__main__":
    sys.exit(main(sys.argv))