From 608f13582e88677fc946c6cb84ec03459f29b4f9 Mon Sep 17 00:00:00 2001 From: Gianni Tedesco Date: Sun, 23 Aug 2020 13:48:18 +0900 Subject: [PATCH] Initial commit --- .gitignore | 17 +++ LICENSE.txt | 202 +++++++++++++++++++++++++++ Makefile | 14 ++ README.md | 150 ++++++++++++++++++++ minotaur/__init__.py | 35 +++++ minotaur/__main__.py | 97 +++++++++++++ minotaur/__version__.py | 8 ++ minotaur/_base.py | 301 ++++++++++++++++++++++++++++++++++++++++ minotaur/_event.py | 13 ++ minotaur/_inotify.c | 129 +++++++++++++++++ minotaur/_inotify.pyi | 36 +++++ minotaur/_mask.py | 69 +++++++++ minotaur/_minotaur.py | 130 +++++++++++++++++ minotaur/py.typed | 0 pre-commit.sh | 19 +++ setup.cfg | 2 + setup.py | 47 +++++++ 17 files changed, 1269 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE.txt create mode 100644 Makefile create mode 100644 README.md create mode 100644 minotaur/__init__.py create mode 100644 minotaur/__main__.py create mode 100644 minotaur/__version__.py create mode 100644 minotaur/_base.py create mode 100644 minotaur/_event.py create mode 100644 minotaur/_inotify.c create mode 100644 minotaur/_inotify.pyi create mode 100644 minotaur/_mask.py create mode 100644 minotaur/_minotaur.py create mode 100644 minotaur/py.typed create mode 100755 pre-commit.sh create mode 100644 setup.cfg create mode 100755 setup.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..134c75b --- /dev/null +++ b/.gitignore @@ -0,0 +1,17 @@ +# perf +/perf.data +/perf.data.old + +# python +__pycache__ +*.pyc +/.mypy_cache +/build +/dist +/MANIFEST + +# Text editors +.*.swp + +/minotaur/*.so +/*.egg-info diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..2913ed0 --- /dev/null +++ b/Makefile @@ -0,0 +1,14 @@ +MAKEFLAGS += --no-builtin-rules +.SUFFIXES: + +TARGET: all + +.PHONY: all +all: + ./setup.py build + +.PHONY: clean +clean: + ./setup.py clean + rm -rf build dist + find . -regex '^.*\(__pycache__\|\.py[co]\)$$' -delete diff --git a/README.md b/README.md new file mode 100644 index 0000000..d8d80a0 --- /dev/null +++ b/README.md @@ -0,0 +1,150 @@ +# Minotaur: A pythonic, asynchronous, inotify interface + +## Examples + +Minotaur provides the `Inotify` class which is to be used as a context +manager, from within which, one may iterate over inotify events: + +```python + with Inotify() as n: + n.add_watch('.', Mask.CREATE | Mask.DELETE | Mask.MOVE) + for evt in n: + print(evt) +``` + +The asynchronous interface works almost identically. The inotify object must +be created in nonblocking mode, and then the mere addition of the `async` +keyword to the iteration over events is all that's required: + +```python + with Inotify(blocking=False) as n: + n.add_watch('.', Mask.CREATE | Mask.DELETE | Mask.MOVE) + async for evt in n: + print(evt) +``` + +Example output would look like this: + +```python +Event(wd=1, mask=, cookie=0, name=PosixPath('foo')) +Event(wd=1, mask=, cookie=0, name=PosixPath('bar')) +Event(wd=1, mask=, cookie=129399, name=PosixPath('foo')) +Event(wd=1, mask=, cookie=129399, name=PosixPath('baz')) +Event(wd=1, mask=, cookie=0, name=PosixPath('bar')) +Event(wd=1, mask=, cookie=0, name=PosixPath('baz')) + +``` + +There is also a command-line tool demonstrating the features +```bash +$ python -m minotaur --help + +usage: minotaur [-h] [--async | --sync] [--fancy] [--mask MASK] dir [dir ...] + +Minotaur: A pythonic, asynchronous, inotify interface. + +A summary of inotify watch flags: + - ACCESS: File was accessed + - ATTRIB: Metaata changed, eg. permissions + - CLOSE_WRITE: File for writing was closed + - CLOSE_NOWRITE: File or dir not opened for writing was closed + - CREATE: File/dir was created + - DELETE: File or dir was deleted + - DELETE_SELF: Watched file/dir was itself deleted + - MODIFY: File was modified + - MOVE_SELF: Watched file/dir was itself moved + - MOVED_FROM: Generated for dir containing old filename when a file is renamed + - MOVED_TO: Generated for dir containing new filename when a file is renamed + - OPEN: File or dir was opened + - MOVE: MOVED_FROM | MOVED_TO + - CLOSE: IN_CLOSE_WRITE | IN_CLOSE_NOWRITE + - DONT_FOLLOW: Don't dereference pathname if it is a symbolic link + - EXCL_UNLINK: Don't generate events after files have been unlinked + - ONESHOT: Only generate one event for this watch + - ONLYDIR: Watch pathname only if it is a dir + - MASK_CREATE: Only watch path if it isn't already being watched + +positional arguments: + dir Watch for events in given dir + +optional arguments: + -h, --help show this help message and exit + --async, -a Use asyncio event loop + --sync, -s Use synchronous interface + --fancy, -f Use fancy interface + --mask MASK, -m MASK Events to watch for + +``` + +## What is different about Minotaur? + +1. C interface provides basic wrapper to syscalls and constants. In future, if + performance becomes a problem, more functionality can be gradually moved + there. + +2. Pythonic. `IntFlags` is used for watch types. Context-managers take care of + fd lifetime, `close()` method is idempotent. Raw `read()` and `readall()` + methods work comparably to python standard `io` objects. Full support for + `mypy`, including typeshed for C interface. Iterator and async-iterator + protocols supported. + +3. Makes no assumptions about the name encoding of filesystems, ie. with + `os.fsencode()` and `os.fsdecode()` + +4. Async interface supports multiple concurrent waiters. Waiting tasks are + woken in a first-come, first-sever manner. + +5. Users can chose between different levels of support: + 1. Raw syscall interface + 2. Low-level inotify object, which takes care of path encoding, reading of + raw inotify data, parsing of binary events in to python objects, and + provides both synchronous and async interface. But is still low-level + because it does no special handling of watches or combining of related + events (eg.`MOVE_FROM` / `MOVE_TO`). + 3. Fancy high-level interfaces, in pure python, built on top of low-level + interface. + +## What is missing + +There is no attempt to abstract file-notification functionality offered by +other operating systems in to a cross-platform interface. + +There are no tests. + +## Development +You should use the provided pre-commit hooks to make sure code type-checks and +is PEP-8 formatted: + +```bash +ln -sf ../../pre-commit.sh .git/hooks/pre-commit +``` + +## Why another one? + +There are several other python inotify packages. So why does this one exist? +Well, this can perhaps be explained best by referring to some of the others: + +1. `PyInotify`: suffers from numerous bugs. The fd closes aren't idempotent, + this can lead to closing unrelated file descriptors. This would be less of + an issue if the fd had a clear ownership and lifetime, or used context + managers. In other words, it's difficult to use safely. + +2. `PyInotify`: Assumes utf-8 filesystem encoding. No `asyncio` interface. + +3. `inotify_simple`: Nicely subclasses `FileIO`, but that precludes `asyncio` + since `FileIO` is meant for blocking I/O on files and cannot be easily + adapted for other purposes. + +4. `python_inotify`: No `asyncio` interface and, it would need to be added in + the C code, or if added in python code would duplicate the C code and work + differently, thus being a new API. + +5. `python-inotify`: It's packaged by RedHat but, similarly to + `python_inotify` the read() syscall is done in the C extension so it + doesn't support `asyncio`, and can't easily be adapted to do so without + changing the interface or duplicating functionality + +6. `asyncinotify`: Easily the best of the bunch. The main downside is that it + doesn't provide a synchronous interface or low-level interfaces. + +The others seem to be parts of larger projects, or systems. diff --git a/minotaur/__init__.py b/minotaur/__init__.py new file mode 100644 index 0000000..4a7eb47 --- /dev/null +++ b/minotaur/__init__.py @@ -0,0 +1,35 @@ +""" +Minotaur. A python, asynchronous, inotify interface. +""" + +from .__version__ import \ + __title__, \ + __description__, \ + __url__, \ + __author__, \ + __author_email__, \ + __copyright__, \ + __license__, \ + __version__ + +from ._mask import Mask +from ._base import Inotify, InotifyBase +from ._event import Event +from ._minotaur import Minotaur +from . import _inotify + +init = _inotify.init +add_watch = _inotify.add_watch +rm_watch = _inotify.rm_watch + +__all__ = ( + 'Mask', + 'Inotify', + 'InotifyBase', + 'Minotaur', + 'Event', + + 'init', + 'add_watch', + 'rm_watch', +) diff --git a/minotaur/__main__.py b/minotaur/__main__.py new file mode 100644 index 0000000..a618dc5 --- /dev/null +++ b/minotaur/__main__.py @@ -0,0 +1,97 @@ +from typing import Sequence +from argparse import ArgumentParser, RawTextHelpFormatter +from pathlib import Path +from functools import reduce +from operator import or_ +import asyncio +import re + +from minotaur import Inotify, Mask, Minotaur +from minotaur import __title__ + + +_splitter = re.compile(r'[,\.|:\-\s]+') + + +def _sync_main(cls, dirs: Sequence[Path], mask: Mask): + with cls() as n: + for p in dirs: + wd = n.add_watch(p, mask) + print(f'{p.resolve()}: wd={wd}') + + for evt in n: + print(evt) + + +async def _task(cls, dirs: Sequence[Path], mask: Mask): + with cls(blocking=False) as n: + for p in dirs: + wd = n.add_watch(p, mask) + print(f'{p.resolve()}: wd={wd}') + + async for evt in n: + print(evt) + + +def _async_main(cls, dirs: Sequence[Path], mask: Mask): + loop = asyncio.get_event_loop() + coro = _task(cls, dirs, mask) + task = loop.create_task(coro) + loop.run_until_complete(task) + + +_mask_help = '\n'.join((f' - {x.name}: {x.__doc__}' + for x in Mask if x.show_help)) + +_prog_desc = f""" +Minotaur: A pythonic, asynchronous, inotify interface. + +A summary of inotify watch flags: +{_mask_help} +""" + +_default_mask = 'create,delete,delete_self,moved_from,moved_to' + + +def _main(): + opts = ArgumentParser(prog=__title__, + formatter_class=RawTextHelpFormatter, + description=_prog_desc) + + opts.set_defaults(func=_async_main, cls=Inotify) + + g = opts.add_mutually_exclusive_group(required=False) + g.add_argument('--async', '-a', + dest='func', + action='store_const', + const=_async_main, + help='Use asyncio event loop') + g.add_argument('--sync', '-s', + dest='func', + action='store_const', + const=_sync_main, + help='Use synchronous interface') + + opts.add_argument('--fancy', '-f', + dest='cls', + action='store_const', + const=Minotaur, + help='Use fancy interface') + opts.add_argument('--mask', '-m', + default=_default_mask, + help='Events to watch for') + opts.add_argument('dir', + nargs='+', + type=Path, + help='Watch for events in given dir') + + args = opts.parse_args() + + flags = (Mask[tok.upper()] for tok in _splitter.split(args.mask)) + mask = reduce(or_, flags) + + args.func(args.cls, args.dir, mask) + + +if __name__ == '__main__': + _main() diff --git a/minotaur/__version__.py b/minotaur/__version__.py new file mode 100644 index 0000000..5b15503 --- /dev/null +++ b/minotaur/__version__.py @@ -0,0 +1,8 @@ +__title__ = 'minotaur' +__description__ = 'Pythonic, asynchronous, inotify interface.' +__url__ = 'https://github.com/giannitedesco/minotaur' +__author__ = 'Gianni Tedesco' +__author_email__ = 'gianni@scaramanga.co.uk' +__copyright__ = 'Copyright 2020 Gianni Tedesco' +__license__ = 'Apache 2.0' +__version__ = '0.0.2' diff --git a/minotaur/_base.py b/minotaur/_base.py new file mode 100644 index 0000000..d7a248f --- /dev/null +++ b/minotaur/_base.py @@ -0,0 +1,301 @@ +from typing import Deque, Optional +from pathlib import Path +from collections import deque +from io import DEFAULT_BUFFER_SIZE +from struct import Struct +import os as _os + +import asyncio + +from . import _inotify +from ._mask import Mask +from ._event import Event + +__all__ = ('InotifyBase', 'Inotify') + + +class InotifyBase: + __slots__ = ( + '_nonblock', + '_cloexec', + '_fd', + '_buf', + '_loop', + '_waitq', + ) + + _loop: Optional[asyncio.AbstractEventLoop] + + _event = Struct('=iIII') + _event_sz = _event.size + + def __init__(self, + blocking=True, + cloexec=True, + loop: Optional[asyncio.AbstractEventLoop] = None): + + self._nonblock = not blocking + self._cloexec = cloexec + self._fd = -1 + + if self._nonblock: + if loop is None: + loop = asyncio.get_event_loop() + self._loop = loop + self._waitq: Deque[asyncio.Future] = deque() + else: + self._loop = None + + @property + def closed(self) -> bool: + """ + True if the inotify fd is closed + """ + return self._fd < 0 + + def open(self): + """ + Create the inotify fd + """ + + assert(self._fd < 0) + nb = self._nonblock and _inotify.IN_NONBLOCK + ce = self._cloexec and _inotify.IN_CLOEXEC + self._fd = _inotify.init(nb | ce) + + def _check_open(self): + if self.closed: + raise ValueError("I/O operation on closed file.") + + def _register_for_read(self): + self._check_open() + assert(self._nonblock) + assert(self._loop is not None) + self._loop.add_reader(self._fd, self._fd_readable) + + def _unregister_for_read(self): + self._check_open() + assert(self._nonblock) + assert(self._loop is not None) + self._loop.remove_reader(self._fd) + + def _fd_readable(self): + """ + Callback for asyncio when the inotify fd has become readable + """ + + while self._waitq: + w = self._waitq.popleft() + if not self._waitq: + self._unregister_for_read() + if not w.done(): + w.set_result(None) + break + + def close(self): + """ + Close the inotify fd + + This will wake all asynchronous waiters. + """ + + if not self.closed: + try: + _os.close(self._fd) + finally: + self._fd = -1 + + # Must wake all waiters to let them exit their loops now that we're + # closed + if self._loop: + assert(self._loop is not None) + try: + self._loop.remove_reader(self._fd) + except ValueError: + pass + while self._waitq: + w = self._waitq.popleft() + if not w.done(): + w.set_result(None) + + def __del__(self): + self.close() + + def __enter__(self): + self.open() + return self + + def __exit__(self, exc_type, exc_val, traceback): + self.close() + + def add_watch(self, p: Path, mask: Mask) -> int: + """ + Add a watch to the inotify fd. + + Return the watch descriptor (an integer). + """ + + self._check_open() + path = _os.fsencode(p) + return _inotify.add_watch(self._fd, path, mask.value) + + def rm_watch(self, wd: int) -> int: + """ + Remove a watch from the inotify fd, given a watch descriptor + """ + + self._check_open() + return _inotify.rm_watch(self._fd, wd) + + def read(self, size: Optional[int] = None) -> bytes: + """ + Read raw bytes from the inotify fd. + + This is probably not the interface that you want. Use the iterator + protocol instead. + """ + + if size is None or size < 0: + return self.readall() + + self._check_open() + return _os.read(self._fd, size) + + def readall(self) -> bytes: + """ + Read all raw bytes from the inotify fd. + + This only works in non-blocking mode, since in blocking mode the + returned object would, by definition, be infinite in size. + """ + + self._check_open() + + result = bytearray() + n = DEFAULT_BUFFER_SIZE + + if not self._nonblock: + raise ValueError("Can't readall on blocking inotify") + + while True: + try: + chunk = _os.read(self._fd, n) + except BlockingIOError: + if result: + break + return b'' + if not chunk: + break + result += chunk + + return bytes(result) + + def __iter__(self): + self._check_open() + + if self._nonblock: + raise ValueError("Synchronous iiter used on non-blocking inotify") + + if hasattr(self, '_buf'): + raise ValueError("Concurrent iteration on inotify fd") + + self._buf = b'' + return self + + def __aiter__(self): + self._check_open() + + if not self._nonblock: + raise ValueError("Async iiter used on blocking inotify") + + if not self._loop: + raise ValueError("No event loop for async iter") + + if hasattr(self, '_buf'): + raise ValueError("Concurrent iteration on inotify fd") + + self._buf = b'' + return self + + def _fill_buffer(self, chunk_size: int = DEFAULT_BUFFER_SIZE) -> bool: + """ + Read chunk_size bytes from the fd and append to the internal buffer. + """ + + chunk = self.read(chunk_size) + if self._buf: + self._buf += chunk + else: + self._buf = chunk + return bool(chunk) + + def _buf_pop(self) -> Optional[Event]: + """ + Parse the first event frm the buffer and return it as an Event object. + If there is not enough data in the buffer then return None. + """ + + hdr_len = self._event_sz + + if len(self._buf) < hdr_len: + return None + + wd, mask, cookie, evlen = self._event.unpack_from(self._buf) + + tot_len = hdr_len + evlen + if len(self._buf) < tot_len: + return None + + path_buf = self._buf[hdr_len:tot_len] + try: + nul = path_buf.find(0) + except ValueError: + pass + else: + path_buf = path_buf[:nul] + + path = _os.fsdecode(path_buf) + self._buf = self._buf[tot_len:] + + return Event(wd, Mask(mask), cookie, Path(path)) + + def _next_event(self) -> Optional[Event]: + while not self.closed: + ret = self._buf_pop() + if ret: + return ret + + self._fill_buffer() + + return None + + async def _next_event_async(self) -> Optional[Event]: + while not self.closed: + ret = self._buf_pop() + if ret: + return ret + + if not self._fill_buffer(-1): + assert(self._loop is not None) + + if not self._waitq: + self._register_for_read() + wake = self._loop.create_future() + self._waitq.append(wake) + await wake + + return None + + +class Inotify(InotifyBase): + def __next__(self) -> Event: + evt = self._next_event() + if evt is not None: + return evt + raise StopIteration + + async def __anext__(self) -> Event: + evt = await self._next_event_async() + if evt is not None: + return evt + raise StopIteration diff --git a/minotaur/_event.py b/minotaur/_event.py new file mode 100644 index 0000000..d002c97 --- /dev/null +++ b/minotaur/_event.py @@ -0,0 +1,13 @@ +from typing import NamedTuple +from pathlib import Path + +from ._mask import Mask + +__all__ = ('Event',) + + +class Event(NamedTuple): + wd: int + mask: Mask + cookie: int + name: Path diff --git a/minotaur/_inotify.c b/minotaur/_inotify.c new file mode 100644 index 0000000..489e3b4 --- /dev/null +++ b/minotaur/_inotify.c @@ -0,0 +1,129 @@ +#define PY_SSIZE_T_CLEAN +#include + +#include + +static PyObject *_inotify_init(PyObject *self, PyObject *args) +{ + int flags = 0, ret; + + if (!PyArg_ParseTuple(args, "|i", &flags)) + return NULL; + + ret = inotify_init1(flags); + + /* FIXME: raise OSError if this fails */ + return PyLong_FromLong(ret); +} + +static PyObject *_inotify_add_watch(PyObject *self, PyObject *args) +{ + int fd, ret; + unsigned int mask; + char *path; + + if (!PyArg_ParseTuple(args, "iyI", &fd, &path, &mask)) + return NULL; + + ret = inotify_add_watch(fd, path, mask); + if (ret < 0) { + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + + return PyLong_FromLong(ret); +} + +static PyObject *_inotify_rm_watch(PyObject *self, PyObject *args) +{ + int fd, wd, ret; + + if (!PyArg_ParseTuple(args, "ii", &fd, &wd)) + return NULL; + + ret = inotify_rm_watch(fd, wd); + if (ret < 0) { + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + + return PyLong_FromLong(ret); +} + +static PyMethodDef _inotify_methods[] = { + { + "init", + _inotify_init, + METH_VARARGS, + "Create inotify fd." + }, + { + "add_watch", + _inotify_add_watch, + METH_VARARGS, + "Add watch to inotify fd." + }, + { + "rm_watch", + _inotify_rm_watch, + METH_VARARGS, + "Remove watch from inotify fd." + }, + { + NULL, + }, +}; + +static struct PyModuleDef _inotify_module = { + PyModuleDef_HEAD_INIT, + "_inotify", + NULL, + -1, + _inotify_methods, +}; + +PyMODINIT_FUNC +PyInit__inotify(void) +{ + PyObject *m; + + m = PyModule_Create(&_inotify_module); + if (m == NULL) + return NULL; + + PyModule_AddIntMacro(m, IN_NONBLOCK); + PyModule_AddIntMacro(m, IN_CLOEXEC); + + PyModule_AddIntMacro(m, IN_ACCESS); + PyModule_AddIntMacro(m, IN_ATTRIB); + PyModule_AddIntMacro(m, IN_CLOSE_WRITE); + PyModule_AddIntMacro(m, IN_CLOSE_NOWRITE); + PyModule_AddIntMacro(m, IN_CREATE); + PyModule_AddIntMacro(m, IN_DELETE); + PyModule_AddIntMacro(m, IN_DELETE_SELF); + PyModule_AddIntMacro(m, IN_MODIFY); + PyModule_AddIntMacro(m, IN_MOVE_SELF); + PyModule_AddIntMacro(m, IN_MOVED_FROM); + PyModule_AddIntMacro(m, IN_MOVED_TO); + PyModule_AddIntMacro(m, IN_OPEN); + + PyModule_AddIntMacro(m, IN_MOVE); + PyModule_AddIntMacro(m, IN_CLOSE); + + PyModule_AddIntMacro(m, IN_DONT_FOLLOW); + PyModule_AddIntMacro(m, IN_EXCL_UNLINK); + PyModule_AddIntMacro(m, IN_MASK_ADD); + PyModule_AddIntMacro(m, IN_ONESHOT); + PyModule_AddIntMacro(m, IN_ONLYDIR); + PyModule_AddIntMacro(m, IN_MASK_CREATE); + + PyModule_AddIntMacro(m, IN_IGNORED); + PyModule_AddIntMacro(m, IN_ISDIR); + PyModule_AddIntMacro(m, IN_Q_OVERFLOW); + PyModule_AddIntMacro(m, IN_UNMOUNT); + +#define EVENT_TYPE_MASK (0xfff) + PyModule_AddIntMacro(m, EVENT_TYPE_MASK); + + return m; +} diff --git a/minotaur/_inotify.pyi b/minotaur/_inotify.pyi new file mode 100644 index 0000000..2f457c3 --- /dev/null +++ b/minotaur/_inotify.pyi @@ -0,0 +1,36 @@ +def init(flags: int = 0) -> int: ... +def add_watch(fd: int, path: bytes, flags: int) -> int: ... +def rm_watch(fd: int, wd: int) -> int: ... + +IN_NONBLOCK: int +IN_CLOEXEC: int + +IN_ACCESS: int +IN_ATTRIB: int +IN_CLOSE_WRITE: int +IN_CLOSE_NOWRITE: int +IN_CREATE: int +IN_DELETE: int +IN_DELETE_SELF: int +IN_MODIFY: int +IN_MOVE_SELF: int +IN_MOVED_FROM: int +IN_MOVED_TO: int +IN_OPEN: int + +IN_MOVE: int +IN_CLOSE: int + +IN_DONT_FOLLOW: int +IN_EXCL_UNLINK: int +IN_MASK_ADD: int +IN_ONESHOT: int +IN_ONLYDIR: int +IN_MASK_CREATE: int + +IN_IGNORED: int +IN_ISDIR: int +IN_Q_OVERFLOW: int +IN_UNMOUNT: int + +EVENT_TYPE_MASK: int diff --git a/minotaur/_mask.py b/minotaur/_mask.py new file mode 100644 index 0000000..eaddd0f --- /dev/null +++ b/minotaur/_mask.py @@ -0,0 +1,69 @@ +from enum import IntFlag as _IntFlag + +from . import _inotify + + +__all__ = ('Mask',) + + +class Mask(_IntFlag): + show_help: bool + + def __new__(cls, value, doc=None, show_help=True): + # int.__new__ needs a stub in the typeshed + # https://github.com/python/typeshed/issues/2686 + # + # but that broke something else, so they removed it + # https://github.com/python/typeshed/issues/1464 + # + # We have no choice but to ignore mypy error here :( + self = int.__new__(cls, value) # type: ignore + self._value_ = value + if doc is not None: + self.__doc__ = doc + self.show_help = show_help + return self + + """ + Flags for establishing inotify watches. + """ + + ACCESS = _inotify.IN_ACCESS, 'File was accessed' + ATTRIB = _inotify.IN_ATTRIB, 'Metaata changed, eg. permissions' + CLOSE_WRITE = _inotify.IN_CLOSE_WRITE, 'File for writing was closed' + CLOSE_NOWRITE = _inotify.IN_CLOSE_NOWRITE, \ + 'File or dir not opened for writing was closed' + CREATE = _inotify.IN_CREATE, 'File/dir was created' + DELETE = _inotify.IN_DELETE, 'File or dir was deleted' + DELETE_SELF = _inotify.IN_DELETE_SELF, \ + 'Watched file/dir was itself deleted' + MODIFY = _inotify.IN_MODIFY, 'File was modified' + MOVE_SELF = _inotify.IN_MOVE_SELF, 'Watched file/dir was itself moved' + MOVED_FROM = _inotify.IN_MOVED_FROM, \ + 'Generated for dir containing old filename when a file is renamed' + MOVED_TO = _inotify.IN_MOVED_TO, \ + 'Generated for dir containing new filename when a file is renamed' + OPEN = _inotify.IN_OPEN, 'File or dir was opened' + MOVE = _inotify.IN_MOVE, 'MOVED_FROM | MOVED_TO' + CLOSE = _inotify.IN_CLOSE, 'IN_CLOSE_WRITE | IN_CLOSE_NOWRITE' + + DONT_FOLLOW = _inotify.IN_DONT_FOLLOW, \ + "Don't dereference pathname if it is a symbolic link" + EXCL_UNLINK = _inotify.IN_EXCL_UNLINK, \ + "Don't generate events after files have been unlinked" + + MASK_ADD = _inotify.IN_MASK_ADD, 'Add flags to an existing watch', False + + ONESHOT = _inotify.IN_ONESHOT, 'Only generate one event for this watch' + ONLYDIR = _inotify.IN_ONLYDIR, 'Watch pathname only if it is a dir' + MASK_CREATE = _inotify.IN_MASK_CREATE, \ + "Only watch path if it isn't already being watched" + + # These are returned in events + IGNORED = _inotify.IN_IGNORED, 'Watch was removed', False + ISDIR = _inotify.IN_ISDIR, 'This event is a dir', False + Q_OVERFLOW = _inotify.IN_Q_OVERFLOW, 'Event queue overflowed', False + UNMOUNT = _inotify.IN_UNMOUNT, \ + 'Filesystem containing watched object was unmounted', False + + EVENT_TYPE = _inotify.EVENT_TYPE_MASK, 'Mask of all event types', False diff --git a/minotaur/_minotaur.py b/minotaur/_minotaur.py new file mode 100644 index 0000000..223822a --- /dev/null +++ b/minotaur/_minotaur.py @@ -0,0 +1,130 @@ +from typing import Dict, Tuple +from pathlib import Path +from abc import ABC, abstractmethod + +import asyncio + +from ._mask import Mask +from ._event import Event +from ._base import InotifyBase + +__all__ = ('Minotaur',) + + +class Notification(ABC): + __slots__ = ( + '_path', + '_type', + '_isdir', + '_unmount', + '_qoverflow', + ) + + def __init__(self, + path: Path, + type: Mask, + isdir: bool, + unmount: bool, + qoverflow: bool = False): + self._path = path + self._type = type + self._isdir = bool(isdir) + self._unmount = bool(unmount) + self._qoverflow = bool(qoverflow) + + @property + def isdir(self) -> bool: + return self._isdir + + @property + def unmount(self) -> bool: + return self._unmount + + @property + def qoverflow(self) -> bool: + return self._qoverflow + + @property + def path(self) -> Path: + return self._path + + def __repr__(self): + t = self._isdir and 'dir' or 'file' + return f'{type(self).__name__}({self._type.name} {t} {self._path})' + + @classmethod + def create(cls, path: Path, mask: Mask): + return cls(path, + mask & Mask.EVENT_TYPE, + bool(mask & Mask.ISDIR), + bool(mask & Mask.UNMOUNT), + bool(mask & Mask.Q_OVERFLOW)) + + +class Minotaur(InotifyBase): + """ + Fancy interface for Inotify which does questionable things like: + + 1. Resolve watch-descriptors back to paths (which races with renames of + original paths and can't be used safely, but other inotify packages + provide this feature, so here it is for your delectation). + 2. Link rename_from/rename_to events together. This feature would be + useful but isn't yet actually implemented. Working on it... + """ + + __slots__ = ( + '_wdmap', + '_cmap', + ) + + _wdmap: Dict[int, Path] + _cmap: Dict[Tuple[int, int], Event] + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._wdmap = {} + self._cmap = {} + + def add_watch(self, p: Path, mask: Mask) -> int: + try: + wd = super().add_watch(p, mask) + except Exception: + raise + else: + self._wdmap[wd] = p.resolve() + return wd + + def rm_watch(self, wd: int) -> int: + try: + return super().rm_watch(wd) + except Exception: + raise + else: + del self._wdmap[wd] + + def _resolve_path(self, wd: int, name: Path): + try: + base_dir = self._wdmap[wd] + except KeyError: + path = name + else: + path = base_dir / name + + return path + + def __next__(self) -> Notification: + evt = super()._next_event() + if evt is None: + raise StopIteration + + # TODO: Link rename_from/rename_to together if we have them + path = self._resolve_path(evt.wd, evt.name) + return Notification.create(path, evt.mask) + + async def __anext__(self) -> Notification: + evt = await super()._next_event_async() + if evt is None: + raise StopIteration + + path = self._resolve_path(evt.wd, evt.name) + return Notification.create(path, evt.mask) diff --git a/minotaur/py.typed b/minotaur/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/pre-commit.sh b/pre-commit.sh new file mode 100755 index 0000000..0092130 --- /dev/null +++ b/pre-commit.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +set -euo pipefail + +exec 1>&2 + +pkgname='minotaur' +scripts=setup.py + +mypy \ + --check-untyped-defs \ + --no-implicit-optional \ + --ignore-missing \ + ${pkgname} ${scripts} + +pycodestyle-3 \ + ${pkgname} + +#./unit-tests.sh diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..b88034e --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[metadata] +description-file = README.md diff --git a/setup.py b/setup.py new file mode 100755 index 0000000..69d59d4 --- /dev/null +++ b/setup.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 + +from typing import Dict, Any +from pathlib import Path +import setuptools + +_c_extension = setuptools.Extension('minotaur._inotify', + ['minotaur/_inotify.c',]) + +version_file = Path('minotaur/__version__.py') +v: Dict[str, Any] = {} +exec(version_file.read_text(), v) + +setuptools.setup( + name=v['__title__'], + version=v['__version__'], + description=v['__description__'], + author=v['__author__'], + author_email=v['__author_email__'], + package_data={ + 'minotaur': ['py.typed'], + }, + long_description=Path('README.md').read_text(), + long_description_content_type="text/markdown", + include_package_data=True, + license=v['__license__'], + platforms='Linux', + packages=[ + 'minotaur', + ], + url=v['__url__'], + ext_modules=[_c_extension,], + classifiers=[ + 'Development Status :: 4 - Beta', + 'Intended Audience :: Developers', + 'Natural Language :: English', + 'Operating System :: POSIX :: Linux', + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: Implementation :: CPython', + ], +)