diff --git a/README.md b/README.md index 67dd0e3..34fa794 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,9 @@ # RunIt +> [!NOTE] +> This tool still has some limitations. +> If you encounter any problems in use, please feel free to ask. + A simple program scheduler for your code on different devices. Let the machine move! @@ -10,39 +14,40 @@ Putting the machine into sleep is a disrespect for time. ```shell $ python run_it.py --help -usage: run_it.py [-h] --interpreter INTERPRETER [--verbose] --gpu-pool GPU_POOL [GPU_POOL ...] --max-workers MAX_WORKERS --cmd-pool CMD_POOL +usage: run_it.py [-h] [--verbose] [--gpu-pool GPU_POOL [GPU_POOL ...]] [--max-workers MAX_WORKERS] --cmd-pool CMD_POOL + [--max-used-ratio MAX_USED_RATIO] optional arguments: -h, --help show this help message and exit - --interpreter INTERPRETER - The path of your interpreter you want to use. --verbose Whether to print the output of the subprocess. --gpu-pool GPU_POOL [GPU_POOL ...] The pool containing all ids of your gpu devices. --max-workers MAX_WORKERS The max number of the workers. - --cmd-pool CMD_POOL The text file containing all your commands. It will be combined with `interpreter`. + --cmd-pool CMD_POOL The text file containing all your commands. It need to contain the launcher. + --max-used-ratio MAX_USED_RATIO + The max used ratio of the gpu. ``` ## demo ```shell -$ python run_it.py --interpreter python --verbose --cmd-pool ./examples/config.txt # with the default `gpu-pool` and `max-workers` -$ python run_it.py --interpreter python --verbose --gpu-pool 0 1 --max-workers 2 --cmd-pool ./examples/config.txt +$ python run_it.py --verbose --cmd-pool ./examples/config.txt # with the default `gpu-pool` and `max-workers` +$ python run_it.py --verbose --gpu-pool 0 1 --max-workers 2 --cmd-pool ./examples/config.txt ```
- -./examples/demo.py + +./examples/config.txt ```shell -$ cat ./examples/config.txt -./examples/demo.py -./examples/demo.py -./examples/demo.py -./examples/demo.py -./examples/demo.py +$ cat ./examples/config.txt +python -m pip list +python --version +python ./examples/demo.py +python ./examples/demo.py +python ./examples/demo.py ```
diff --git a/examples/config.txt b/examples/config.txt index e613104..71ffcab 100644 --- a/examples/config.txt +++ b/examples/config.txt @@ -1,5 +1,5 @@ -./examples/demo.py -./examples/demo.py -./examples/demo.py -./examples/demo.py -./examples/demo.py +python -m pip list +python --version +python ./examples/demo.py +python ./examples/demo.py +python ./examples/demo.py diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..7c26d9d --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,59 @@ +[tool.ruff] +# Exclude a variety of commonly ignored directories. +exclude = [ + ".bzr", + ".direnv", + ".eggs", + ".git", + ".git-rewrite", + ".hg", + ".ipynb_checkpoints", + ".mypy_cache", + ".nox", + ".pants.d", + ".pyenv", + ".pytest_cache", + ".pytype", + ".ruff_cache", + ".svn", + ".tox", + ".venv", + ".vscode", + "__pypackages__", + "_build", + "buck-out", + "build", + "dist", + "node_modules", + "site-packages", + "venv", +] + +# Same as Black. +line-length = 118 +indent-width = 4 + +[tool.ruff.lint] +# Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`) codes by default. +select = ["E4", "E7", "E9", "F"] +ignore = [] + +# Allow fix for all enabled rules (when `--fix`) is provided. +fixable = ["ALL"] +unfixable = [] + +# Allow unused variables when underscore-prefixed. +dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" + +[tool.ruff.format] +# Like Black, use double quotes for strings. +quote-style = "double" + +# Like Black, indent with spaces, rather than tabs. +indent-style = "space" + +# Like Black, respect magic trailing commas. +skip-magic-trailing-comma = false + +# Like Black, automatically detect the appropriate line ending. +line-ending = "auto" diff --git a/run_it.py b/run_it.py index ae3d7c7..50c5f3c 100644 --- a/run_it.py +++ b/run_it.py @@ -26,12 +26,18 @@ class MyProcess: curr_task_id = 0 def __init__( - self, interpreter_path, gpu_id, verbose=True, stdin=None, stdout=None, stderr=None, num_cmds=None, - max_used_ratio=0.5 + self, + gpu_id, + verbose=True, + num_cmds=None, + max_used_ratio=0.5, + *, + stdin=None, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, ): super().__init__() self.gpu_id = gpu_id - self.interpreter_path = interpreter_path self.verbose = verbose self.num_cmds = num_cmds @@ -60,13 +66,13 @@ def get_used_mem(self, return_ratio=False): def _create_sub_proc(self, cmd=""): self.sub_proc = subprocess.Popen( - args=f"CUDA_VISIBLE_DEVICES={self.gpu_id} {self.interpreter_path} -u {cmd}", + args=cmd, stdin=self.stdin, stdout=self.stdout, stderr=self.stderr, shell=True, - executable="bash", - env=None, + # executable="bash", + # env=None, close_fds=True, bufsize=1, text=True, @@ -117,26 +123,19 @@ def read_cmds_from_txt(path): return cmds +# fmt: off def get_args(): parser = argparse.ArgumentParser() - parser.add_argument("--interpreter", type=str, required=True, help="The path of your interpreter you want to use.") parser.add_argument("--verbose", action="store_true", help="Whether to print the output of the subprocess.") - parser.add_argument( - "--gpu-pool", nargs="+", type=int, default=[0], - help="The pool containing all ids of your gpu devices." - ) + parser.add_argument("--gpu-pool", nargs="+", type=int, default=[0], help="The pool containing all ids of your gpu devices.") parser.add_argument("--max-workers", type=int, help="The max number of the workers.") - parser.add_argument( - "--cmd-pool", - type=str, - required=True, - help="The text file containing all your commands. It will be combined with `interpreter`.", - ) + parser.add_argument("--cmd-pool",type=str, required=True, help="The text file containing all your commands. It need to contain the launcher.") parser.add_argument("--max-used-ratio", type=float, default=0.5, help="The max used ratio of the gpu.") args = parser.parse_args() if args.max_workers is None: args.max_workers = len(args.gpu_pool) return args +# fmt: on def main(): @@ -152,11 +151,8 @@ def main(): for i in range(min(args.max_workers, len(cmd_pool))): # 确保slots数量小于等于命令数量 gpu_id = i % num_gpus proc = MyProcess( - interpreter_path=args.interpreter, gpu_id=args.gpu_pool[gpu_id], verbose=args.verbose, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, num_cmds=len(cmd_pool), max_used_ratio=args.max_used_ratio, )