Remove the argument interpreter and directly use the full command i…

…n the config file.
lartpang · Feb 21, 2024 · 6a6f95a · 6a6f95a
1 parent 056b1cf
commit 6a6f95a
Show file tree

Hide file tree

Showing 4 changed files with 99 additions and 39 deletions.
diff --git a/README.md b/README.md
@@ -1,5 +1,9 @@
 # RunIt
 
+> [!NOTE]
+> This tool still has some limitations.
+> If you encounter any problems in use, please feel free to ask.
+
 A simple program scheduler for your code on different devices.
 
 Let the machine move!
@@ -10,39 +14,40 @@ Putting the machine into sleep is a disrespect for time.
 
 ```shell
 $ python run_it.py --help
-usage: run_it.py [-h] --interpreter INTERPRETER [--verbose] --gpu-pool GPU_POOL [GPU_POOL ...] --max-workers MAX_WORKERS --cmd-pool CMD_POOL
+usage: run_it.py [-h] [--verbose] [--gpu-pool GPU_POOL [GPU_POOL ...]] [--max-workers MAX_WORKERS] --cmd-pool CMD_POOL
+                 [--max-used-ratio MAX_USED_RATIO]
 
 optional arguments:
   -h, --help            show this help message and exit
-  --interpreter INTERPRETER
-                        The path of your interpreter you want to use.
   --verbose             Whether to print the output of the subprocess.
   --gpu-pool GPU_POOL [GPU_POOL ...]
                         The pool containing all ids of your gpu devices.
   --max-workers MAX_WORKERS
                         The max number of the workers.
-  --cmd-pool CMD_POOL   The text file containing all your commands. It will be combined with `interpreter`.
+  --cmd-pool CMD_POOL   The text file containing all your commands. It need to contain the launcher.
+  --max-used-ratio MAX_USED_RATIO
+                        The max used ratio of the gpu.
 ```
 
 ## demo
 
 ```shell
-$ python run_it.py --interpreter python --verbose --cmd-pool ./examples/config.txt # with the default `gpu-pool` and `max-workers`
-$ python run_it.py --interpreter python --verbose --gpu-pool 0 1 --max-workers 2 --cmd-pool ./examples/config.txt
+$ python run_it.py --verbose --cmd-pool ./examples/config.txt # with the default `gpu-pool` and `max-workers`
+$ python run_it.py --verbose --gpu-pool 0 1 --max-workers 2 --cmd-pool ./examples/config.txt
 ```
 
 <details>
-<summary> 
-./examples/demo.py
+<summary>
+./examples/config.txt
 </summary>
 
 ```shell
-$ cat ./examples/config.txt 
-./examples/demo.py
-./examples/demo.py
-./examples/demo.py
-./examples/demo.py
-./examples/demo.py
+$ cat ./examples/config.txt
+python -m pip list
+python --version
+python ./examples/demo.py
+python ./examples/demo.py
+python ./examples/demo.py
 ```
 
  </details>

diff --git a/examples/config.txt b/examples/config.txt
@@ -1,5 +1,5 @@
-./examples/demo.py
-./examples/demo.py
-./examples/demo.py
-./examples/demo.py
-./examples/demo.py
+python -m pip list
+python --version
+python ./examples/demo.py
+python ./examples/demo.py
+python ./examples/demo.py
diff --git a/pyproject.toml b/pyproject.toml
@@ -0,0 +1,59 @@
+[tool.ruff]
+# Exclude a variety of commonly ignored directories.
+exclude = [
+    ".bzr",
+    ".direnv",
+    ".eggs",
+    ".git",
+    ".git-rewrite",
+    ".hg",
+    ".ipynb_checkpoints",
+    ".mypy_cache",
+    ".nox",
+    ".pants.d",
+    ".pyenv",
+    ".pytest_cache",
+    ".pytype",
+    ".ruff_cache",
+    ".svn",
+    ".tox",
+    ".venv",
+    ".vscode",
+    "__pypackages__",
+    "_build",
+    "buck-out",
+    "build",
+    "dist",
+    "node_modules",
+    "site-packages",
+    "venv",
+]
+
+# Same as Black.
+line-length = 118
+indent-width = 4
+
+[tool.ruff.lint]
+# Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`)  codes by default.
+select = ["E4", "E7", "E9", "F"]
+ignore = []
+
+# Allow fix for all enabled rules (when `--fix`) is provided.
+fixable = ["ALL"]
+unfixable = []
+
+# Allow unused variables when underscore-prefixed.
+dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
+
+[tool.ruff.format]
+# Like Black, use double quotes for strings.
+quote-style = "double"
+
+# Like Black, indent with spaces, rather than tabs.
+indent-style = "space"
+
+# Like Black, respect magic trailing commas.
+skip-magic-trailing-comma = false
+
+# Like Black, automatically detect the appropriate line ending.
+line-ending = "auto"
diff --git a/run_it.py b/run_it.py
@@ -26,12 +26,18 @@ class MyProcess:
     curr_task_id = 0
 
     def __init__(
-        self, interpreter_path, gpu_id, verbose=True, stdin=None, stdout=None, stderr=None, num_cmds=None,
-        max_used_ratio=0.5
+        self,
+        gpu_id,
+        verbose=True,
+        num_cmds=None,
+        max_used_ratio=0.5,
+        *,
+        stdin=None,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
     ):
         super().__init__()
         self.gpu_id = gpu_id
-        self.interpreter_path = interpreter_path
         self.verbose = verbose
         self.num_cmds = num_cmds
 
@@ -60,13 +66,13 @@ def get_used_mem(self, return_ratio=False):
 
     def _create_sub_proc(self, cmd=""):
         self.sub_proc = subprocess.Popen(
-            args=f"CUDA_VISIBLE_DEVICES={self.gpu_id} {self.interpreter_path} -u {cmd}",
+            args=cmd,
             stdin=self.stdin,
             stdout=self.stdout,
             stderr=self.stderr,
             shell=True,
-            executable="bash",
-            env=None,
+            # executable="bash",
+            # env=None,
             close_fds=True,
             bufsize=1,
             text=True,
@@ -117,26 +123,19 @@ def read_cmds_from_txt(path):
     return cmds
 
 
+# fmt: off
 def get_args():
     parser = argparse.ArgumentParser()
-    parser.add_argument("--interpreter", type=str, required=True, help="The path of your interpreter you want to use.")
     parser.add_argument("--verbose", action="store_true", help="Whether to print the output of the subprocess.")
-    parser.add_argument(
-        "--gpu-pool", nargs="+", type=int, default=[0],
-        help="The pool containing all ids of your gpu devices."
-    )
+    parser.add_argument("--gpu-pool", nargs="+", type=int, default=[0], help="The pool containing all ids of your gpu devices.")
     parser.add_argument("--max-workers", type=int, help="The max number of the workers.")
-    parser.add_argument(
-        "--cmd-pool",
-        type=str,
-        required=True,
-        help="The text file containing all your commands. It will be combined with `interpreter`.",
-    )
+    parser.add_argument("--cmd-pool",type=str, required=True, help="The text file containing all your commands. It need to contain the launcher.")
     parser.add_argument("--max-used-ratio", type=float, default=0.5, help="The max used ratio of the gpu.")
     args = parser.parse_args()
     if args.max_workers is None:
         args.max_workers = len(args.gpu_pool)
     return args
+# fmt: on
 
 
 def main():
@@ -152,11 +151,8 @@ def main():
     for i in range(min(args.max_workers, len(cmd_pool))):  # 确保slots数量小于等于命令数量
         gpu_id = i % num_gpus
         proc = MyProcess(
-            interpreter_path=args.interpreter,
             gpu_id=args.gpu_pool[gpu_id],
             verbose=args.verbose,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.STDOUT,
             num_cmds=len(cmd_pool),
             max_used_ratio=args.max_used_ratio,
         )