From f0a2b8fadb721f6dd021a32a27c1d66f9dfe367f Mon Sep 17 00:00:00 2001 From: tw-yshuang Date: Thu, 24 Aug 2023 18:49:27 +0800 Subject: [PATCH 1/7] [bug] When booking.py has someone write the data to the booking.csv during the Monitor.py running, Monitor.py will recover the action of booking.py src/monitor/Monitor.py, wait to fix it --- src/monitor/Monitor.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/monitor/Monitor.py b/src/monitor/Monitor.py index 592474a..f51ed79 100644 --- a/src/monitor/Monitor.py +++ b/src/monitor/Monitor.py @@ -293,6 +293,8 @@ def exec(self) -> None: self.update_sdf(self.using, self.used, now_using, move2used, close_results, msg="using to used") if not (move2using is None or move2using.empty): + #! issue: When booking.py has someone write the data to the booking.csv during the Monitor.py running, Monitor.py will recover the action of booking.py + # TODO: add the hash-table for the booking.csv to check the write time, make sure the information can integrate! run_results = np.array([self.check_space(user_id) for user_id in move2using[SC.user_id]], dtype=np.bool_) run_results[run_results] = self.run_containers(move2using[run_results]) self.update_sdf(self.booking, self.using, now_booking, move2using, run_results, msg="booking to using") From 891bfe53981966709b9e1e95fdace0aab8186917 Mon Sep 17 00:00:00 2001 From: tw-yshuang Date: Fri, 13 Oct 2023 13:38:55 +0800 Subject: [PATCH 2/7] [bug] booking stage, someone use booking at the same time, it will be cover it, the user_config you want to update maybe loss. The idea of fix this issue is also in the comment. --- src/booking/booking.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/booking/booking.py b/src/booking/booking.py index 68c79ff..fee40b8 100755 --- a/src/booking/booking.py +++ b/src/booking/booking.py @@ -262,6 +262,8 @@ def __get_bookingtime(user_id: str) -> BasicCapability: def __update_users_config_and_yaml(user_id: str, user_config: UserConfig): + # ! bug: someone use booking at the same time, it will be cover it, the user_config you want to update maybe loss. + # TODO: a better method is to append this user_config to the end of the yaml file, next time load it, it will get the latest info., and create a mechanism use a period of time to update all of it, like monitor mechanism. checker.users_config.ids[user_id] = user_config dump_yaml(checker.users_config.to_dict(), PROJECT_DIR / checker.deploy_info.users_config_yaml) From 828449f0c3917e762d070aca0fe51e71aa4fb71c Mon Sep 17 00:00:00 2001 From: tw-yshuang Date: Fri, 13 Oct 2023 13:47:31 +0800 Subject: [PATCH 3/7] [mod] docs/FUTURE_WORK.md, add some important feature for the futrue. --- docs/FUTURE_WORK.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/FUTURE_WORK.md b/docs/FUTURE_WORK.md index ed3c5c8..0fa38eb 100644 --- a/docs/FUTURE_WORK.md +++ b/docs/FUTURE_WORK.md @@ -1,5 +1,15 @@ # Future Work +--- + +## `booking.csv`, `using.csv`, `used.csv` + +Convert it to the database(suggest use `sqlite`), more efficient, more easy to manipulate. + +Another reason is that `pandas` is a slow tool. + +--- + ## `src/booking/booking.py` 1. Add new CLI feature From 4f2fea8533a31bf7aa6025a931216f520a7ff945 Mon Sep 17 00:00:00 2001 From: tw-yshuang Date: Fri, 13 Oct 2023 15:48:28 +0800 Subject: [PATCH 4/7] [mod] add 'append' method for yaml file. src/HostInfo.py, change-func.: dump_yaml() -> write_yaml(); add-func.: append_yaml(), for append yaml src/booking/booking.py, change method all the __update_users_config_and_yaml() now replace to __increase_user_config_and_yaml(); add-func.: __increase_user_config_and_yaml(), increase user info by using append_yaml() --- src/HostInfo.py | 9 ++++++++- src/booking/booking.py | 15 ++++++++++----- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/HostInfo.py b/src/HostInfo.py index c3a1b0d..79650f0 100644 --- a/src/HostInfo.py +++ b/src/HostInfo.py @@ -24,12 +24,19 @@ def load_yaml(filename: str) -> dict: return yaml.load(f, Loader=yaml.SafeLoader) -def dump_yaml(info_dict: dict, filename: str): +def write_yaml(info_dict: dict, filename: str): with open(filename, 'w') as f: yaml.dump(info_dict, f, Dumper=CustomDumper, sort_keys=False) return True +def append_yaml(info_dict: dict, filename: str): + with open(filename, 'a') as f: + f.writelines('\n') + yaml.dump(info_dict, f, Dumper=CustomDumper, sort_keys=False) + return True + + @dataclass class BookingTime: ''' diff --git a/src/booking/booking.py b/src/booking/booking.py index fee40b8..f701cc8 100755 --- a/src/booking/booking.py +++ b/src/booking/booking.py @@ -15,7 +15,7 @@ sys.path.append(str(PROJECT_DIR)) from lib.WordOperator import str_format, ask_yn -from src.HostInfo import BookingTime, BasicCapability, UserConfig, ScheduleDF, dump_yaml +from src.HostInfo import BookingTime, BasicCapability, UserConfig, ScheduleDF, write_yaml, append_yaml from src.HostInfo import ScheduleColumnNames as SC from src.booking.Checker import Checker @@ -266,7 +266,12 @@ def __update_users_config_and_yaml(user_id: str, user_config: UserConfig): # TODO: a better method is to append this user_config to the end of the yaml file, next time load it, it will get the latest info., and create a mechanism use a period of time to update all of it, like monitor mechanism. checker.users_config.ids[user_id] = user_config - dump_yaml(checker.users_config.to_dict(), PROJECT_DIR / checker.deploy_info.users_config_yaml) + write_yaml(checker.users_config.to_dict(), PROJECT_DIR / checker.deploy_info.users_config_yaml) + + +def __increase_user_config_and_yaml(user_id: str, user_config: UserConfig): + checker.users_config.ids[user_id] = user_config + append_yaml({user_id: user_config.to_dict()}, PROJECT_DIR / checker.deploy_info.users_config_yaml) def __add_new_user_config(user_id: str) -> UserConfig: @@ -285,7 +290,7 @@ def __add_new_user_config(user_id: str) -> UserConfig: volume_backup_dir=f"{checker.deploy_info.volume_backup_dir}/{user_id}", ) - __update_users_config_and_yaml(user_id, user_config) + __increase_user_config_and_yaml(user_id, user_config) print(str_format(f"{user_id}'s profile:", fore='y')) for k, v in user_config.dict.items(): if 'volume' not in k: @@ -360,13 +365,13 @@ def __setting_user_options(user_id: str, user_config: UserConfig): else: user_config.password = new_password checker.users_config.ids[user_id].password = new_password - __update_users_config_and_yaml(user_id, checker.users_config.ids[user_id]) + __increase_user_config_and_yaml(user_id, checker.users_config.ids[user_id]) print(str_format("Update default Password!", fore='g')) break # Update users_config.yaml if ask_yn("The previous setting is for the once, do you want to update the default config?"): - __update_users_config_and_yaml(user_id, user_config) + __increase_user_config_and_yaml(user_id, user_config) print(str_format("Update your user_config!", fore='g')) return user_config From 99a5c8de67e133f863962b8adfaed3b897ca4515 Mon Sep 17 00:00:00 2001 From: tw-yshuang Date: Fri, 13 Oct 2023 17:17:00 +0800 Subject: [PATCH 5/7] [fix][mod] fix bug, better code, and update initial password method. fix: src/booking/booking.py, refresh users_config.yaml and save it at the end of cli() mod: src/HostInfo.py, add-attr.: HostDeployInfo.deploy_config_yaml src/booking/booking.py, change-attr.: DEFAULT_PASSWORD='0000' -> PASSWORD_LEAST_LEN=4; change-func.: __update_users_config_and_yaml(user_id: str, user_config: UserConfig) -> __update_users_config2yaml(users_config: UsersConfig); add-func.: __create_new_password(), more easy to manage the method of create password __setting_user_options(), the option order of the password become first --- src/HostInfo.py | 2 + src/booking/booking.py | 108 ++++++++++++++++++++++------------------- 2 files changed, 60 insertions(+), 50 deletions(-) diff --git a/src/HostInfo.py b/src/HostInfo.py index 79650f0..428dc62 100644 --- a/src/HostInfo.py +++ b/src/HostInfo.py @@ -173,11 +173,13 @@ class HostDeployInfo: volume_dataset_dir: Path volume_backup_dir: Path + deploy_config_yaml: Path capability_config_yaml: Path users_config_yaml: Path images: List[str] def __init__(self, yaml: Path = PROJECT_DIR / 'host_deploy.yaml') -> None: + self.deploy_config_yaml = yaml for k, v in load_yaml(yaml).items(): setattr(self, k, v) diff --git a/src/booking/booking.py b/src/booking/booking.py index f701cc8..f77c9bc 100755 --- a/src/booking/booking.py +++ b/src/booking/booking.py @@ -15,18 +15,23 @@ sys.path.append(str(PROJECT_DIR)) from lib.WordOperator import str_format, ask_yn -from src.HostInfo import BookingTime, BasicCapability, UserConfig, ScheduleDF, write_yaml, append_yaml +from src.HostInfo import BookingTime, BasicCapability, UserConfig, UsersConfig, ScheduleDF, write_yaml, append_yaml from src.HostInfo import ScheduleColumnNames as SC from src.booking.Checker import Checker -checker = Checker(deploy_yaml=PROJECT_DIR / 'cfg/test/host_deploy.yaml') +checker = Checker( + deploy_yaml=PROJECT_DIR / 'cfg/test/host_deploy.yaml', + booking_csv=PROJECT_DIR / 'jobs/test/booking.csv', + using_csv=PROJECT_DIR / 'jobs/test/using.csv', + used_csv=PROJECT_DIR / 'jobs/test/used.csv', +) MONITOR_EXEC_PATH: Path = PROJECT_DIR / 'jobs/monitor_exec' MIN_CPUS: float = 1 MIN_MEMORY: int = 1 MIN_GPUS: int = 0 -DEFAULT_PASSWORD: str = '0000' +PASSWORD_LEAST_LEN: int = 4 FORWARD_PORT_BEGIN: int = 10001 FORWARD_PORT_END: int = 11000 @@ -51,29 +56,26 @@ def cli(user_id: str = None, use_options: bool = False, list_schedule: bool = Fa print(str_format("InputError: Unknown account! Check your user account or connect to the Host Maintainer(MLOps).", fore='r')) return False - # check user_id is new user - if user_id not in checker.users_config.ids: - password = DEFAULT_PASSWORD - print(f"The password for the new user: {DEFAULT_PASSWORD}") - else: + # check old user_id password + if user_id in checker.users_config.ids: password = checker.users_config.ids[user_id].password - - isWrong = True # a flag for checking if tne login success - for _ in range(3): # There are three times chances for user to enter password correctly - input_password = getpass.getpass(prompt="Please enter the password: ") - if input_password == password: - isWrong = False # login successfully - print(str_format(f"Login Successfully!!", fore='g')) - break - else: - print("Wrong password!!") - if isWrong: - print("ByeBye~~") # login failed - return False + isWrong = True # a flag for checking if tne login success + for _ in range(3): # There are three times chances for user to enter password correctly + input_password = getpass.getpass(prompt="Please enter the password: ") + if input_password == password: + isWrong = False # login successfully + print(str_format(f"Login Successfully!!", fore='g')) + break + else: + print("Wrong password!!") + if isWrong: + print("ByeBye~~") # login failed + return False if user_id in checker.users_config.ids: user_config = copy(checker.users_config.ids[user_id]) else: + print(str_format("New User, Welcome~~~", fore='g')) user_config = __add_new_user_config(user_id) if use_options: @@ -97,6 +99,10 @@ def cli(user_id: str = None, use_options: bool = False, list_schedule: bool = Fa booking(user_id, cap_info, booking_time, user_config) + # * refresh users_config.yaml and save it. + # ? another mechanism is to use a period of time to update all of it, like monitor mechanism. + __update_users_config2yaml(UsersConfig(yaml_file=checker.deploy_info.users_config_yaml)) + def __get_caps_info(user_id: str) -> BasicCapability: if user_id in checker.cap_config.max_custom_capability: @@ -261,12 +267,8 @@ def __get_bookingtime(user_id: str) -> BasicCapability: return BookingTime(*start2end_datetime) -def __update_users_config_and_yaml(user_id: str, user_config: UserConfig): - # ! bug: someone use booking at the same time, it will be cover it, the user_config you want to update maybe loss. - # TODO: a better method is to append this user_config to the end of the yaml file, next time load it, it will get the latest info., and create a mechanism use a period of time to update all of it, like monitor mechanism. - checker.users_config.ids[user_id] = user_config - - write_yaml(checker.users_config.to_dict(), PROJECT_DIR / checker.deploy_info.users_config_yaml) +def __update_users_config2yaml(users_config: UsersConfig): + write_yaml(users_config.to_dict(), PROJECT_DIR / checker.deploy_info.users_config_yaml) def __increase_user_config_and_yaml(user_id: str, user_config: UserConfig): @@ -274,6 +276,25 @@ def __increase_user_config_and_yaml(user_id: str, user_config: UserConfig): append_yaml({user_id: user_config.to_dict()}, PROJECT_DIR / checker.deploy_info.users_config_yaml) +def __create_new_password() -> str: + while True: + new_password = getpass.getpass(prompt="Please enter the new Password: ") + if new_password == '': + print(str_format("Incorrect!! The password can't be empty!!", fore='r')) + continue + elif len(new_password) < PASSWORD_LEAST_LEN: + print(str_format(f"Incorrect!! The length of the password at least {PASSWORD_LEAST_LEN}!!", fore='r')) + continue + else: + check_new_password = getpass.getpass(prompt="Please enter the new password again: ") + + if new_password != check_new_password: + print(str_format("UpdatePasswordError: Two input passwords are not the same!!", fore='r')) + continue + break + return new_password + + def __add_new_user_config(user_id: str) -> UserConfig: while True: random_forward_ports = random.randint(FORWARD_PORT_BEGIN, FORWARD_PORT_END) @@ -281,7 +302,7 @@ def __add_new_user_config(user_id: str) -> UserConfig: break user_config = UserConfig( - password=DEFAULT_PASSWORD, + password=__create_new_password(), forward_port=random_forward_ports, image=None, extra_command=None, @@ -318,7 +339,13 @@ def __setting_forward_port(user_id: str, default_forward_port: int): return forward_port -def __setting_user_options(user_id: str, user_config: UserConfig): +def __setting_user_options(user_id: str, user_config: UsersConfig): + # Update Password + if ask_yn("Do you want to update the password?"): + user_config.password = __create_new_password() + __increase_user_config_and_yaml(user_id, user_config) + print(str_format("Update default Password!", fore='g')) + # Forward Port user_config.forward_port = __setting_forward_port(user_id, user_config.forward_port) @@ -349,26 +376,6 @@ def __setting_user_options(user_id: str, user_config: UserConfig): extra_command = input("Please enter the extra command when running the image. (default: None, none by default): ") user_config.extra_command = extra_command if extra_command != '' else None - # Update Password - isUpdate = ask_yn("Do you want to update the password?") - while isUpdate: - new_password = getpass.getpass(prompt="Please enter the new Password: ") - if new_password == '': - print(str_format("Incorrect!! The password can't be empty!!", fore='r')) - continue - else: - check_new_password = getpass.getpass(prompt="Please enter the new password again: ") - - if new_password != check_new_password: - print(str_format("UpdatePasswordError: Two input passwords are not the same!!", fore='r')) - continue - else: - user_config.password = new_password - checker.users_config.ids[user_id].password = new_password - __increase_user_config_and_yaml(user_id, checker.users_config.ids[user_id]) - print(str_format("Update default Password!", fore='g')) - break - # Update users_config.yaml if ask_yn("The previous setting is for the once, do you want to update the default config?"): __increase_user_config_and_yaml(user_id, user_config) @@ -416,7 +423,8 @@ def booking(user_id: str, cap_info: BasicCapability, booking_time: BookingTime, if __name__ == '__main__': # sys.argv = ['booking.py', '-id', 'm11007s05-2', '-use-opt'] # sys.argv = ['booking.py', '-use-opt'] - cli() + + cli(standalone_mode=True) # cap_info = __get_caps_info('m11007s05-3') From 3baecd54818f584b20550b1754d434a677646508 Mon Sep 17 00:00:00 2001 From: tw-yshuang Date: Fri, 13 Oct 2023 17:31:16 +0800 Subject: [PATCH 6/7] [mod] update docs/DEV_DOCUMENT.md --- docs/DEV_DOCUMENT.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/DEV_DOCUMENT.md b/docs/DEV_DOCUMENT.md index 30b2ac6..a538522 100644 --- a/docs/DEV_DOCUMENT.md +++ b/docs/DEV_DOCUMENT.md @@ -208,34 +208,34 @@ if use_options is True: #### 5. Optional(use_options=True) -#### 5.1. `forward_port` +#### 5.1. Update Password + +- "Do you want to update the password?", using `ask_yn()` to ask, return: + - `False`, pass it. + - `True`, "Please enter the new password: ", after entering, "Please enter the new password again: ", both new_password must be same. + - If there are not the same, (red-font)"Incorrect!!", back to [Q.5.1.](#51-update-password) + - Only update the password in `users_config.yaml`, (green-font)"Update default Password!" + +#### 5.2. `forward_port` - "Please enter the forward port(default: xxxxx, none by default): ", the default forward_port can find it from *`Checker.users_config.ids[user_id].forward_port`*. - The forward port only can assign port: `10000~11000`, due to application service port. please check [List of TCP and UDP port numbers](https://en.wikipedia.org/wiki/List_of_TCP_and_UDP_port_numbers). - Use *`Checker.check_forward_port_empty()`* to check the forward port is not duplicated. - - `False`, sent message (red-font)"Forward Port Duplicated!!", back to [Q.5.1.](#51-forward_port). + - `False`, sent message (red-font)"Forward Port Duplicated!!", back to [Q.5.2](#52-forward_port). -#### 5.2. `image` +#### 5.3. `image` - Using *`Checker.deploy_info.images`* to show the docker images first. - "Please enter the image 'repository/tag'(default: xxx, none by default): ", the default image can find it from *`Checker.users_config.ids[user_id].image`*, if is `None`, then show the image "rober5566a/aivc-server:latest". - If the response is "", then `Checker.users_config.ids[user_id].image = None`. - Using *`Checker.check_image_isexists()`* to check image is exists. -#### 5.3. `extra_command` +#### 5.4. `extra_command` - "Please enter the extra command when running the image. (default: None, none by default): ", no need to check. - Note: if the image repository is "rober5566a/aivc-server" actually it has an extra command: `/.script/ssh_start.sh {password}`, see [monitor/run_container.py](#run_containerpy). -#### 5.4. Update Password - -- "Do you want to update the password?", using `ask_yn()` to ask, return: - - `False`, pass it. - - `True`, "Please enter the new password: ", after entering, "Please enter the new password again: ", both new_password must be same. - - If there are not the same, (red-font)"Incorrect!!", back to [Q.5.4.](#54-update-password) - - Only update the password in `users_config.yaml`, (green-font)"Update default Password!" - #### 5.5. Update `users_config.yaml` - "The previous setting is for the once, do you want to update the default config?", using `ask_yn()` to ask, return: From 7b30575efd6cccfd853f05a5d537f59f0c81ef73 Mon Sep 17 00:00:00 2001 From: tw-yshuang Date: Fri, 13 Oct 2023 20:25:02 +0800 Subject: [PATCH 7/7] [adj] {src/booking/booking.py, src/monitor/Monitor.py}, remove useless comments --- src/booking/booking.py | 2 +- src/monitor/Monitor.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/booking/booking.py b/src/booking/booking.py index 291616d..b0b8509 100755 --- a/src/booking/booking.py +++ b/src/booking/booking.py @@ -51,7 +51,7 @@ def cli(user_id: str = None, use_options: bool = False, list_schedule: bool = Fa print(str_format("InputError: Unknown account! Check your user account or connect to the Host Maintainer(MLOps).", fore='r')) return False - # check old user_id password + # check old user_id's password if user_id in checker.users_config.ids: password = checker.users_config.ids[user_id].password isWrong = True # a flag for checking if tne login success diff --git a/src/monitor/Monitor.py b/src/monitor/Monitor.py index a3c284e..6b6a240 100644 --- a/src/monitor/Monitor.py +++ b/src/monitor/Monitor.py @@ -293,8 +293,6 @@ def exec(self) -> None: self.update_sdf(self.using, self.used, now_using, move2used, close_results, msg="using to used") if not (move2using is None or move2using.empty): - #! issue: When booking.py has someone write the data to the booking.csv during the Monitor.py running, Monitor.py will recover the action of booking.py - # TODO: add the hash-table for the booking.csv to check the write time, make sure the information can integrate! run_results = np.array([self.check_space(user_id) for user_id in move2using[SC.user_id]], dtype=np.bool_) run_results[run_results] = self.run_containers(move2using[run_results]) self.update_sdf(self.booking, self.using, now_booking, move2using, run_results, msg="booking to using")