From 41f333b1b3ab5edb099e86918a7bb568b8b798d1 Mon Sep 17 00:00:00 2001 From: Jan Gutsche Date: Fri, 26 Jan 2024 21:51:08 +0100 Subject: [PATCH] Deploy: WIP fast mode --- scripts/deploy/deploy_robots.py | 16 ++++++- scripts/deploy/tasks/sync.py | 85 ++++++++++++++++++++++----------- 2 files changed, 73 insertions(+), 28 deletions(-) diff --git a/scripts/deploy/deploy_robots.py b/scripts/deploy/deploy_robots.py index faa3dcb6f..2dd5f1b34 100644 --- a/scripts/deploy/deploy_robots.py +++ b/scripts/deploy/deploy_robots.py @@ -10,6 +10,7 @@ print_bit_bot, print_debug, print_err, + print_info, print_known_targets, print_success, ) @@ -30,7 +31,7 @@ class DeployRobots: def __init__(self): self._bitbots_main_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - print_debug(f"Bit-Bots meta path: {self._bitbots_main_path}") + print_info(f"Bit-Bots main path: {self._bitbots_main_path}") os.chdir(self._bitbots_main_path) # Handle arguments @@ -98,6 +99,12 @@ def _parse_arguments(self) -> argparse.Namespace: ) # Optional arguments + parser.add_argument( + "-f", + "--fast", + action="store_true", + help="Don't build/compile on the target machines, instead synchronize the complete install directory from the local machine.", + ) parser.add_argument("-p", "--package", default="", help="Synchronize and build only the given ROS package.") parser.add_argument( "--clean", @@ -139,6 +146,12 @@ def _parse_arguments(self) -> argparse.Namespace: args.build = args.only_build args.launch = args.only_launch + if args.fast: + print_info("Fast mode enabled. We will sync the current local colcon workspace and skip the build task.") + args.sync = True + args.build = False + args.clean = True + if args.clean: args.clean_src = True args.clean_build = True @@ -178,6 +191,7 @@ def _register_tasks(self) -> list[AbstractTask]: self._args.workspace, self._args.package, self._args.clean_src, + self._args.fast, ) ) diff --git a/scripts/deploy/tasks/sync.py b/scripts/deploy/tasks/sync.py index 128902241..ea010cf80 100644 --- a/scripts/deploy/tasks/sync.py +++ b/scripts/deploy/tasks/sync.py @@ -10,28 +10,31 @@ class Sync(AbstractTask): - def __init__(self, local_workspace: str, remote_workspace: str, package: str = "", pre_clean: bool = False) -> None: + def __init__( + self, + local_src_bitbots_main: str, + remote_workspace: str, + package: str = "", + pre_clean: bool = False, + fast: bool = False, + ) -> None: """ Sync task that synchronizes (copies) the local source directory to the remote server. - :param local_workspace: Path to the local workspace to sync + :param _local_src_bitbots_main: Path to the local src/bitbots_main directory to sync :param remote_workspace: Path to the remote workspace to sync to :param package: Limit to file from this package, if empty, all files are included :param pre_clean: Whether to clean the source directory before syncing + :param fast: Whether to sync the current local colcon workspace including build artifacts """ super().__init__() - self._local_workspace = local_workspace + self._local_src_bitbots_main = local_src_bitbots_main self._remote_workspace = remote_workspace self._package = package self._pre_clean = pre_clean + self._fast = fast - self._remote_src_path = os.path.join(self._remote_workspace, "src") - - self._includes = self._get_includes_from_file( - os.path.join(self._local_workspace, "sync_includes_wolfgang_nuc.yaml"), self._package - ) - - def _get_includes_from_file(self, file_path: str, package: str = "") -> list: + def _get_includes_from_file(self, file_path: str, package: str = "") -> list[str]: """ Retrieve the include and exclude arguments for rsync from a yaml file. @@ -84,8 +87,34 @@ def _run(self, connections: Group) -> GroupResult: return clean_results connections = get_connections_from_succeeded(clean_results) - # Synchronize the local source directory to the remote workspace with rsync - rsync_results = self._rsync(connections) + if self._fast: + rsync_results = self._rsync( + connections, + os.path.dirname(os.path.dirname(self._local_src_bitbots_main)) + + "/", # NOTE: The trailing slash is important for rsync + [ + "'--include=- log'", + "'--include=+ build'", + "'--include=+ build/**'", + "'--include=+ install'", + "'--include=+ install/**'", + "'--include=+ src'", + "'--include=+ src/**'", + "'--include=- *'", + ], + self._remote_workspace, + ) + else: + # Synchronize the local source directory to the remote workspace with rsync + includes = self._get_includes_from_file( + os.path.join(self._local_src_bitbots_main, "sync_includes_wolfgang_nuc.yaml"), self._package + ) + rsync_results = self._rsync( + connections, + self._local_src_bitbots_main + "/", # NOTE: The trailing slash is important for rsync + includes, + os.path.join(self._remote_workspace, "src"), + ) return rsync_results def _clean(self, connections: Group) -> GroupResult: @@ -94,6 +123,7 @@ def _clean(self, connections: Group) -> GroupResult: :return: The results of the task. """ + remote_src_path = os.path.join(self._remote_workspace, "src") def _remove_directories_on_single_host(connection: Connection, result: Result) -> Optional[Result]: """Removes directories given in the output of the previous find command. @@ -117,13 +147,13 @@ def _remove_directories_on_single_host(connection: Connection, result: Result) - return Result(connection=connection, cmd=rm_cmd, exited=1, stdout="", stderr=str(e)) return rm_results - if self._package: # Package given, clean only the package directory + if self._package and not self._fast: # Package given, clean only the package directory # Only clean the package directory # To do this, we find sub-directories with the package name and remove them print_debug( - f"Searching for package '{self._package}' in remote source directory: '{self._remote_src_path} to clean it'" + f"Searching for package '{self._package}' in remote source directory: '{remote_src_path} to clean it'" ) - find_cmd = f'find {self._remote_src_path} -type d -not -path "*/.git/*" -name "{self._package}"' + find_cmd = f'find {remote_src_path} -type d -not -path "*/.git/*" -name "{self._package}"' print_debug(f"Calling '{find_cmd}'") try: find_results = connections.run(find_cmd, hide=hide_output()) @@ -160,8 +190,8 @@ def _remove_directories_on_single_host(connection: Connection, result: Result) - else: # No package given, clean the entire source directory # First, remove the source directory - print_debug(f"Removing source directory: '{self._remote_src_path}'") - rm_cmd = f"rm -rf {self._remote_src_path}" + print_debug(f"Removing source directory: '{remote_src_path}'") + rm_cmd = f"rm -rf {remote_src_path}" print_debug(f"Calling '{rm_cmd}'") try: @@ -177,8 +207,8 @@ def _remove_directories_on_single_host(connection: Connection, result: Result) - connections = get_connections_from_succeeded(rm_results) # Second, create an empty directory again - print_debug(f"Creating source directory: '{self._remote_src_path}'") - mkdir_cmd = f"mkdir -p {self._remote_src_path}" + print_debug(f"Creating source directory: '{remote_src_path}'") + mkdir_cmd = f"mkdir -p {remote_src_path}" print_debug(f"Calling '{mkdir_cmd}'") try: @@ -191,12 +221,15 @@ def _remove_directories_on_single_host(connection: Connection, result: Result) - return e.result return mkdir_results - def _rsync(self, connections: Group) -> GroupResult: + def _rsync(self, connections: Group, local_path: str, includes: list[str], remote_path: str) -> GroupResult: """ - Synchronize (copy) the local source directory to the remotes using the rsync tool. + Synchronize (copy) the local directory to the remotes using the rsync tool. This happens in parallel for all connections. :param connections: The connections to remote servers. + :param local_path: The path to the local directory to sync from. + :param includes: The include and exclude arguments for rsync. + :param remote_path: The path to the remote directory to sync to. :return: The results of the task. """ @@ -210,11 +243,11 @@ def _sync_single(connection: Connection) -> Optional[Result]: ] if not be_quiet(): cmd.append("--verbose") - cmd.extend(self._includes) + cmd.extend(includes) cmd.extend( [ - self._local_workspace + "/", # NOTE: The trailing slash is important for rsync - f"{connection.user}@{connection.host}:{self._remote_src_path}", + local_path, + f"{connection.user}@{connection.host}:{remote_path}", ] ) cmd = " ".join(cmd) @@ -238,9 +271,7 @@ def _sync_single(connection: Connection) -> Optional[Result]: connection=connection, ) - print_debug( - f"Synchronizing local source directory ('{self._local_workspace}') to the remote directory: '{self._remote_src_path}'" - ) + print_debug(f"Synchronizing local source directory ('{local_path}') to the remote directory: '{remote_path}'") # Collect results of the group results = GroupResult()