Skip to content

Commit

Permalink
Deploy: WIP fast mode
Browse files Browse the repository at this point in the history
  • Loading branch information
@jaagut
jaagut committed Jan 26, 2024
1 parent 17fd700 commit 41f333b
Show file tree
Hide file tree
Showing 2 changed files with 73 additions and 28 deletions.
16 changes: 15 additions & 1 deletion scripts/deploy/deploy_robots.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
print_bit_bot,
print_debug,
print_err,
print_info,
print_known_targets,
print_success,
)
Expand All @@ -30,7 +31,7 @@
class DeployRobots:
def __init__(self):
self._bitbots_main_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
print_debug(f"Bit-Bots meta path: {self._bitbots_main_path}")
print_info(f"Bit-Bots main path: {self._bitbots_main_path}")
os.chdir(self._bitbots_main_path)

# Handle arguments
Expand Down Expand Up @@ -98,6 +99,12 @@ def _parse_arguments(self) -> argparse.Namespace:
)

# Optional arguments
parser.add_argument(
"-f",
"--fast",
action="store_true",
help="Don't build/compile on the target machines, instead synchronize the complete install directory from the local machine.",
)
parser.add_argument("-p", "--package", default="", help="Synchronize and build only the given ROS package.")
parser.add_argument(
"--clean",
Expand Down Expand Up @@ -139,6 +146,12 @@ def _parse_arguments(self) -> argparse.Namespace:
args.build = args.only_build
args.launch = args.only_launch

if args.fast:
print_info("Fast mode enabled. We will sync the current local colcon workspace and skip the build task.")
args.sync = True
args.build = False
args.clean = True

if args.clean:
args.clean_src = True
args.clean_build = True
Expand Down Expand Up @@ -178,6 +191,7 @@ def _register_tasks(self) -> list[AbstractTask]:
self._args.workspace,
self._args.package,
self._args.clean_src,
self._args.fast,
)
)

Expand Down
85 changes: 58 additions & 27 deletions scripts/deploy/tasks/sync.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,28 +10,31 @@


class Sync(AbstractTask):
def __init__(self, local_workspace: str, remote_workspace: str, package: str = "", pre_clean: bool = False) -> None:
def __init__(
self,
local_src_bitbots_main: str,
remote_workspace: str,
package: str = "",
pre_clean: bool = False,
fast: bool = False,
) -> None:
"""
Sync task that synchronizes (copies) the local source directory to the remote server.
:param local_workspace: Path to the local workspace to sync
:param _local_src_bitbots_main: Path to the local src/bitbots_main directory to sync
:param remote_workspace: Path to the remote workspace to sync to
:param package: Limit to file from this package, if empty, all files are included
:param pre_clean: Whether to clean the source directory before syncing
:param fast: Whether to sync the current local colcon workspace including build artifacts
"""
super().__init__()
self._local_workspace = local_workspace
self._local_src_bitbots_main = local_src_bitbots_main
self._remote_workspace = remote_workspace
self._package = package
self._pre_clean = pre_clean
self._fast = fast

self._remote_src_path = os.path.join(self._remote_workspace, "src")

self._includes = self._get_includes_from_file(
os.path.join(self._local_workspace, "sync_includes_wolfgang_nuc.yaml"), self._package
)

def _get_includes_from_file(self, file_path: str, package: str = "") -> list:
def _get_includes_from_file(self, file_path: str, package: str = "") -> list[str]:
"""
Retrieve the include and exclude arguments for rsync from a yaml file.
Expand Down Expand Up @@ -84,8 +87,34 @@ def _run(self, connections: Group) -> GroupResult:
return clean_results
connections = get_connections_from_succeeded(clean_results)

# Synchronize the local source directory to the remote workspace with rsync
rsync_results = self._rsync(connections)
if self._fast:
rsync_results = self._rsync(
connections,
os.path.dirname(os.path.dirname(self._local_src_bitbots_main))
+ "/", # NOTE: The trailing slash is important for rsync
[
"'--include=- log'",
"'--include=+ build'",
"'--include=+ build/**'",
"'--include=+ install'",
"'--include=+ install/**'",
"'--include=+ src'",
"'--include=+ src/**'",
"'--include=- *'",
],
self._remote_workspace,
)
else:
# Synchronize the local source directory to the remote workspace with rsync
includes = self._get_includes_from_file(
os.path.join(self._local_src_bitbots_main, "sync_includes_wolfgang_nuc.yaml"), self._package
)
rsync_results = self._rsync(
connections,
self._local_src_bitbots_main + "/", # NOTE: The trailing slash is important for rsync
includes,
os.path.join(self._remote_workspace, "src"),
)
return rsync_results

def _clean(self, connections: Group) -> GroupResult:
Expand All @@ -94,6 +123,7 @@ def _clean(self, connections: Group) -> GroupResult:
:return: The results of the task.
"""
remote_src_path = os.path.join(self._remote_workspace, "src")

def _remove_directories_on_single_host(connection: Connection, result: Result) -> Optional[Result]:
"""Removes directories given in the output of the previous find command.
Expand All @@ -117,13 +147,13 @@ def _remove_directories_on_single_host(connection: Connection, result: Result) -
return Result(connection=connection, cmd=rm_cmd, exited=1, stdout="", stderr=str(e))
return rm_results

if self._package: # Package given, clean only the package directory
if self._package and not self._fast: # Package given, clean only the package directory
# Only clean the package directory
# To do this, we find sub-directories with the package name and remove them
print_debug(
f"Searching for package '{self._package}' in remote source directory: '{self._remote_src_path} to clean it'"
f"Searching for package '{self._package}' in remote source directory: '{remote_src_path} to clean it'"
)
find_cmd = f'find {self._remote_src_path} -type d -not -path "*/.git/*" -name "{self._package}"'
find_cmd = f'find {remote_src_path} -type d -not -path "*/.git/*" -name "{self._package}"'
print_debug(f"Calling '{find_cmd}'")
try:
find_results = connections.run(find_cmd, hide=hide_output())
Expand Down Expand Up @@ -160,8 +190,8 @@ def _remove_directories_on_single_host(connection: Connection, result: Result) -

else: # No package given, clean the entire source directory
# First, remove the source directory
print_debug(f"Removing source directory: '{self._remote_src_path}'")
rm_cmd = f"rm -rf {self._remote_src_path}"
print_debug(f"Removing source directory: '{remote_src_path}'")
rm_cmd = f"rm -rf {remote_src_path}"

print_debug(f"Calling '{rm_cmd}'")
try:
Expand All @@ -177,8 +207,8 @@ def _remove_directories_on_single_host(connection: Connection, result: Result) -
connections = get_connections_from_succeeded(rm_results)

# Second, create an empty directory again
print_debug(f"Creating source directory: '{self._remote_src_path}'")
mkdir_cmd = f"mkdir -p {self._remote_src_path}"
print_debug(f"Creating source directory: '{remote_src_path}'")
mkdir_cmd = f"mkdir -p {remote_src_path}"

print_debug(f"Calling '{mkdir_cmd}'")
try:
Expand All @@ -191,12 +221,15 @@ def _remove_directories_on_single_host(connection: Connection, result: Result) -
return e.result
return mkdir_results

def _rsync(self, connections: Group) -> GroupResult:
def _rsync(self, connections: Group, local_path: str, includes: list[str], remote_path: str) -> GroupResult:
"""
Synchronize (copy) the local source directory to the remotes using the rsync tool.
Synchronize (copy) the local directory to the remotes using the rsync tool.
This happens in parallel for all connections.
:param connections: The connections to remote servers.
:param local_path: The path to the local directory to sync from.
:param includes: The include and exclude arguments for rsync.
:param remote_path: The path to the remote directory to sync to.
:return: The results of the task.
"""

Expand All @@ -210,11 +243,11 @@ def _sync_single(connection: Connection) -> Optional[Result]:
]
if not be_quiet():
cmd.append("--verbose")
cmd.extend(self._includes)
cmd.extend(includes)
cmd.extend(
[
self._local_workspace + "/", # NOTE: The trailing slash is important for rsync
f"{connection.user}@{connection.host}:{self._remote_src_path}",
local_path,
f"{connection.user}@{connection.host}:{remote_path}",
]
)
cmd = " ".join(cmd)
Expand All @@ -238,9 +271,7 @@ def _sync_single(connection: Connection) -> Optional[Result]:
connection=connection,
)

print_debug(
f"Synchronizing local source directory ('{self._local_workspace}') to the remote directory: '{self._remote_src_path}'"
)
print_debug(f"Synchronizing local source directory ('{local_path}') to the remote directory: '{remote_path}'")

# Collect results of the group
results = GroupResult()
Expand Down

0 comments on commit 41f333b

Please sign in to comment.