Feat (deploy): Port localhost deployment code

autonomy/cli/deploy.py

@@ -22,7 +22,7 @@
 import os
 import shutil
 from pathlib import Path
-from typing import Optional, cast
+from typing import List, Optional, cast
 
 import click
 from aea import AEA_DIR
@@ -40,6 +40,7 @@
     build_and_deploy_from_token,
     build_deployment,
     run_deployment,
+    run_host_deployment,
     stop_deployment,
 )
 from autonomy.cli.helpers.env import load_env_file
@@ -60,6 +61,7 @@
 from autonomy.deploy.constants import INFO, LOGGING_LEVELS
 from autonomy.deploy.generators.docker_compose.base import DockerComposeGenerator
 from autonomy.deploy.generators.kubernetes.base import KubernetesGenerator
+from autonomy.deploy.generators.localhost.base import HostDeploymentGenerator
 
 
 def _validate_packages_path(path: Optional[Path] = None) -> Path:
@@ -130,12 +132,18 @@ def deploy_group(
     default=1,
     help="Number of services.",
 )
+@click.option(
+    "--localhost",
+    "deployment_type",
+    flag_value=HostDeploymentGenerator.deployment_type,
+    help="Use localhost as a backend.",
+)
 @click.option(
     "--docker",
     "deployment_type",
     flag_value=DockerComposeGenerator.deployment_type,
     default=True,
-    help="Use docker as a backend.",
+    help="Use docker as a backend. (default)",
 )
 @click.option(
     "--kubernetes",
@@ -215,6 +223,13 @@ def deploy_group(
     help="Set agent memory usage limit.",
     default=DEFAULT_AGENT_MEMORY_LIMIT,
 )
+@click.option(
+    "--mkdir",
+    type=str,
+    help="Directory names to create in the build directory.",
+    default=[],
+    multiple=True,
+)
 @registry_flag()
 @password_option(confirmation_prompt=True)
 @image_author_option
@@ -226,6 +241,7 @@ def build_deployment_command(  # pylint: disable=too-many-arguments, too-many-lo
     output_dir: Optional[Path],
     dev_mode: bool,
     registry: str,
+    mkdir: List[str],
     number_of_agents: Optional[int] = None,
     number_of_services: int = 1,
     password: Optional[str] = None,
@@ -290,6 +306,7 @@ def build_deployment_command(  # pylint: disable=too-many-arguments, too-many-lo
                 use_acn=use_acn,
                 use_tm_testnet_setup=use_tm_testnet_setup,
                 image_author=image_author,
+                mkdir=mkdir,
                 resources={
                     "agent": {
                         "limit": {"cpu": agent_cpu_limit, "memory": agent_memory_limit},
@@ -329,16 +346,42 @@ def build_deployment_command(  # pylint: disable=too-many-arguments, too-many-lo
     default=False,
     help="Run service in the background.",
 )
+@click.option(
+    "--localhost",
+    "deployment_type",
+    flag_value="localhost",
+    help="Use localhost as a backend.",
+)
+@click.option(
+    "--docker",
+    "deployment_type",
+    flag_value="docker",
+    help="Use docker as a backend. (default)",
+    default=True,
+)
 def run(
-    build_dir: Path, no_recreate: bool, remove_orphans: bool, detach: bool = False
+    build_dir: Path,
+    no_recreate: bool,
+    remove_orphans: bool,
+    detach: bool,
+    deployment_type: str,
 ) -> None:
     """Run deployment."""
     build_dir = Path(build_dir or Path.cwd()).absolute()
-    if not (build_dir / DockerComposeGenerator.output_name).exists():
+    deployment = (
+        HostDeploymentGenerator
+        if deployment_type == "localhost"
+        else DockerComposeGenerator
+    )
+    if not (build_dir / deployment.output_name).exists():
         raise click.ClickException(
             f"Deployment configuration does not exist @ {build_dir}"
         )
-    run_deployment(build_dir, no_recreate, remove_orphans, detach=detach)
+    click.echo(f"Running build @ {build_dir}")
+    if deployment_type == "localhost":
+        run_host_deployment(build_dir, detach)
+    else:
+        run_deployment(build_dir, no_recreate, remove_orphans, detach=detach)
 
 
 @deploy_group.command(name="stop")

autonomy/cli/helpers/deployment.py

@@ -18,13 +18,18 @@
 # ------------------------------------------------------------------------------
 
 """Deployment helpers."""
+import json
 import os
+import platform
 import shutil
+import subprocess  # nosec
+import sys
 import time
 from pathlib import Path
-from typing import Dict, List, Optional, Tuple
+from typing import Callable, Dict, List, Optional, Tuple
 
 import click
+from aea.configurations.constants import AGENT
 from aea.configurations.data_types import PublicId
 from aea.helpers.base import cd
 from compose.cli import main as docker_compose
@@ -45,30 +50,37 @@
 from autonomy.deploy.build import generate_deployment
 from autonomy.deploy.constants import (
     AGENT_KEYS_DIR,
+    AGENT_VARS_CONFIG_FILE,
     BENCHMARKS_DIR,
+    DEATTACH_WINDOWS_FLAG,
     INFO,
     LOG_DIR,
     PERSISTENT_DATA_DIR,
+    TENDERMINT_FLASK_APP_PATH,
+    TENDERMINT_VARS_CONFIG_FILE,
     TM_STATE_DIR,
     VENVS_DIR,
 )
 from autonomy.deploy.generators.kubernetes.base import KubernetesGenerator
+from autonomy.deploy.generators.localhost.utils import check_tendermint_version
 from autonomy.deploy.image import build_image
 
 
-def _build_dirs(build_dir: Path) -> None:
+def _build_dirs(build_dir: Path, mkdir: List[str]) -> None:
     """Build necessary directories."""
 
+    mkdirs = [(new_dir_name,) for new_dir_name in mkdir]
+
     for dir_path in [
         (PERSISTENT_DATA_DIR,),
         (PERSISTENT_DATA_DIR, LOG_DIR),
         (PERSISTENT_DATA_DIR, TM_STATE_DIR),
         (PERSISTENT_DATA_DIR, BENCHMARKS_DIR),
         (PERSISTENT_DATA_DIR, VENVS_DIR),
         (AGENT_KEYS_DIR,),
-    ]:
+    ] + mkdirs:
         path = Path(build_dir, *dir_path)
-        path.mkdir()
+        path.mkdir(exist_ok=True, parents=True)
         # TOFIX: remove this safely
         try:
             os.chown(path, 1000, 1000)
@@ -119,7 +131,6 @@ def run_deployment(
     detach: bool = False,
 ) -> None:
     """Run deployment."""
-    click.echo(f"Running build @ {build_dir}")
     try:
         project = _load_compose_project(build_dir=build_dir)
         commands = docker_compose.TopLevelCommand(project=project)
@@ -163,6 +174,65 @@ def run_deployment(
         stop_deployment(build_dir=build_dir)
 
 
+def _get_deattached_creation_flags() -> int:
+    """Get Popen creation flag based on the platform."""
+    return DEATTACH_WINDOWS_FLAG if platform.system() == "Windows" else 0
+
+
+def _start_localhost_agent(working_dir: Path, detach: bool) -> None:
+    """Start localhost agent process."""
+    env = json.loads((working_dir / AGENT_VARS_CONFIG_FILE).read_text())
+    process_fn: Callable = subprocess.Popen if detach else subprocess.run  # type: ignore[assignment]
+    process = process_fn(  # pylint: disable=subprocess-run-check # nosec
+        args=[sys.executable, "-m", "aea.cli", "run"],
+        cwd=working_dir / AGENT,
+        env={**os.environ, **env},
+        creationflags=_get_deattached_creation_flags(),  # Detach process from the main process
+    )
+    (working_dir / "agent.pid").write_text(
+        data=str(process.pid),
+    )
+
+
+def _start_localhost_tendermint(working_dir: Path) -> subprocess.Popen:
+    """Start localhost tendermint process."""
+    check_tendermint_version()
+    env = json.loads((working_dir / TENDERMINT_VARS_CONFIG_FILE).read_text())
+    flask_app_path = Path(__file__).parents[3] / TENDERMINT_FLASK_APP_PATH
+    process = subprocess.Popen(  # pylint: disable=consider-using-with # nosec
+        args=[
+            "flask",
+            "run",
+            "--host",
+            "localhost",
+            "--port",
+            "8080",
+        ],
+        cwd=working_dir,
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
+        env={**os.environ, **env, "FLASK_APP": f"{flask_app_path}:create_server"},
+        creationflags=_get_deattached_creation_flags(),  # Detach process from the main process
+    )
+    (working_dir / "tendermint.pid").write_text(
+        data=str(process.pid),
+    )
+    return process
+
+
+def run_host_deployment(build_dir: Path, detach: bool = False) -> None:
+    """Run host deployment."""
+    tm_process = _start_localhost_tendermint(build_dir)
+    try:
+        _start_localhost_agent(build_dir, detach)
+    except Exception as e:  # pylint: disable=broad-except
+        click.echo(e)
+        tm_process.terminate()
+    finally:
+        if not detach:
+            tm_process.terminate()
+
+
 def stop_deployment(build_dir: Path) -> None:
     """Stop running deployment."""
     try:
@@ -196,6 +266,7 @@ def build_deployment(  # pylint: disable=too-many-arguments, too-many-locals
     resources: Optional[Resources] = None,
     service_hash_id: Optional[str] = None,
     service_offset: int = 0,
+    mkdir: Optional[List[str]] = None,
 ) -> None:
     """Build deployment."""
 
@@ -209,8 +280,6 @@ def build_deployment(  # pylint: disable=too-many-arguments, too-many-locals
 
     click.echo(f"Building deployment @ {build_dir}")
     build_dir.mkdir()
-    _build_dirs(build_dir)
-
     if service_hash_id is None:
         service_hash_id = build_hash_id()
 
@@ -237,6 +306,9 @@ def build_deployment(  # pylint: disable=too-many-arguments, too-many-locals
         image_author=image_author,
         resources=resources,
     )
+    if mkdir is not None:
+        _build_dirs(build_dir, mkdir)
+
     click.echo(report)
 
 

autonomy/deploy/base.py

@@ -66,6 +66,7 @@
 
 KUBERNETES_DEPLOYMENT = "kubernetes"
 DOCKER_COMPOSE_DEPLOYMENT = "docker-compose"
+LOCALHOST_DEPLOYMENT = "localhost"
 
 LOOPBACK = "127.0.0.1"
 LOCALHOST = "localhost"
@@ -466,14 +467,14 @@ def _try_update_tendermint_params(
         """Try update the tendermint parameters"""
 
         is_kubernetes_deployment = self.deplopyment_type == KUBERNETES_DEPLOYMENT
+        is_localhost_deployment = self.deplopyment_type == LOCALHOST_DEPLOYMENT
 
         def _update_tendermint_params(
             param_args: Dict,
             idx: int,
-            is_kubernetes_deployment: bool = False,
         ) -> None:
             """Update tendermint params"""
-            if is_kubernetes_deployment:
+            if is_kubernetes_deployment or is_localhost_deployment:
                 param_args[TENDERMINT_URL_PARAM] = TENDERMINT_NODE_LOCAL
                 param_args[TENDERMINT_COM_URL_PARAM] = TENDERMINT_COM_LOCAL
             else:
@@ -504,7 +505,6 @@ def _update_tendermint_params(
                         _update_tendermint_params(
                             param_args=param_args,
                             idx=0,
-                            is_kubernetes_deployment=is_kubernetes_deployment,
                         )
                 else:
                     param_args = self._get_config_from_json_path(
@@ -513,7 +513,6 @@ def _update_tendermint_params(
                     _update_tendermint_params(
                         param_args=param_args,
                         idx=0,
-                        is_kubernetes_deployment=is_kubernetes_deployment,
                     )
                 return
 
@@ -530,7 +529,6 @@ def _update_tendermint_params(
                 _update_tendermint_params(
                     param_args=param_args,
                     idx=agent_idx,
-                    is_kubernetes_deployment=is_kubernetes_deployment,
                 )
         except KeyError:  # pragma: nocover
             logging.warning(
@@ -611,9 +609,9 @@ def _update_abci_connection_config(
         processed_overrides = deepcopy(overrides)
         if self.service.number_of_agents == 1:
             processed_overrides["config"]["host"] = (
-                LOOPBACK
-                if self.deplopyment_type == KUBERNETES_DEPLOYMENT
-                else self.get_abci_container_name(index=0)
+                self.get_abci_container_name(index=0)
+                if self.deplopyment_type == DOCKER_COMPOSE_DEPLOYMENT
+                else LOOPBACK
             )
             processed_overrides["config"]["port"] = processed_overrides["config"].get(
                 "port", DEFAULT_ABCI_PORT
@@ -627,9 +625,9 @@ def _update_abci_connection_config(
 
         for idx, override in processed_overrides.items():
             override["config"]["host"] = (
-                LOOPBACK
-                if self.deplopyment_type == KUBERNETES_DEPLOYMENT
-                else self.get_abci_container_name(index=idx)
+                self.get_abci_container_name(index=idx)
+                if self.deplopyment_type == DOCKER_COMPOSE_DEPLOYMENT
+                else LOOPBACK
             )
             override["config"]["port"] = override["config"].get(
                 "port", DEFAULT_ABCI_PORT

autonomy/deploy/build.py

@@ -24,11 +24,13 @@
 from autonomy.deploy.constants import DEPLOYMENT_REPORT, INFO
 from autonomy.deploy.generators.docker_compose.base import DockerComposeGenerator
 from autonomy.deploy.generators.kubernetes.base import KubernetesGenerator
+from autonomy.deploy.generators.localhost.base import HostDeploymentGenerator
 
 
 DEPLOYMENT_OPTIONS: Dict[str, Type[BaseDeploymentGenerator]] = {
     "kubernetes": KubernetesGenerator,
     "docker-compose": DockerComposeGenerator,
+    "localhost": HostDeploymentGenerator,
 }
 
 
@@ -56,9 +58,14 @@ def generate_deployment(  # pylint: disable=too-many-arguments, too-many-locals
     resources: Optional[Resources] = None,
 ) -> str:
     """Generate the deployment for the service."""
-    if dev_mode and type_of_deployment != DockerComposeGenerator.deployment_type:
+    if type_of_deployment == HostDeploymentGenerator.deployment_type:
+        if number_of_agents is not None and number_of_agents > 1:
+            raise RuntimeError(
+                "Host deployment currently only supports single agent deployments"
+            )
+    elif dev_mode:
         raise RuntimeError(
-            "Development mode can only be used with docker-compose deployments"
+            "Development mode can only be used with localhost deployments"
         )
 
     service_builder = ServiceBuilder.from_dir(