Setup Telegraf in provisioner (#279)

* Setup telegraf in provisioner

* fix uploading files

* remove get file for now

* reboot server during provision when required

* add tests for telegraf

* add buildbuddy test for teleport

* disable buildbuddy devcontainer

.bazelrc

@@ -25,37 +25,37 @@ common --enable_bzlmod=true
 build --@io_bazel_rules_docker//transitions:enable=false
 
 # Enable builds without the bytes https://github.com/bazelbuild/bazel/issues/6862
-build:remote --remote_download_minimal
+build:buildbuddy --remote_download_minimal
 
 # From https://www.buildbuddy.io/docs/cloud
 # Use BuildBuddy
-build --bes_results_url=https://app.buildbuddy.io/invocation/
-build --bes_backend=grpcs://remote.buildbuddy.io
-build --remote_cache=grpcs://remote.buildbuddy.io
-build --remote_timeout=3600
+build:buildbuddy --bes_results_url=https://app.buildbuddy.io/invocation/
+build:buildbuddy --bes_backend=grpcs://remote.buildbuddy.io
+build:buildbuddy --remote_cache=grpcs://remote.buildbuddy.io
+build:buildbuddy --remote_timeout=3600
 
 # Suggestions from BuildBuddy
-build --experimental_remote_build_event_upload=minimal
-build --experimental_remote_cache_compression
-build --experimental_stream_log_file_uploads
+build:buildbuddy --experimental_remote_build_event_upload=minimal
+build:buildbuddy --experimental_remote_cache_compression
+build:buildbuddy --experimental_stream_log_file_uploads
 common --nolegacy_important_outputs
 
 # For BuildBuddy Remote Build Execution
-build:remote --remote_executor=grpcs://remote.buildbuddy.io
-build:remote --host_platform=@buildbuddy_toolchain//:platform
-build:remote --platforms=@buildbuddy_toolchain//:platform
-build:remote --extra_execution_platforms=@buildbuddy_toolchain//:platform
-build:remote --crosstool_top=@buildbuddy_toolchain//:toolchain
-build:remote --extra_toolchains=@buildbuddy_toolchain//:cc_toolchain
-build:remote --javabase=@buildbuddy_toolchain//:javabase_jdk8
-build:remote --host_javabase=@buildbuddy_toolchain//:javabase_jdk8
-build:remote --java_toolchain=@buildbuddy_toolchain//:toolchain_jdk8
-build:remote --host_java_toolchain=@buildbuddy_toolchain//:toolchain_jdk8
-build:remote --define=EXECUTOR=remote
-build:remote --jobs=50
+build:buildbuddy_rbe --remote_executor=grpcs://remote.buildbuddy.io
+build:buildbuddy_rbe --host_platform=@buildbuddy_toolchain//:platform
+build:buildbuddy_rbe --platforms=@buildbuddy_toolchain//:platform
+build:buildbuddy_rbe --extra_execution_platforms=@buildbuddy_toolchain//:platform
+build:buildbuddy_rbe --crosstool_top=@buildbuddy_toolchain//:toolchain
+build:buildbuddy_rbe --extra_toolchains=@buildbuddy_toolchain//:cc_toolchain
+build:buildbuddy_rbe --javabase=@buildbuddy_toolchain//:javabase_jdk8
+build:buildbuddy_rbe --host_javabase=@buildbuddy_toolchain//:javabase_jdk8
+build:buildbuddy_rbe --java_toolchain=@buildbuddy_toolchain//:toolchain_jdk8
+build:buildbuddy_rbe --host_java_toolchain=@buildbuddy_toolchain//:toolchain_jdk8
+build:buildbuddy_rbe --define=EXECUTOR=remote
+build:buildbuddy_rbe --jobs=50
 # To make the remote build work from macbook M1 (https://buildbuddy.slack.com/archives/CUHBFVATU/p1651609573695259?thread_ts=1651609249.864879&cid=CUHBFVATU)
-build:remote --host_cpu=k8
-build:remote --cpu=k8
+build:buildbuddy_rbe --host_cpu=k8
+build:buildbuddy_rbe --cpu=k8
 
 # The CI has some specific configuration which we only want to load in the CI
 try-import %workspace%/ci.bazelrc

buildbuddy.yaml

@@ -10,7 +10,17 @@ actions:
         branches:
           - "*"
     bazel_commands:
-      - "test //... @rules_task//... --config remote"
+      - "test //... @rules_task//... --config buildbuddy --config buildbuddy_rbe"
+
+  - name: "Test Teleport connection"
+    user: buildbuddy
+    container_image: "ubuntu-20.04"
+    triggers:
+      pull_request:
+        branches:
+          - "*"
+    bazel_commands:
+      - "run //tools/teleport:connection_test --config buildbuddy"
 
   - name: "Deploy provisioner"
     user: buildbuddy
@@ -20,4 +30,4 @@ actions:
         branches:
           - "master"
     bazel_commands:
-      - "run //provisioner:deploy"
+      - "run //provisioner:deploy --config buildbuddy"

provisioner/BUILD.bazel

@@ -8,21 +8,27 @@ pyinfra_run(
     srcs = [
         "connectors/teleport.py",
         "deploys/microk8s/tasks/install_microk8s.py",
+        "deploys/monitoring/tasks/install_monitoring.py",
         "deploys/network/tasks/install_network.py",
         "deploys/teleport/tasks/install_teleport.py",
         "group_data/dev.py",
         "group_data/prod.py",
         "group_data/test.py",
+        "utils.py",
     ],
     args = [
         "--data install_network=True",
+        "--data install_monitoring=True",
         "--data install_microk8s=True",
         "--data install_teleport=True",
     ],
     data = [
         "deploys/microk8s/files/cmdline.txt",
+        "deploys/monitoring/files/json_file_output.conf",
+        "deploys/monitoring/files/telegraf.conf",
         "deploys/network/files/99_config.yaml",
         "deploys/teleport/files/teleport.yaml.j2",
+        "deploys/teleport/files/teleport_health_check.conf",
     ],
     deploy = "deploy.py",
     env = {

provisioner/connectors/teleport.py

@@ -70,23 +70,6 @@ def ssh(self, command):
         ]
         return StringCommand(*args)
 
-    def scp_upload(self, local_file, remote_file):
-        remote_file = f"{self.teleport_host}:{remote_file}"
-        return self.scp(local_file, remote_file)
-
-    def scp_download(self, remote_file, local_file):
-        remote_file = f"{self.teleport_host}:{remote_file}"
-        return self.scp(remote_file, local_file)
-
-    def scp(self, from_file, to_file):
-        args = [
-            *self._get_connection_args(),
-            "scp",
-            from_file,
-            to_file,
-        ]
-        return StringCommand(*args)
-
     def _get_connection_args(self):
         args = [self.tsh_binary]
 
@@ -167,14 +150,27 @@ def run_shell_command(
     )
 
 
-def _put_file(host, filename_or_io, temp_file):
+def _put_file(host, state, filename_or_io, temp_file):
     teleport_client = host.connector_data["teleport_client"]
-    local.shell(
-        teleport_client.scp_upload(
-            local_file=filename_or_io,
-            remote_file=temp_file,
+
+    with get_file_io(filename_or_io) as file_io:
+        stdin = file_io.read().decode("utf-8")
+
+        teleport_command = teleport_client.ssh(
+            command=f"'cat > {temp_file}'"
         ).get_raw_value()
-    )
+
+        run_local_shell_command(
+            state,
+            host,
+            teleport_command,
+            timeout=10,
+            stdin=stdin,
+            success_exit_codes=[0],
+            print_output=False,
+            print_input=False,
+            return_combined_output=False,
+        )
 
 
 # Inspired by https://github.com/Fizzadar/pyinfra/blob/6eca1a52d955a0497cd33c02cb9a94176f93583d/pyinfra/connectors/ssh.py#L493
@@ -203,7 +199,7 @@ def put_file(
     if sudo or doas or su_user:
         # Get temp file location
         temp_file = remote_temp_filename or state.get_temp_filename(remote_filename)
-        _put_file(host, filename_or_io, temp_file)
+        _put_file(host, state, filename_or_io, temp_file)
 
         # Make sure our sudo/su user can access the file
         if su_user:
@@ -290,16 +286,6 @@ def put_file(
     return True
 
 
-def _get_file(host: "Host", remote_filename: str, filename_or_io):
-    teleport_client = host.connector_data["teleport_client"]
-    local.shell(
-        teleport_client.scp_download(
-            local_file=filename_or_io,
-            remote_file=remote_filename,
-        ).get_raw_value()
-    )
-
-
 def get_file(
     state: "State",
     host: "Host",
@@ -313,66 +299,4 @@ def get_file(
     print_input: bool = False,
     **command_kwargs,
 ):
-    """
-    Download a file from the remote host using SFTP. Supports download files
-    with sudo by copying to a temporary directory with read permissions,
-    downloading and then removing the copy.
-    """
-
-    if sudo or su_user:
-        # Get temp file location
-        temp_file = remote_temp_filename or state.get_temp_filename(remote_filename)
-
-        # Copy the file to the tempfile location and add read permissions
-        command = "cp {0} {1} && chmod +r {0}".format(remote_filename, temp_file)
-
-        copy_status, _, stderr = run_shell_command(
-            state,
-            host,
-            command,
-            sudo=sudo,
-            sudo_user=sudo_user,
-            su_user=su_user,
-            print_output=print_output,
-            print_input=print_input,
-            **command_kwargs,
-        )
-
-        if copy_status is False:
-            logger.error("File download copy temp error: {0}".format("\n".join(stderr)))
-            return False
-
-        try:
-            _get_file(host, temp_file, filename_or_io)
-
-        # Ensure that, even if we encounter an error, we (attempt to) remove the
-        # temporary copy of the file.
-        finally:
-            remove_status, _, stderr = run_shell_command(
-                state,
-                host,
-                "rm -f {0}".format(temp_file),
-                sudo=sudo,
-                sudo_user=sudo_user,
-                su_user=su_user,
-                print_output=print_output,
-                print_input=print_input,
-                **command_kwargs,
-            )
-
-        if remove_status is False:
-            logger.error(
-                "File download remove temp error: {0}".format("\n".join(stderr))
-            )
-            return False
-
-    else:
-        _get_file(host, remote_filename, filename_or_io)
-
-    if print_output:
-        click.echo(
-            "{0}file downloaded: {1}".format(host.print_prefix, remote_filename),
-            err=True,
-        )
-
-    return True
+    raise NotImplementedError

provisioner/deploy.py

@@ -1,11 +1,32 @@
 from provisioner.deploys.network.tasks.install_network import install_network
 from provisioner.deploys.microk8s.tasks.install_microk8s import install_microk8s
 from provisioner.deploys.teleport.tasks.install_teleport import install_teleport
+from provisioner.deploys.monitoring.tasks.install_monitoring import install_monitoring
+from provisioner.utils import wait_for_reconnect
+
 from pyinfra import host
+from pyinfra.facts.files import File
+from pyinfra.operations import server
+
+if host.get_fact(File, "/var/run/reboot-required"):
+    server.reboot(
+        name="Reboot required. Rebooting the server and wait to reconnect",
+        delay=60,
+        reboot_timeout=600,
+        _sudo=True,
+        _ignore_errors=True,  # restarting the service will disconnect us and result in error
+    )
+
+    wait_for_reconnect(
+        name="Wait for teleport to reconnect",
+    )
 
 if host.data.get("install_network"):
     install_network()
 
+if host.data.get("install_monitoring"):
+    install_monitoring()
+
 if host.data.get("install_teleport"):
     install_teleport()
 

provisioner/deploys/monitoring/files/json_file_output.conf

@@ -0,0 +1,7 @@
+[[outputs.file]]
+  files = ["/tmp/metrics.out"]
+  data_format = "json"
+  json_timestamp_units = "1s"
+  rotation_interval = "1h"
+  rotation_max_size = "250MB"
+  rotation_max_archives = 24