client: remove unused Docker images and containers on startup

client/client_state.cpp

@@ -15,6 +15,8 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with BOINC.  If not, see <http://www.gnu.org/licenses/>.
 
+// client initialization and main loop
+
 #ifdef _WIN32
 #include "boinc_win.h"
 #else
@@ -729,6 +731,8 @@ int CLIENT_STATE::init() {
 
     process_gpu_exclusions();
 
+    docker_cleanup();
+
     check_clock_reset();
 
     // Check to see if we can write the state file.

client/client_state.h

@@ -378,6 +378,7 @@ struct CLIENT_STATE {
     int app_finished(ACTIVE_TASK&);
     bool handle_finished_apps();
     void check_overdue();
+    void docker_cleanup();
 
     ACTIVE_TASK* get_task(RESULT*);
 

client/cs_apps.cpp

@@ -1,6 +1,6 @@
 // This file is part of BOINC.
-// http://boinc.berkeley.edu
-// Copyright (C) 2008 University of California
+// https://boinc.berkeley.edu
+// Copyright (C) 2024 University of California
 //
 // BOINC is free software; you can redistribute it and/or modify it
 // under the terms of the GNU Lesser General Public License
@@ -16,7 +16,7 @@
 // along with BOINC.  If not, see <http://www.gnu.org/licenses/>.
 
 // The "policy" part of task execution is here.
-// The "mechanism" part is in app.C
+// The "mechanism" part is in app.cpp
 //
 
 #include "cpp.h"
@@ -29,11 +29,14 @@
 #include <csignal>
 #endif
 
+#include <algorithm>
+
 #include "error_numbers.h"
 #include "filesys.h"
 #include "md5_file.h"
 #include "shmem.h"
 #include "util.h"
+#include "url.h"
 
 #include "client_msgs.h"
 #include "client_state.h"
@@ -382,3 +385,109 @@ void CLIENT_STATE::check_overdue() {
     active_tasks.report_overdue();
     t = now + 86400;
 }
+
+////////////// DOCKER CLEANUP ///////////////////
+
+// lists of image and container names for active jobs
+//
+struct DOCKER_JOB_INFO {
+    vector<string> images;
+    vector<string> containers;
+    bool image_present(string name) {
+        return std::find(images.begin(), images.end(), name) != images.end();
+    }
+    bool container_present(string name) {
+        return std::find(containers.begin(), containers.end(), name) != containers.end();
+    }
+};
+
+// clean up a Docker installation
+// (Unix: the host; Win: a WSL distro)
+//
+void cleanup_docker(DOCKER_JOB_INFO &info, DOCKER_CONN &dc) {
+    int retval;
+    vector<string> out, out2;
+    char cmd[1024];
+    string name;
+
+    // first containers
+    //
+    retval = dc.command("ps --all", out);
+    if (retval) {
+        fprintf(stderr, "Docker command failed: ps --all\n");
+    } else {
+        for (string line: out) {
+            retval = dc.parse_container_name(line, name);
+            if (retval) continue;
+            if (!docker_is_boinc_name(name.c_str())) continue;
+            if (info.container_present(name)) continue;
+            sprintf(cmd, "rm %s", name.c_str());
+            retval = dc.command(cmd, out);
+            if (retval) {
+                fprintf(stderr, "Docker command failed: %s\n", cmd);
+                continue;
+            }
+            msg_printf(NULL, MSG_INFO,
+                "Removed unused Docker container: %s", name.c_str()
+            );
+        }
+    }
+
+    // then images
+    //
+    retval = dc.command("images", out);
+    if (retval) {
+        fprintf(stderr, "Docker command failed: images\n");
+    } else {
+        for (string line: out) {
+            retval = dc.parse_image_name(line, name);
+            if (retval) continue;
+            if (!docker_is_boinc_name(name.c_str())) continue;
+            if (info.image_present(name)) continue;
+            sprintf(cmd, "image rm %s", name.c_str());
+            retval = dc.command(cmd, out2);
+            if (retval) {
+                fprintf(stderr, "Docker command failed: %s\n", cmd);
+                continue;
+            }
+            msg_printf(NULL, MSG_INFO,
+                "Removed unused Docker image: %s", name.c_str()
+            );
+        }
+    }
+}
+
+// remove old BOINC images and containers from Docker installations
+//
+void CLIENT_STATE::docker_cleanup() {
+    // make lists of the images and containers used by active jobs
+    //
+    DOCKER_JOB_INFO info;
+    for (ACTIVE_TASK *atp: active_tasks.active_tasks) {
+        if (!strstr(atp->app_version->plan_class, "docker")) continue;
+        char buf[256];
+        escape_project_url(atp->wup->project->master_url, buf);
+        string s = docker_image_name(buf, atp->wup->name);
+        info.images.push_back(s);
+        s = docker_container_name(buf, atp->result->name);
+        info.containers.push_back(s);
+    }
+
+    // go through local Docker installations and remove
+    // BOINC images and containers not in the above lists
+    //
+#ifdef _WIN32
+    for (WSL_DISTRO &wd: host_info.wsl_distros.distros) {
+        if (wd.docker_version.empty()) continue;
+        DOCKER_CONN dc;
+        dc.init(wd.docker_type, wd.distro_name);
+        cleanup_docker(info, dc);
+    }
+#else
+    if (strlen(host_info.docker_version)) {
+        DOCKER_CONN dc;
+        dc.init(host_info.docker_type, false);
+        cleanup_docker(info, dc);
+    }
+#endif
+}

lib/util.cpp

@@ -63,6 +63,7 @@ extern "C" {
 #include "mfile.h"
 #include "miofile.h"
 #include "parse.h"
+#include "hostinfo.h"
 #include "util.h"
 
 using std::min;
@@ -698,3 +699,115 @@ string parse_ldd_libc(const char* input) {
     strip_whitespace(s);
     return s;
 }
+
+#ifdef _WIN32
+int DOCKER_CONN::init(DOCKER_TYPE docker_type, string distro_name, bool _verbose) {
+    cli_prog = docker_cli_prog(docker_type);
+    if (docker_type == DOCKER) {
+        int retval = ctl_wc.setup();
+        if (retval) return retval;
+        retval = ctl_wc.run_program_in_wsl(distro_name, "", true);
+        if (retval) return retval;
+    } else if (docker_type == PODMAN) {
+        int retval = ctl_wc.setup_root(distro_name.c_str());
+        if (retval) return retval;
+    } else {
+        return -1;
+    }
+    verbose = _verbose;
+    return 0;
+}
+#else
+int DOCKER_CONN::init(DOCKER_TYPE docker_type, bool _verbose) {
+    cli_prog = docker_cli_prog(docker_type);
+    verbose = _verbose;
+    return 0;
+}
+#endif
+
+int DOCKER_CONN::command(const char* cmd, vector<string> &out) {
+    char buf[1024];
+    int retval;
+    if (verbose) {
+        fprintf(stderr, "running docker command: %s\n", cmd);
+    }
+#ifdef _WIN32
+    string output;
+
+    sprintf(buf, "%s %s; echo EOM\n", cli_prog, cmd);
+    write_to_pipe(ctl_wc.in_write, buf);
+    retval = read_from_pipe(
+        ctl_wc.out_read, ctl_wc.proc_handle, output, TIMEOUT, "EOM"
+    );
+    if (retval) {
+        fprintf(stderr, "read_from_pipe() error: %s\n", boincerror(retval));
+        return retval;
+    }
+    out = split(output, '\n');
+#else
+    sprintf(buf, "%s %s\n", cli_prog, cmd);
+    retval = run_command(buf, out);
+    if (retval) {
+        if (verbose) {
+            fprintf(stderr, "command failed: %s\n", boincerror(retval));
+        }
+        return retval;
+    }
+#endif
+    if (verbose) {
+        fprintf(stderr, "command output:\n");
+        for (string line: out) {
+            fprintf(stderr, "%s\n", line.c_str());
+        }
+    }
+    return 0;
+}
+
+// REPOSITORY                          TAG         IMAGE ID      CREATED       SIZE
+// localhost/boinc__app_test__test_wu  latest      cbc1498dfc49  43 hours ago  121 MB
+
+int DOCKER_CONN::parse_image_name(string line, string &name) {
+    char buf[1024];
+    strcpy(buf, line.c_str());
+    if (strstr(buf, "REPOSITORY")) return -1;
+    if (strstr(buf, "localhost/") != buf) return -1;
+    char *p = buf + strlen("localhost/");
+    char *q = strstr(p, " ");
+    if (!q) return -1;
+    *q = 0;
+    name = (string)p;
+    return 0;
+}
+
+// CONTAINER ID  IMAGE                                      COMMAND               CREATED        STATUS                   PORTS       NAMES
+// 6d4877e0d071  localhost/boinc__app_test__test_wu:latest  /bin/sh -c ./work...  43 hours ago   Exited (0) 21 hours ago              boinc__app_test__test_result
+
+int DOCKER_CONN::parse_container_name(string line, string &name) {
+    char buf[1024];
+    strcpy(buf, line.c_str());
+    if (strstr(buf, "CONTAINER")) return -1;
+    char *p = strrchr(buf, ' ');
+    if (!p) return -1;
+    name = (string)(p+1);
+    return 0;
+}
+
+string docker_image_name(
+    const char* proj_url_esc, const char* wu_name
+) {
+    char buf[1024];
+    sprintf(buf, "boinc__%s__%s", proj_url_esc, wu_name);
+    return string(buf);
+}
+
+string docker_container_name(
+    const char* proj_url_esc, const char* result_name
+){
+    char buf[1024];
+    sprintf(buf, "boinc__%s__%s", proj_url_esc, result_name);
+    return string(buf);
+}
+
+bool docker_is_boinc_name(const char* name) {
+    return strstr(name, "boinc__") == name;
+}

lib/util.h

@@ -25,7 +25,10 @@
 #include <vector>
 #ifdef _WIN32
 #include "boinc_win.h"
+#include "win_util.h"
 #endif
+#include "common_defs.h"
+
 extern double dtime();
 extern double dday();
 extern void boinc_sleep(double);
@@ -142,4 +145,39 @@ extern double simtime;
 #define time(x) ((int)simtime)
 #endif
 
+// represents a connection to a Docker/Podman installation
+// used from docker_wrapper and the client
+//
+struct DOCKER_CONN {
+    DOCKER_TYPE type;
+    const char* cli_prog;
+    bool verbose;
+#ifdef _WIN32
+    WSL_CMD ctl_wc;
+    int init(DOCKER_TYPE type, std::string distro_name, bool verbose);
+#else
+    int init(DOCKER_TYPE, bool verbose);
+#endif
+    int command(const char* cmd, std::vector<std::string> &out);
+
+    static const int TIMEOUT = 10;    // timeout for docker commands
+
+    // parse a line from "docker images" output; return name
+    int parse_image_name(std::string line, std::string &name);
+
+    // parse a line from "docker ps --all" output; return name
+    int parse_container_name(std::string line, std::string &name);
+};
+
+extern std::string docker_image_name(
+    const char* proj_url_esc,       // escaped project URL
+    const char* wu_name
+);
+extern std::string docker_container_name(
+    const char* proj_url_esc,       // escaped project URL
+    const char* result_name
+);
+// is the name (of a Docker image or container) a BOINC name?
+extern bool docker_is_boinc_name(const char* name);
+
 #endif

lib/win_util.cpp

@@ -244,7 +244,7 @@ int WSL_CMD::run_program_in_wsl(
     return (ret == S_OK)?0:-1;
 }
 
-PIPE_READ_RET read_from_pipe(
+int read_from_pipe(
     HANDLE pipe, HANDLE proc_handle, string& out, double timeout,
     const char* eom
 ) {
@@ -258,23 +258,23 @@ PIPE_READ_RET read_from_pipe(
         PeekNamedPipe(pipe, NULL, 0, NULL, &avail, NULL);
         if (avail) {
             ret = ReadFile(pipe, buf, sizeof(buf) - 1, &nread, NULL);
-            if (!ret) return READ_ERROR;
+            if (!ret) return ERR_READ;
             buf[nread] = 0;
             out += buf;
             if (eom) {
                 if (out.find(eom) != std::string::npos) {
-                    return GOT_EOM;
+                    return 0;
                 }
             }
         } else {
             if (exited) {
-                return PROC_DIED;
+                return ERR_CONNECT;
             }
             Sleep(200);
             if (timeout) {
                 elapsed += .2;
                 if (elapsed > timeout) {
-                    return TIMEOUT;
+                    return ERR_TIMEOUT;
                 }
             }
             if (proc_handle) {

lib/win_util.h

@@ -68,16 +68,14 @@ struct WSL_CMD {
     );
 };
 
-enum PIPE_READ_RET {GOT_EOM, PROC_DIED, TIMEOUT, READ_ERROR};
-
 // read from the pipe until either
 // - we get the eom string (if any)
 //      If you want to read at least 1 line, use "\n"
 // - there's no more data and the given process (if any) doesn't exist
 // - there's no more data and the given timeout (if any) is reached
 // - a read fails
 //
-extern PIPE_READ_RET read_from_pipe(
+extern int read_from_pipe(
     HANDLE pipe,
     HANDLE proc_handle,
     std::string& out,