1) Add support for BOINC Universal Docker Application (BUDA).

    This lets you run arbitrary Docker applications using a single
    BOINC app (and one app version per platform).
    The Dockerfile and science executables are in the workunit.

    The script tools/submit_buda lets you test this.
    Basic tests were successful.

2) Unify the scripts for testing job submission
    The 'test' assimilators (sample_assimilate.py and sample_assimilator.cpp)
        do the same thing: they copy result files to
        <proj_dir>/results/<batch_id>/<wu_name> (if 1 output file)
        <proj_dir>/results/<batch_id>/<wu_name>_i (if >1 output file)
        where <batch_id> is 0 if WU is not in a batch
        ... and they write error code to <wu_name>_error if the WU errored out
    Scripts to submit jobs:
        submit_job
        submit_batch
        submit_buda
    Script to query jobs:
        query_job
    This works for either jobs or batches,
    as long as the app uses one of the above assimilators

3) Add plan class 'docker' to plan_class_spec.xml.sample

py/Boinc/setup_project.py

@@ -435,17 +435,20 @@ def dest(*dirs):
             'cancel_jobs',
             'create_work',
             'dbcheck_files_exist',
-            'demo_query',
-            'demo_submit',
             'dir_hier_move',
             'dir_hier_path',
             'grep_logs',
             'manage_privileges',
             'parse_config',
+            'query_job',
             'run_in_ops',
+            'sample_assimilate.py',
             'sign_executable',
             'stage_file',
             'stage_file_native',
+            'submit_batch',
+            'submit_buda',
+            'submit_job',
             'update_versions',
             'xadd',
         ]

sched/plan_class_spec.xml.sample

@@ -124,4 +124,8 @@
         <name>                  wsl         </name>
         <wsl/>
     </plan_class>
+    <plan_class>
+        <name>                  docker      </name>
+        <docker/>
+    </plan_class>
 </plan_classes>

sched/sample_assimilator.cpp

@@ -16,10 +16,13 @@
 // along with BOINC.  If not, see <http://www.gnu.org/licenses/>.
 
 // A sample assimilator that:
-// 1) if success, copy the output file(s) to a directory ('sample_results')
+// 1) if success, copy the output file(s) to a directory
+//      ../sample_results/batchid
 //      If 1 output file, its name is the WU name
-//      If >1 files, file i is named wuname_i
-// 2) if failure, append a message to an error log
+//      If >1 files, file i is named <wuname>_i
+// 2) if failure, write a message to <wuname>_error
+
+// Note: daemons like this run in project/tmp_<host>
 
 #include <vector>
 #include <string>
@@ -30,24 +33,20 @@
 #include "filesys.h"
 #include "sched_msgs.h"
 #include "validate_util.h"
-#include "sched_config.h"
 #include "assimilate_handler.h"
 
 using std::vector;
 using std::string;
 
-const char* outdir = "sample_results";
+const char* outdir = "../results";
 
-int write_error(char* p) {
-    static FILE* f = 0;
-    if (!f) {
-        char path[1024];
-        sprintf(path, "%s/errors", outdir);
-        f = fopen(config.project_path(path), "a");
-        if (!f) return ERR_FOPEN;
-    }
+int write_error(WORKUNIT &wu, char* p) {
+    char path[1024];
+    sprintf(path, "%s/%d/%s_error", outdir, wu.batch, wu.name);
+    FILE* f = fopen(path, "a");
+    if (!f) return ERR_FOPEN;
     fprintf(f, "%s", p);
-    fflush(f);
+    fclose(f);
     return 0;
 }
 
@@ -77,38 +76,38 @@ int assimilate_handler(
     char buf[1024];
     unsigned int i;
 
-    retval = boinc_mkdir(config.project_path(outdir));
+    retval = boinc_mkdir(outdir);
+    if (retval) return retval;
+    sprintf(buf, "%s/%d", outdir, wu.batch);
+    retval = boinc_mkdir(buf);
     if (retval) return retval;
 
     if (wu.canonical_resultid) {
         vector<OUTPUT_FILE_INFO> output_files;
-        const char *copy_path;
         get_output_file_infos(canonical_result, output_files);
         unsigned int n = output_files.size();
         bool file_copied = false;
         for (i=0; i<n; i++) {
             OUTPUT_FILE_INFO& fi = output_files[i];
             if (n==1) {
-                sprintf(buf, "%s/%s", outdir, wu.name);
+                sprintf(buf, "%s/%d/%s", outdir, wu.batch, wu.name);
             } else {
-                sprintf(buf, "%s/%s_%d", outdir, wu.name, i);
+                sprintf(buf, "%s/%d/%s_%d", outdir, wu.batch, wu.name, i);
             }
-            copy_path = config.project_path(buf);
-            retval = boinc_copy(fi.path.c_str() , copy_path);
+            retval = boinc_copy(fi.path.c_str() , buf);
             if (!retval) {
                 file_copied = true;
             }
         }
         if (!file_copied) {
-            sprintf(buf, "%s/%s_no_output_files", outdir, wu.name);
-            copy_path = config.project_path(buf);
-            FILE* f = fopen(copy_path, "w");
+            sprintf(buf, "%s/%d/%s_no_output_files", outdir, wu.batch, wu.name);
+            FILE* f = fopen(buf, "w");
             if (!f) return ERR_FOPEN;
             fclose(f);
         }
     } else {
-        sprintf(buf, "%s: 0x%x\n", wu.name, wu.error_mask);
-        return write_error(buf);
+        sprintf(buf, "0x%x\n", wu.error_mask);
+        return write_error(wu, buf);
     }
     return 0;
 }

sched/script_assimilator.cpp

@@ -28,16 +28,14 @@
 //
 // files        list of output files of the job's canonical result
 // wu_id        workunit ID
+// wu_name      workunit name
 // result_id    ID of the canonical result
 // runtime      runtime of the canonical result
 // batch_id     the job's batch ID
 //
-// if no args are specified, the script is invoked as
-// scriptname wu_id files
-//
 // If the workunit has no canonical result (i.e. it failed)
 // the script is invoked as
-// scriptname --error N wu_id
+// scriptname --error N wu_name wu_id batch_id
 // where N is an integer encoding the reasons for the job's failure
 // (see WU_ERROR_* in html/inc/common_defs.inc)
 
@@ -111,6 +109,12 @@ int assimilate_handler(
             } else if (s == "wu_id") {
                 sprintf(buf, " %lu", wu.id);
                 strcat(cmd, buf);
+            } else if (s == "wu_name") {
+                sprintf(buf, " %s", wu.name);
+                strcat(cmd, buf);
+            } else if (s == "result_id") {
+                sprintf(buf, " %lu", wu.canonical_resultid);
+                strcat(cmd, buf);
             } else if (s == "runtime") {
                 sprintf(buf, " %f", canonical_result.elapsed_time);
                 strcat(cmd, buf);

tools/Makefile.am

@@ -17,11 +17,14 @@ toolbin_PROGRAMS = \
 dist_toolbin_SCRIPTS = \
 	    boinc_submit \
 		dbcheck_files_exist \
-		demo_submit \
-		demo_query \
 		grep_logs \
 		make_project \
 		parse_config \
+		query_job \
+		sample_assimilate.py \
+		submit_batch \
+		submit_buda \
+		submit_job \
 		update_versions \
 		upgrade \
 		vote_monitor \

tools/demo_query → tools/query_job

@@ -1,23 +1,18 @@
 #! /usr/bin/env php
 
 <?php
-// query a job created with demo_submit
+// query a job or batch for an app that uses sample assimilator
 //
-// usage: bin/demo_query jobname
-
+// usage: bin/query_job jobname|batchid
 
 chdir("html/ops");
 require_once("../inc/boinc_db.inc");
+require_once("../inc/submit_db.inc");
 require_once("../inc/common_defs.inc");
 require_once("../inc/result.inc");
 chdir("../..");
 
-function main($wu_name) {
-    $wu = BoincWorkunit::lookup("name='$wu_name'");
-    if (!$wu) {
-        die("no such job: $wu_name\n");
-    }
-
+function show_wu($wu, $dir) {
     if ($wu->error_mask) {
         echo sprintf("Job failed: %s\n",
             wu_error_mask_str($wu->error_mask)
@@ -46,22 +41,45 @@ function main($wu_name) {
         $nofs = $ofs->count();
         for ($i=0; $i<$nofs; $i++) {
             if ($nofs == 1) {
-                $path = "sample_results/$wu_name";
+                $path = "$dir/$wu->name";
             } else {
-                $path = sprintf("sample_results/%s_%d", $wu_name, $i);
+                $path = sprintf("$dir/%s_%d", $wu->name, $i);
             }
             if (!is_file($path)) {
-                die("output file is missing: $path\n");
+                die("output file $i is missing: $path\n");
             }
             echo "Output file $i ($path):\n";
-            readfile($path);
+            $x = file_get_contents($path);
+            if (strlen($x) > 256) {
+                $x = substr($x, 0, 256);
+                $x .= '...';
+            }
+            echo "$x\n";
         }
         break;
     }
 }
 
 if ($argc != 2) {
-    die("usage: demo_query jobname\n");
+    die("usage: demo_query jobname|batchid\n");
+}
+$x = $argv[1];
+if (is_numeric($x)) {
+    $id = (int)$x;
+    $batch = BoincBatch::lookup_id($id);
+    if (!$batch) {
+        die("no such batch\n");
+    }
+    $wus = BoincWorkunit::enum("batch=$id");
+    foreach ($wus as $wu) {
+        echo "-------------------\nWorkunit $wu->id:\n";
+        show_wu($wu, "results/$id");
+    }
+} else {
+    $wu = BoincWorkunit::lookup("name='$x'");
+    if (!$wu) {
+        die("no such job: $wu_name\n");
+    }
+    show_wu($wu, "results/$wu->batch");
 }
-main($argv[1]);
 ?>