Merge pull request #5923 from BOINC/dpa_buda4

Web: job submission enhancements

html/inc/bootstrap.inc

@@ -500,8 +500,14 @@ function form_submit($text, $attrs='') {
 function form_checkbox($label, $name, $checked=false) {
     echo sprintf('
         <div class="form-group">
-            <input type="checkbox" name="%s" %s> &nbsp; <span class="lead">%s</span>
+            <label align=right class="%s">%s</label>
+            <div class="%s">
+        ',
+        FORM_LEFT_CLASS, $label, FORM_RIGHT_CLASS
+    );
+    echo sprintf('
+        <input type="checkbox" name="%s" %s>
         </div>
-        ', $name, $checked?"checked":"", $label
+        ', $name, $checked?"checked":""
     );
 }

html/inc/submit_util.inc

@@ -78,7 +78,7 @@ function has_submit_access($user, $app_id) {
 function has_admin_access($user, $app_id) {
     $us = BoincUserSubmit::lookup_userid($user->id);
     if (!$us) return false;
-    if ($us->admin_all) return true;
+    if ($us->manage_all) return true;
     $usa = BoincUserSubmitApp::lookup("user_id=$user->id and app_id=$app_id");
     if (!$usa) return false;
     return $usa->manage;
@@ -126,11 +126,23 @@ function delete_remote_submit_user($user) {
     BoincUserSubmitApp::delete_user($user->id);
 }
 
-
-// given its WUs, compute progress of a batch
-// (fraction done, est completion time etc.)
-// NOTE: this is inefficient because we need all the WUs.
-// it could be done by server components
+// given its WUs, compute parameters of a batch:
+//   credit_canonical: credit granted to canonical instances
+//   fraction_done: frac of jobs that are done (success or failed)
+//   state: whether complete (all jobs done)
+//   completion_time: if newly complete
+//   nerror_jobs: # of failed jobs
+// Update the above in DB.
+// Also compute (not in DB):
+//   njobs_success: # of jobs with canonical instance
+//   njobs_in_prog: # of jobs not success or fail,
+//      and at least one result in progress
+//
+// return the batch object, with these values
+//
+// NOTE: this involves reading the batch's WUs and results,
+// which could be inefficient for huge batches.
+// It could instead be done by server components
 // (transitioner, validator etc.) as jobs complete or time out
 //
 // TODO: update est_completion_time
@@ -141,51 +153,61 @@ function get_batch_params($batch, $wus) {
         //
         return $batch;
     }
+    if (!$wus) {
+        if ($batch->njobs) {
+            $batch->update('njobs=0');
+            $batch->njobs = 0;
+        }
+        return $batch;
+    }
+
+    // make list of WU IDs with an in-progress result
+    $res_in_prog = BoincResult::enum(
+        sprintf('batch=%d and server_state<>%d',
+            $batch->id, RESULT_SERVER_STATE_IN_PROGRESS
+        )
+    );
+    $wus_in_prog = [];
+    foreach ($res_in_prog as $res) {
+        $wus_in_prog[$res->workunitid] = true;
+    }
+
     $fp_total = 0;
     $fp_done = 0;
     $completed = true;
     $batch->nerror_jobs = 0;
     $batch->credit_canonical = 0;
+    $njobs_success = 0;
+    $njobs_in_prog = 0;
     foreach ($wus as $wu) {
         $fp_total += $wu->rsc_fpops_est;
         if ($wu->canonical_resultid) {
             $fp_done += $wu->rsc_fpops_est;
+            $njobs_success++;
             $batch->credit_canonical += $wu->canonical_credit;
         } else if ($wu->error_mask) {
             $batch->nerror_jobs++;
         } else {
             $completed = false;
+            if (array_key_exists($wu->id, $wus_in_prog)) {
+                $njobs_in_prog++;
+            }
         }
     }
-    if ($fp_total) {
-        $batch->fraction_done = $fp_done / $fp_total;
-    }
+    $njobs = count($wus);
+    $batch->njobs = $njobs;
+    $batch->fraction_done = ($njobs_success + $batch->nerror_jobs)/$batch->njobs;
     if ($completed && $batch->state == BATCH_STATE_IN_PROGRESS) {
         $batch->state = BATCH_STATE_COMPLETE;
         $batch->completion_time = time();
     }
-    $batch->update("fraction_done = $batch->fraction_done, nerror_jobs = $batch->nerror_jobs, state=$batch->state, completion_time = $batch->completion_time, credit_canonical = $batch->credit_canonical");
+    $batch->update("fraction_done = $batch->fraction_done, nerror_jobs = $batch->nerror_jobs, state=$batch->state, completion_time = $batch->completion_time, credit_canonical = $batch->credit_canonical, njobs=$njobs");
 
-    $batch->credit_estimate = flops_to_credit($fp_total);
+    $batch->njobs_success = $njobs_success;
+    $batch->njobs_in_prog = $njobs_in_prog;
     return $batch;
 }
 
-// get the number of WUs for which we've sent at least 1 instance
-// TODO: do this more efficiently (single query)
-//
-function wus_nsent($wus) {
-    $n = 0;
-    foreach ($wus as $wu) {
-        $res = BoincResult::enum(
-            sprintf('workunitid=%d and server_state<>%d',
-                $wu->id, RESULT_SERVER_STATE_UNSENT
-            )
-        );
-        if (count($res) > 0) $n++;
-    }
-    return $n;
-}
-
 // get the physical names of a result's output files.
 //
 function get_outfile_phys_names($result) {

html/user/buda_submit.php

@@ -35,18 +35,22 @@ function submit_form($user) {
     if (!is_valid_filename($variant)) die('bad arg');
 
     $desc = "<br><small>
-        A zipped directory with one subdirectory per job,
-        containing the input file(s) for that job
+        A zip file with one directory per job.
+        Each directory contains the input file(s) for that job
         and an optional file <code>cmdline</code>
         containing command-line arguments.
         <a href=https://github.com/BOINC/boinc/wiki/BUDA-job-submission>Details</a></small>.
     ";
+    $desc2 = "<br><small>
+        Write Docker commands and output to stderr (for debugging).
+    ";
     page_head("Submit jobs to $app ($variant)");
     form_start('buda_submit.php');
     form_input_hidden('action', 'submit');
     form_input_hidden('app', $app);
     form_input_hidden('variant', $variant);
     form_select("Batch zip file $desc", 'batch_file', $sbitems_zip);
+    form_checkbox("Verbose Docker output? $desc2", 'wrapper_verbose');
     form_submit('OK');
     form_end();
     page_tail();
@@ -179,7 +183,7 @@ function stage_input_files($batch_dir, $batch_desc, $batch_id) {
 }
 
 function create_jobs(
-    $variant_desc, $batch_desc, $batch_id, $batch_dir_name
+    $variant_desc, $batch_desc, $batch_id, $batch_dir_name, $wrapper_verbose
 ) {
     global $buda_app;
 
@@ -205,8 +209,10 @@ function create_jobs(
         $job_cmds .= "$job_cmd\n";
     }
     $cmd = sprintf(
-        'cd ../..; bin/create_work --appname %s --batch %d --stdin --command_line "--dockerfile %s --verbose" --wu_template %s --result_template %s',
-        $buda_app->name, $batch_id, $variant_desc->dockerfile,
+        'cd ../..; bin/create_work --appname %s --batch %d --stdin --command_line "--dockerfile %s %s" --wu_template %s --result_template %s',
+        $buda_app->name, $batch_id,
+        $variant_desc->dockerfile,
+        $wrapper_verbose?'--verbose':'',
         "buda_batches/$batch_dir_name/template_in",
         "buda_batches/$batch_dir_name/template_out"
     );
@@ -307,6 +313,7 @@ function handle_submit($user) {
     if (!is_valid_filename($variant)) die('bad arg');
     $batch_file = get_str('batch_file');
     if (!is_valid_filename($batch_file)) die('bad arg');
+    $wrapper_verbose = get_str('wrapper_verbose', true);
 
     $variant_dir = "../../buda_apps/$app/$variant";
     $variant_desc = json_decode(
@@ -331,7 +338,8 @@ function handle_submit($user) {
     stage_input_files($batch_dir, $batch_desc, $batch->id);
 
     create_jobs(
-        $variant_desc, $batch_desc, $batch->id, $batch_dir_name
+        $variant_desc, $batch_desc, $batch->id, $batch_dir_name,
+        $wrapper_verbose
     );
 
     // mark batch as in progress

html/user/get_output.php

@@ -52,7 +52,7 @@ function get_output_file($instance_name, $file_num, $auth_str) {
         return_error("bad authenticator");
     }
 
-    $names = get_outfile_names($result);
+    $names = get_outfile_phys_names($result);
     if ($file_num >= count($names)) {
         return_error("bad file num: $file_num > ".count($names));
     }
@@ -105,7 +105,7 @@ function get_batch_output_files($auth_str) {
     foreach ($wus as $wu) {
         if (!$wu->canonical_resultid) continue;
         $result = BoincResult::lookup_id($wu->canonical_resultid);
-        $names = get_outfile_names($result);
+        $names = get_outfile_phys_names($result);
         foreach ($names as $name) {
             $path = dir_hier_path($name, $upload_dir, $fanout);
             if (is_file($path)) {
@@ -143,7 +143,7 @@ function get_wu_output_file($wu_name, $file_num, $auth_str) {
         return_error("no canonical result for wu $wu->name");
     }
     $result = BoincResult::lookup_id($wu->canonical_resultid);
-    $names = get_outfile_names($result);
+    $names = get_outfile_phys_names($result);
     $path = dir_hier_path($names[$file_num], $upload_dir, $fanout);
     if (file_exists($path)) {
         do_download($path);
@@ -181,7 +181,7 @@ function get_wu_output_files($wu_id, $auth_str) {
         return_error("no canonical result for wu $wu->name");
     }
     $result = BoincResult::lookup_id($wu->canonical_resultid);
-    $names = get_outfile_names($result);
+    $names = get_outfile_phys_names($result);
     foreach ($names as $name) {
         $path = dir_hier_path($name, $upload_dir, $fanout);
         if (is_file($path)) {

html/user/get_output2.php

@@ -52,7 +52,7 @@ function check_auth($auth, $batch) {
 }
 
 function do_result_aux($result, $batch, $file_num=null) {
-    $phys_names = get_outfile_names($result);
+    $phys_names = get_outfile_phys_names($result);
     $log_names = get_outfile_log_names($result);
     if ($file_num !== null) {
         $path = upload_path($phys_names[$file_num]);
@@ -123,7 +123,7 @@ function do_batch($batch_id, $auth) {
     $wus = BoincWorkunit::enum("batch=$batch_id and canonical_resultid<>0");
     foreach ($wus as $wu) {
         $result = BoincResult::lookup_id($wu->canonical_resultid);
-        $phys_names = get_outfile_names($result);
+        $phys_names = get_outfile_phys_names($result);
         $log_names = get_outfile_log_names($result);
         if (count($phys_names) == 1) {
             $cmd = sprintf('ln -s %s %s/%s__%s',

html/user/job_file.php

@@ -99,7 +99,7 @@ function upload_error_description($errno) {
 
 function query_files($r) {
     xml_start_tag("query_files");
-    list($user, $user_submit) = check_remote_submit_permissions($r, null);
+    $user = check_remote_submit_permissions($r, null);
     $absent_files = array();
     $now = time();
     $delete_time = (int)$r->delete_time;
@@ -177,7 +177,7 @@ function delete_uploaded_files() {
 
 function upload_files($r) {
     xml_start_tag("upload_files");
-    list($user, $user_submit) = check_remote_submit_permissions($r, null);
+    $user = check_remote_submit_permissions($r, null);
     $fanout = parse_config(get_config(), "<uldl_dir_fanout>");
     $delete_time = (int)$r->delete_time;
     $batch_id = (int)$r->batch_id;

html/user/sandbox.php

@@ -44,9 +44,9 @@ function list_files($user, $notice) {
         <h3>Upload files</h3>
         <p>
         NOTE: if you upload text files from Windows,
-        they will have CRLF line endings.
-        If they are shell scripts, they won't work on Linux.
-        Add shell scripts using Add File.
+        they will be given CRLF line endings.
+        Then, if they are shell scripts, they won't work on Linux.
+        Add shell scripts using 'Add text file' below.
         <p>
 
         <form action=sandbox.php method=post ENCTYPE=\"multipart/form-data\">
@@ -55,7 +55,7 @@ function list_files($user, $notice) {
         <p> <input class=\"btn btn-success\" type=submit value=Upload>
         </form>
         <hr>
-        <h3>Add file</h3>
+        <h3>Add text file</h3>
     ";
     form_start('sandbox.php', 'post');
     form_input_hidden('action', 'add_file');
@@ -180,7 +180,7 @@ function view_file($user) {
 }
 
 $user = get_logged_in_user();
-if (!submit_permissions($user)) error_page("no job submission access");
+if (!has_file_access($user)) error_page("no job submission access");
 
 $action = get_str('action', true);
 if (!$action) $action = post_str('action', true);