From bc4cb3bb6853c968cb7d151d2f86b45e81a7da48 Mon Sep 17 00:00:00 2001 From: David Anderson Date: Mon, 18 Nov 2024 13:43:44 -0800 Subject: [PATCH 01/12] - Add web interface for creating BUDA apps and variants variants are stored in /buda_apps//. This includes app files (copied from sandbox) and templates (generated by handler) - Add web interface for submitting BUDA jobs (not finished) - Change implementation of user file sandbox old: sandbox dir had 'link files' containing md5 and size; actual file is in download hierarchy with sb_md5 name new: sandbox dir has actual files. parallel .md5/ dir has 'info files' (md5 size) Files are not stored in download hierarchy. New philosophy: names in the download hierarchy include not only an MD5 (for uniqueness) but also text describing the use of the file (input file for a batch, part of a BUDA app, etc.). This may allow duplicate files, but it makes it possible to always clean up unused files. - use readdir() instead of opendir()/scandir() --- html/inc/db_conn.inc | 6 +- html/inc/dir_hier.inc | 21 ++- html/inc/sandbox.inc | 166 ++++++------------- html/inc/submit_util.inc | 36 +++++ html/inc/util.inc | 6 +- html/user/buda.php | 330 ++++++++++++++++++++++++++++++++++++++ html/user/buda_submit.php | 73 +++++++++ html/user/sandbox.php | 113 ++++--------- tools/submit_buda | 37 +++-- 9 files changed, 568 insertions(+), 220 deletions(-) create mode 100644 html/user/buda.php create mode 100644 html/user/buda_submit.php diff --git a/html/inc/db_conn.inc b/html/inc/db_conn.inc index b39ae8d36f5..37605a16e6f 100644 --- a/html/inc/db_conn.inc +++ b/html/inc/db_conn.inc @@ -116,8 +116,8 @@ class DbConn { function enum_general($classname, $query) { $result = $this->do_query($query); - if (!$result) return null; - $x = array(); + if (!$result) return []; + $x = []; while ($obj = $result->fetch_object($classname)) { $x[] = $obj; } @@ -133,7 +133,7 @@ class DbConn { $where_clause = "where $where_clause"; } $query = "select $fields from DBNAME.$table $where_clause $order_clause"; - return $this->enum_general($classname,$query); + return $this->enum_general($classname, $query); } function enum($table, $classname, $where_clause=null, $order_clause=null) { diff --git a/html/inc/dir_hier.inc b/html/inc/dir_hier.inc index ea898bab91b..9f3b54fd045 100644 --- a/html/inc/dir_hier.inc +++ b/html/inc/dir_hier.inc @@ -120,9 +120,7 @@ function check_download_file($path, $dl_path) { } } -// Stage the given file, which is assumed to not be in download dir already. -// -function stage_file_basic($dir, $fname) { +function get_hier_info() { static $dl_dir = null; static $fanout = null; if (!$dl_dir) { @@ -130,6 +128,13 @@ function stage_file_basic($dir, $fname) { $dl_dir = parse_config($conf, ""); $fanout = parse_config($conf, ""); } + return [$dl_dir, $fanout]; +} + +// Stage the given file, which is assumed to not be in download dir already. +// +function stage_file_basic($dir, $fname) { + [$dl_dir, $fanout] = get_hier_info(); $old_path = "$dir/$fname"; $new_path = dir_hier_path($fname, $dl_dir, $fanout); $md5 = md5_file($old_path); @@ -138,4 +143,14 @@ function stage_file_basic($dir, $fname) { rename($old_path, $new_path); } +// copy the given file (with known size/md5) +// to the download dir w/ given phys name +// +function stage_file_aux($path, $md5, $size, $phys_name) { + [$dl_dir, $fanout] = get_hier_info(); + $phys_path = dir_hier_path($phys_name, $dl_dir, $fanout); + copy($path, $phys_path); + file_put_contents("$phys_path.md5", "$md5 $size\n"); +} + ?> diff --git a/html/inc/sandbox.inc b/html/inc/sandbox.inc index 2746048f43f..620c8e4d5a0 100644 --- a/html/inc/sandbox.inc +++ b/html/inc/sandbox.inc @@ -19,14 +19,23 @@ // Utility functions for user file sandbox feature // // In this system: -// - sandbox files live in the download hierarchy, -// with names of the form sb_userid_md5 -// - each user has a "sandbox dir" project/sandbox/userid. -// The files in this have user-visible names, and contents of the form -// sb file_size file_md5 +// - each user (job submitter) has a 'sandbox' where they can store files +// on the BOINC server, via a web interface. +// These files are mutable; you can modify a file w/ a given name. +// - files are stored in a dir project/sandbox/ +// - When a file is uploaded, its size and MD5 are computed and stored +// in an 'info file' in a parallel dir, project/sandbox//.md5 +// +// Sandbox files can be used for web-based job submissions systems +// like BUDA and autodock on BOINC Central. +// Typically they are used as job input files or app files, +// in which case they are downloadable. +// When a file is used in this way, +// it must be copied to the download hierarchy, +// and assigned a physical name that includes its MD5. +// The name depends on the role of the file. require_once("../inc/util.inc"); -require_once("../inc/submit_db.inc"); require_once("../inc/dir_hier.inc"); // Return path of sandbox directory for the given user. @@ -44,75 +53,53 @@ function sandbox_dir($user) { if (!is_dir($d)) { mkdir($d); } + if (!is_dir("$d/.md5")) { + mkdir("$d/.md5"); + } return $d; } -function sandbox_write_link_file($path, $size, $md5) { - file_put_contents($path, "sb $size $md5"); -} - -// check if a link with the given md5 exists +// parse a sandbox file's info file. +// If missing, create it. // -function sandbox_lf_exists($user, $md5) { - $exist = false; - $elf = ""; +function sandbox_parse_info_file($user, $name) { $dir = sandbox_dir($user); - $files = sandbox_file_names($user); - foreach ($files as $file) { - $path = "$dir/$file"; - [$err, $file_size, $file_md5] = sandbox_parse_link_file($path); - if (!$err){ - if (strcmp($md5, $file_md5) == 0) { - $exist = true; - $elf = $file; - break; - } - } + $info_path = "$dir/.md5/$name"; + $info = parse_info_file($info_path); + if ($info) { + return $info; } - return array($exist, $elf); -} - -// parse a link file and return (error, size, md5) -// -function sandbox_parse_link_file($path) { - if (!file_exists($path)) { return array(true, null, null); } - $x = file_get_contents($path); - $n = sscanf($x, "%s %d %s", $s, $size, $md5); - if ($n != 3) return array(true, null, null); - if ($s != 'sb') return array(true, null, null); - return array(false, $size, $md5); -} - -$fanout = parse_config(get_config(), ""); - -// return the physical name of the file -// -function sandbox_file_name($user, $md5) { - return "sb_".$user->id."_".$md5; -} - -// return the path of the file in the download directory -// -function sandbox_physical_path($user, $md5) { - global $fanout; - $f = sandbox_file_name($user, $md5); - return dir_hier_path($f, parse_config(get_config(), ""), $fanout); + [$size, $md5] = get_file_info("$dir/$name"); + write_info_file($info_path, $md5, $size); + return [$md5, $size]; } // return list of files in sandbox // function sandbox_file_names($user) { - $d = opendir(sandbox_dir($user)); + $files = scandir(sandbox_dir($user)); $names = array(); - while (($f = readdir($d)) !== false) { - if ($f == '.') continue; - if ($f == '..') continue; + foreach ($files as $f) { + if ($f[0] == '.') continue; $names[] = $f; } natsort($names); return $names; } +// return list of files matching given pattern, +// in the format used for form_select() and form_select_multiple() +// +function sandbox_select_items($user, $pattern=null) { + $sbfiles = sandbox_file_names($user); + $sbitems = []; + foreach ($sbfiles as $f) { + if ($pattern && !preg_match($pattern, $f)) continue; + $sbitems[] = [$f, $f]; + } + return $sbitems; +} + // return a - Upload files to your sandbox:

+

Sandbox contents

"; $files = array(); - while (($f = readdir($d)) !== false) { - if ($f == '.') continue; - if ($f == '..') continue; + foreach (scandir($dir) as $f) { + if ($f[0] == '.') continue; $files[] = $f; } if (count($files) == 0) { @@ -66,19 +60,10 @@ function list_files($user, $notice) { } else { sort($files); start_table(); - table_header("Name

(click to view)

", "Modified", "Size (bytes)", "MD5", "Delete","Download"); + table_header("Name

(click to view text files)

", "Modified", "Size (bytes)", "MD5", "Delete","Download"); foreach ($files as $f) { + [$md5, $size] = sandbox_parse_info_file($user, $f); $path = "$dir/$f"; - list($error, $size, $md5) = sandbox_parse_link_file($path); - if ($error) { - table_row($f, "Can't parse link file", "", "delete"); - continue; - } - $p = sandbox_physical_path($user, $md5); - if (!is_file($p)) { - table_row($f, "Physical file not found", "", ""); - continue; - } $ct = time_str(filemtime($path)); table_row( "$f", @@ -100,10 +85,13 @@ function list_files($user, $notice) { page_tail(); } +// upload one or more files + function upload_file($user) { $notice = ""; + $dir = sandbox_dir($user); $count = count($_FILES['new_file']['tmp_name']); - for ($i = 0; $i < $count; $i++) { + for ($i=0; $i<$count; $i++) { $tmp_name = $_FILES['new_file']['tmp_name'][$i]; if (!is_uploaded_file($tmp_name)) { error_page("$tmp_name is not uploaded file"); @@ -112,79 +100,48 @@ function upload_file($user) { if (strstr($name, "/")) { error_page("no / allowed"); } - $md5 = md5_file($tmp_name); - $s = stat($tmp_name); - $size = $s['size']; - [$exists, $elf] = sandbox_lf_exists($user, $md5); - if (!$exists){ - // move file to download dir - // - $phys_path = sandbox_physical_path($user, $md5); - move_uploaded_file($tmp_name, $phys_path); + if (file_exists("$dir/$name")) { + $notice .= "can't upload $name; file exists.
"; + continue; } + move_uploaded_file($tmp_name, "$dir/$name"); - // write link file + // write info file // - $dir = sandbox_dir($user); - $link_path = "$dir/$name"; - sandbox_write_link_file($link_path, $size, $md5); + [$md5, $size] = get_file_info("$dir/$name"); + write_info_file("$dir/.md5/$name", $md5, $size); + $notice .= "Uploaded file $name
"; } list_files($user, $notice); } -// delete a link to a file. -// check if currently being used by a batch. -// If the last link w/ that contents, delete the file itself +// delete a sandbox file. // function delete_file($user) { $name = get_str('name'); $dir = sandbox_dir($user); - list($error, $size, $md5) = sandbox_parse_link_file("$dir/$name"); - if ($error) { - error_page("can't parse link file"); - } - $p = sandbox_physical_path($user, $md5); - if (!is_file($p)) { - error_page("physical file is missing"); - } - $bused = sandbox_file_in_use($user, $name); - if ($bused){ - $notice = "$name is being used by batch(es), you can not delete it now!
"; - } else{ - unlink("$dir/$name"); - $notice = "$name was deleted from your sandbox
"; - [$exists, $elf] = sandbox_lf_exists($user, $md5); - if (!$exists) { - unlink($p); - } - - } + unlink("$dir/$name"); + unlink("$dir/.md5/$name"); + $notice = "$name was deleted from your sandbox
"; list_files($user, $notice); - //Header("Location: sandbox.php"); } + function download_file($user) { $name = get_str('name'); $dir = sandbox_dir($user); - list($err, $size, $md5) = sandbox_parse_link_file("$dir/$name"); - if ($err) { - error_page("can't parse link file"); - } - $p = sandbox_physical_path($user, $md5); - if (!is_file($p)) { - error_page("$p does not exist!"); - } - do_download($p, $name); + do_download("$dir/$name"); } + function view_file($user) { $name = get_str('name'); $dir = sandbox_dir($user); - list($error, $size, $md5) = sandbox_parse_link_file("$dir/$name"); - if ($error) error_page("no such link file"); - $p = sandbox_physical_path($user, $md5); - if (!is_file($p)) error_page("no such physical file"); + $path = "$dir/$name"; + if (!is_file($path)) { + error_path("no such file $name"); + } echo "
\n";
-    readfile($p);
+    readfile($path);
     echo "
\n"; } diff --git a/tools/submit_buda b/tools/submit_buda index 5b5950c3a76..add5f0abfb3 100755 --- a/tools/submit_buda +++ b/tools/submit_buda @@ -1,35 +1,40 @@ #! /usr/bin/env php Date: Tue, 19 Nov 2024 17:55:31 -0800 Subject: [PATCH 02/12] added (undebugged) code for BUDA job submission. changes: - creating a variant creates a JSON file, variant.json, describing the dockerfile, app files, and in/out files. Template files are now generated during job submission. - no aliasing of files. If your main prog is foo, your Dockerfile must end with CMD ./foo - batch zip file must have shared input files in a directory shared_input_files/. All other directories are jobs. --- html/inc/dir_hier.inc | 16 ++- html/user/buda.php | 88 +++++-------- html/user/buda_submit.php | 261 ++++++++++++++++++++++++++++++++++++-- 3 files changed, 294 insertions(+), 71 deletions(-) diff --git a/html/inc/dir_hier.inc b/html/inc/dir_hier.inc index 9f3b54fd045..e10a4f48ba0 100644 --- a/html/inc/dir_hier.inc +++ b/html/inc/dir_hier.inc @@ -131,24 +131,32 @@ function get_hier_info() { return [$dl_dir, $fanout]; } -// Stage the given file, which is assumed to not be in download dir already. +function download_hier_path($fname) { + [$dl_dir, $fanout] = get_hier_info(); + return dir_hier_path($fname, $dl_dir, $fanout); +} + +// Stage (move, not copy) the given file, +// which is assumed to not be in download dir already. // function stage_file_basic($dir, $fname) { [$dl_dir, $fanout] = get_hier_info(); $old_path = "$dir/$fname"; - $new_path = dir_hier_path($fname, $dl_dir, $fanout); + $new_path = download_hier_path($fname); $md5 = md5_file($old_path); $size = filesize($old_path); file_put_contents("$new_path.md5", "$md5 $size\n"); rename($old_path, $new_path); } -// copy the given file (with known size/md5) -// to the download dir w/ given phys name +// copy the given file (with given md5/size) +// to the download dir w/ given phys name; and create .md5 file +// If phys name is already there, do nothing. // function stage_file_aux($path, $md5, $size, $phys_name) { [$dl_dir, $fanout] = get_hier_info(); $phys_path = dir_hier_path($phys_name, $dl_dir, $fanout); + if (file_exists($phys_path)) return; copy($path, $phys_path); file_put_contents("$phys_path.md5", "$md5 $size\n"); } diff --git a/html/user/buda.php b/html/user/buda.php index 46d3d8ec5a4..fc40ae3d945 100644 --- a/html/user/buda.php +++ b/html/user/buda.php @@ -19,12 +19,7 @@ // web interface for managing BUDA science apps // // in the following, 'app' means BUDA science app -// and 'variant' means a variant of one of these -// -// files can have 3 names -// - sandbox name -// - physical name (e.g. sb_) -// - logical name (how the science app refers to it) +// and 'variant' means a variant of one of these (e.g. CPU, GPU) require_once('../inc/util.inc'); require_once('../inc/sandbox.inc'); @@ -90,7 +85,7 @@ function variant_view() { table_header('name', 'size', 'md5'); foreach(scandir($dir) as $f) { if ($f[0] == '.') continue; - [$size, $md5] = parse_info_file("$dir/.md5/$f"); + [$md5, $size] = parse_info_file("$dir/.md5/$f"); table_row( "$f", $size, @@ -118,10 +113,9 @@ function variant_form($user) { form_start('buda.php'); form_input_hidden('app', $app); form_input_hidden('action', 'variant_action'); - form_input_text('Plan class', 'plan_class'); + form_input_text('Plan class', 'variant'); form_select('Dockerfile', 'dockerfile', $sbitems); - form_select('Main program', 'main_prog', $sbitems); - form_select_multiple('Other application files', 'others', $sbitems); + form_select_multiple('Application files', 'app_files', $sbitems); form_input_text('Input file names', 'input_file_names'); form_input_text('Output file names', 'output_file_names'); form_submit('OK'); @@ -129,12 +123,16 @@ function variant_form($user) { page_tail(); } +function buda_file_phys_name($app, $variant, $md5) { + return sprintf('buda_%s_%s_%s', $app, $variant, $md5); +} + // copy file from sandbox to variant dir, and stage to download hier // -function copy_and_stage_file($user, $fname, $dir, $variant) { +function copy_and_stage_file($user, $fname, $dir, $app, $variant) { copy_sandbox_file($user, $fname, $dir); [$md5, $size] = parse_info_file("$dir/.md5/$fname"); - $phys_name = sprintf('buda_%s_%s_%s', $app, $variant, $md5); + $phys_name = buda_file_phys_name($app, $variant, $md5); stage_file_aux("$dir/$fname", $md5, $size, $phys_name); } @@ -142,67 +140,43 @@ function copy_and_stage_file($user, $fname, $dir, $variant) { // function variant_action($user) { global $buda_root; - $plan_class = get_str('plan_class'); + $variant = get_str('variant'); $app = get_str('app'); $dockerfile = get_str('dockerfile'); - $main_prog = get_str('main_prog'); - $others = get_array('others'); + $app_files = get_array('app_files'); $input_file_names = explode(' ', get_str('input_file_names')); $output_file_names = explode(' ', get_str('output_file_names')); - if (file_exists("$buda_root/$app/$plan_class")) { - error_page("Variant '$plan_class' already exists."); + if (file_exists("$buda_root/$app/$variant")) { + error_page("Variant '$variant' already exists."); } - $dir = "$buda_root/$app/$plan_class"; + $dir = "$buda_root/$app/$variant"; mkdir($dir); mkdir("$dir/.md5"); // copy files from sandbox to variant dir // copy_and_stage_file($user, $dockerfile, $dir, $app, $variant); - copy_and_stage_file($user, $main_prog, $dir, $app, $variant); - foreach ($others as $fname) { + foreach ($app_files as $fname) { copy_and_stage_file($user, $fname, $dir, $app, $variant); } - // create input template + // create variant description JSON file // - $x = "\n"; - $ninfiles = 2 + count($input_file_names) + count($others); - for ($i=0; $i<$ninfiles; $i++) { - $x .= " \n \n"; - } - $x .= " \n"; - $x .= file_ref_in('Dockerfile'); - $x .= file_ref_in($main_prog); - foreach ($others as $fname) { - $x .= file_ref_in($fname); - } - foreach ($input_file_names as $fname) { - $x .= file_ref_in($fname); - } - $x .= " \n\n"; - file_put_contents("$dir/template_in", $x); - - // create output template - // - $x = "\n"; - $i = 0; - foreach ($output_file_names as $fname) { - $x .= file_info_out($i++); - } - $x .= " \n"; - $i = 0; - foreach ($output_file_names as $fname) { - $x .= file_ref_out($i++, $fname); - } - $x .= " \n\n"; - file_put_contents("$dir/template_out", $x); + $desc = new StdClass; + $desc->dockerfile = $dockerfile; + $desc->app_files = $app_files; + $desc->input_file_names = $input_file_names; + $desc->output_file_names = $output_file_names; + file_put_contents( + "$dir/variant.json", + json_encode($desc, JSON_PRETTY_PRINT) + ); // Note: we don't currently allow indirect file access. // If we did, we'd need to create job.toml to mount project dir - app_list("Variant $plan_class added for app $app."); + app_list("Variant $variant added for app $app."); } function file_ref_in($fname) { @@ -249,10 +223,11 @@ function variant_delete() { // foreach (scandir("$dir/.md5") as $fname) { if ($fname[0] == '.') continue; - [$size, $md5] = parse_file_info("$dir/$fname"); - $phys_name = buda_app_file_phys_name($app, $variant, $md5); - $phys_path = download_path($phys_name); + [$md5, $size] = parse_info_file("$dir/.md5/$fname"); + $phys_name = buda_file_phys_name($app, $variant, $md5); + $phys_path = download_hier_path($phys_name); unlink($phys_path); + unlink("$phys_path.md5"); } system("rm -r $buda_root/$app/$variant", $ret); if ($ret) { @@ -300,6 +275,7 @@ function view_file() { $fname = get_str('fname'); echo "
\n";
     readfile("$buda_root/$app/$variant/$fname");
+    echo "
\n"; } $user = get_logged_in_user(); diff --git a/html/user/buda_submit.php b/html/user/buda_submit.php index 0dc2379a867..4f19afe7a6c 100644 --- a/html/user/buda_submit.php +++ b/html/user/buda_submit.php @@ -37,35 +37,274 @@ function submit_form($user) { form_input_hidden('app', $app); form_input_hidden('variant', $variant); form_input_text('Batch name', 'batch_name'); - form_select('Job file', 'job_file', $sbitems_zip); + form_select('Batch file', 'batch_file', $sbitems_zip); form_submit('OK'); form_end(); page_tail(); } -function handle_submit() { - // stage app files if not already staged - // +// unzip batch file into a temp dir; return dir +// +function unzip_batch_file($user, $batch_file) { + @mkdir("../../buda_batches"); + for ($i=0; $i<1000; $i++) { + $batch_dir = "../../buda_batches/$i"; + $ret = mkdir($batch_dir); + if ($ret) break; + } + if (!$ret) error_page("can't create batch dir"); + $sb_dir = sandbox_dir($user); + if (!file_exists("$sb_dir/$batch_file")) { + error_page("no batch file $batch_file"); + } + system("cd $batch_dir; unzip $sb_dir/$batch_file", $ret); + if ($ret) { + error_page("unzip error: $ret"); + } + return $batch_dir; +} - // create batch - // +// check validity of batch dir. +// top level should have only infiles (shared) +// job dirs should have only remaining infiles and possibly cmdline +// +// return struct describing the batch, and the md5/size of files +// +function parse_batch_dir($batch_dir, $variant_desc) { + $input_files = sort($variant_desc->input_file_names); + $shared_files = []; + $shared_file_infos = []; + if (is_dir("$batch_dir/shared_input_files")) { + foreach (scandir("$batch_dir/shared_input_files") as $fname) { + if ($fname[0] == '.') continue; + if (!in_array($fname, $input_files)) { + error_page("$fname is not an input file name"); + } + $shared_files[] = $fname; + $shared_file_infos[] = get_file_info("$batch_dir/shared_input_files/$fname"); + } + } + $unshared_files = sort(array_dir($input_files, $shared_files)); + $jobs = []; + foreach (scandir($batch_dir) as $fname) { + if ($fname[0] == '.') continue; + if ($fname == 'shared_input_files') continue; + if (!is_dir("$batch_dir/$fname")) { + error_page("$batch_dir/$fname is not a directory"); + } + $job_files = []; + $cmdline = ''; + foreach(scandir("$batch_dir/$fname") as $f2) { + if ($f2[0] == '.') continue; + if ($f2 == 'cmdline') { + $cmdline = file_get_contents("$batch_dir/$f2"); + } + if (!in_array($f2, $unshared_files)) { + error_page("$fname/$f2 is not an input file name"); + } + $job_files[] = $f2; + } + if (sort($job_files) != $unshared_files) { + error_page("$fname doesn't have all input files"); + } + + $file_infos = []; + foreach ($unshared_files as $f2) { + $file_infos[] = get_file_info("$batch_dir/$fname/$f2"); + } + + $job = new StdClass; + $job->dir = $fname; + $job->cmdline = $cmdline; + $job->file_infos = $file_infos; + $jobs[] = $job; + } + $batch_desc = new StdClass; + $batch_desc->shared_files = $shared_files; + $batch_desc->shared_file_infos = $shared_file_infos; + $batch_desc->unshared_files = $unshared_files; + $batch_desc->jobs = $jobs; + return $batch_desc; +} + +function create_batch($user, $njobs, $boinc_app) { + $now = time(); + $batch_name = sprintf('buda_%d_%d', $user->id, $now); + $batch_id = BoincBatch::insert(sprintf( + "(user_id, create_time, logical_start_time, logical_end_time, est_completion_time, njobs, fraction_done, nerror_jobs, state, completion_time, credit_estimate, credit_canonical, credit_total, name, app_id, project_state, description, expire_time) values (%d, %d, 0, 0, 0, %d, 0, 0, %d, 0, 0, 0, 0, '%s', %d, 0, '', 0)", + $user->id, $now, $njobs, BATCH_STATE_INIT, $batch_name, $boinc_app->id + )); + return $batch_id; +} + +function stage_input_files($batch_dir, $batch_desc, $batch_id) { + $n = count($batch_desc->shared_files); + $batch_desc->shared_files_phys = []; + for ($i=0; $ishared_files[$i]); + [$md5, $size] = $batch_desc->shared_file_infos[$i]; + $phys_name = sprintf('batch_%d_%s', $batch_id, $md5); + stage_file_aux($path, $md5, $size, $phys_name); + $batch_desc->shared_files_phys_names[] = $phys_name; + } + foreach ($batch_desc->jobs as $job) { + $n = count($batch_desc->unshared_files); + $job->phys_names = []; + for ($i=0; $i<$n; $i++) { + $path = sprintf('%s/%s/%d', + $batch_dir, $job->dir, $batch_desc->unshared_files[$i] + ); + [$md5, $size] = $job->file_infos[$i]; + $phys_name = sprintf('batch_%d_%s', $batch_id, $md5); + stage_file_aux($path, $md5, $size, $phys_name); + $job->phys_names[] = $phys_name; + } + } +} + +function create_jobs($batch_desc, $batch_id, $boinc_app) { + $job_cmds = ''; + foreach ($batch_desc->jobs as $job) { + $job_cmd = ''; + if ($job->cmdline) { + $job_cmd .= sprintf('--command_line "%s"', $job->cmdline); + } + foreach ($batch_desc->shared_file_phys_names as $x) { + $job_cmd .= " $x"; + } + foreach ($job->phys_names as $x) { + $job_cmd .= " $x"; + } + $job_cmds[] .= "$job_cmd\n"; + } + $cmd = sprintf( + '../../bin/create_work --appname %s --batch %d --stdin --command_line "--dockerfile %s"', + $boinc_app->name, $batch_id, $batch_desc->dockerfile + ); + $h = popen($cmd, "w"); + if (!$h) error_paage('create_work launch failed'); + fwrite($h, $job_cmds); + $ret = pclose($h); + if ($ret) { + error_page('create_work failed'); + } +} - // unzip batch file +///////////////// TEMPLATE CREATION ////////////// + +function file_ref_in($fname) { + return(sprintf( +' + %s + + +', + $fname + )); +} +function file_info_out($i) { + return sprintf( +' + + + + 5000000 + + +', + $i + ); +} + +function file_ref_out($i, $fname) { + return sprintf( +' + + %s + +', $i, $fname + ); +} + +// create templates and put them in batch dir +// +function create_templates($batch_desc, $batch_dir) { + // input template // + $x = "\n"; + $ninfiles = 1 + count($batch_desc->input_file_names) + count($batch_desc->app_files); + for ($i=0; $i<$ninfiles; $i++) { + $x .= " \n \n \n"; + } + $x .= " \n"; + $x .= file_ref_in($batch_desc->dockerfile); + foreach ($batch_desc->app_files as $fname) { + $x .= file_ref_in($fname); + } + foreach ($batch_desc->input_file_names as $fname) { + $x .= file_ref_in($fname); + } + $x .= " \n\n"; + file_put_contents("$batch_dir/template_in", $x); - // stage top-level input files + // output template // + $x = "\n"; + $i = 0; + foreach ($batch_desc->output_file_names as $fname) { + $x .= file_info_out($i++); + } + $x .= " \n"; + $i = 0; + foreach ($batch_desc->output_file_names as $fname) { + $x .= file_ref_out($i++, $fname); + } + $x .= " \n\n"; + file_put_contents("$batch_dir/template_out", $x); +} + +function handle_submit($user) { + $boinc_app = BoincApp::lookup_name('buda'); + if (!$boinc_app) { + error_page("No buda app found"); + } + $app = get_str('app'); + $variant = get_str('variant'); + $batch_name = get_str('batch_name', true); + $batch_file = get_str('batch_file'); + + $variant_dir = "../../buda_apps/$app/$variant"; + $variant_desc = json_decode( + file_get_contents("$variant_dir/variant.json") + ); - // scan jobs; stage per-job input files and make create_work input + // unzip batch file into temp dir + $batch_dir = unzip_batch_file($user, $batch_file); + + // scan batch dir; validate and return struct + $batch_desc = parse_batch_dir($batch_dir, $variant_desc); + + create_templates($batch_desc, $batch_dir); + + $batch_id = create_batch($user, count($batch_desc->jobs), $boinc_app); + + // stage input files and record the physical names // + stage_input_files($batch_desc, $batch_dir, $batch_id); + + create_jobs($batch_desc, $batch_id, $boinc_app); - // create jobs + page_head("BUDA jobs submitted"); + echo sprintf('Submitted %d jobs to app %s variant %s', + count($batch_desc->jobs), $app, $variant + ); + page_tail(); } $user = get_logged_in_user(); $action = get_str('action', true); if ($action == 'submit') { - handle_submit(); + handle_submit($user); } else { submit_form($user); } From df4a8b50a71709611b513436bf688f6783d8d15e Mon Sep 17 00:00:00 2001 From: David Anderson Date: Tue, 19 Nov 2024 23:52:56 -0800 Subject: [PATCH 03/12] BUDA job submission: fill in some missing logic. It works now, to the extent of staging files properly and creating plausible-looking workunits. Next step: see if they work. --- html/user/buda.php | 52 ++++++------------------- html/user/buda_submit.php | 81 +++++++++++++++++++++++++-------------- tools/create_work.cpp | 4 -- 3 files changed, 65 insertions(+), 72 deletions(-) diff --git a/html/user/buda.php b/html/user/buda.php index fc40ae3d945..2bc4392ec25 100644 --- a/html/user/buda.php +++ b/html/user/buda.php @@ -134,6 +134,7 @@ function copy_and_stage_file($user, $fname, $dir, $app, $variant) { [$md5, $size] = parse_info_file("$dir/.md5/$fname"); $phys_name = buda_file_phys_name($app, $variant, $md5); stage_file_aux("$dir/$fname", $md5, $size, $phys_name); + return $phys_name; } // create variant @@ -154,13 +155,6 @@ function variant_action($user) { mkdir($dir); mkdir("$dir/.md5"); - // copy files from sandbox to variant dir - // - copy_and_stage_file($user, $dockerfile, $dir, $app, $variant); - foreach ($app_files as $fname) { - copy_and_stage_file($user, $fname, $dir, $app, $variant); - } - // create variant description JSON file // $desc = new StdClass; @@ -168,6 +162,17 @@ function variant_action($user) { $desc->app_files = $app_files; $desc->input_file_names = $input_file_names; $desc->output_file_names = $output_file_names; + + // copy files from sandbox to variant dir + // + $pname = copy_and_stage_file($user, $dockerfile, $dir, $app, $variant); + $desc->dockerfile_phys = $pname; + $desc->app_files_phys = []; + foreach ($app_files as $fname) { + $pname = copy_and_stage_file($user, $fname, $dir, $app, $variant); + $desc->app_files_phys[] = $pname; + } + file_put_contents( "$dir/variant.json", json_encode($desc, JSON_PRETTY_PRINT) @@ -179,39 +184,6 @@ function variant_action($user) { app_list("Variant $variant added for app $app."); } -function file_ref_in($fname) { - return(sprintf( -' - %s - -', - $fname - )); -} -function file_info_out($i) { - return sprintf( -' - - - - 5000000 - - -', - $i - ); -} - -function file_ref_out($i, $fname) { - return sprintf( -' - - %s - -', $i, $fname - ); -} - function variant_delete() { global $buda_root; $app = get_str('app'); diff --git a/html/user/buda_submit.php b/html/user/buda_submit.php index 4f19afe7a6c..a86b3490de3 100644 --- a/html/user/buda_submit.php +++ b/html/user/buda_submit.php @@ -19,6 +19,7 @@ // web interface for submitting BUDA jobs require_once('../inc/util.inc'); +require_once('../inc/submit_util.inc'); require_once('../inc/sandbox.inc'); display_errors(); @@ -43,13 +44,14 @@ function submit_form($user) { page_tail(); } -// unzip batch file into a temp dir; return dir +// unzip batch file into a temp dir; return dir name // function unzip_batch_file($user, $batch_file) { @mkdir("../../buda_batches"); for ($i=0; $i<1000; $i++) { $batch_dir = "../../buda_batches/$i"; - $ret = mkdir($batch_dir); + $batch_dir_name = $i; + $ret = @mkdir($batch_dir); if ($ret) break; } if (!$ret) error_page("can't create batch dir"); @@ -57,11 +59,11 @@ function unzip_batch_file($user, $batch_file) { if (!file_exists("$sb_dir/$batch_file")) { error_page("no batch file $batch_file"); } - system("cd $batch_dir; unzip $sb_dir/$batch_file", $ret); + system("cd $batch_dir; unzip $sb_dir/$batch_file > /dev/null", $ret); if ($ret) { error_page("unzip error: $ret"); } - return $batch_dir; + return $batch_dir_name; } // check validity of batch dir. @@ -71,7 +73,8 @@ function unzip_batch_file($user, $batch_file) { // return struct describing the batch, and the md5/size of files // function parse_batch_dir($batch_dir, $variant_desc) { - $input_files = sort($variant_desc->input_file_names); + $input_files = $variant_desc->input_file_names; + sort($input_files); $shared_files = []; $shared_file_infos = []; if (is_dir("$batch_dir/shared_input_files")) { @@ -84,7 +87,8 @@ function parse_batch_dir($batch_dir, $variant_desc) { $shared_file_infos[] = get_file_info("$batch_dir/shared_input_files/$fname"); } } - $unshared_files = sort(array_dir($input_files, $shared_files)); + $unshared_files = array_diff($input_files, $shared_files); + sort($unshared_files); $jobs = []; foreach (scandir($batch_dir) as $fname) { if ($fname[0] == '.') continue; @@ -139,8 +143,8 @@ function create_batch($user, $njobs, $boinc_app) { function stage_input_files($batch_dir, $batch_desc, $batch_id) { $n = count($batch_desc->shared_files); - $batch_desc->shared_files_phys = []; - for ($i=0; $ishared_files_phys_names = []; + for ($i=0; $i<$n; $i++) { $path = sprintf('%s/%s', $batch_dir, $batch_desc->shared_files[$i]); [$md5, $size] = $batch_desc->shared_file_infos[$i]; $phys_name = sprintf('batch_%d_%s', $batch_id, $md5); @@ -151,7 +155,7 @@ function stage_input_files($batch_dir, $batch_desc, $batch_id) { $n = count($batch_desc->unshared_files); $job->phys_names = []; for ($i=0; $i<$n; $i++) { - $path = sprintf('%s/%s/%d', + $path = sprintf('%s/%s/%s', $batch_dir, $job->dir, $batch_desc->unshared_files[$i] ); [$md5, $size] = $job->file_infos[$i]; @@ -162,31 +166,49 @@ function stage_input_files($batch_dir, $batch_desc, $batch_id) { } } -function create_jobs($batch_desc, $batch_id, $boinc_app) { +function create_jobs( + $variant_desc, $batch_desc, $batch_id, $boinc_app, $batch_dir_name +) { + // get list of names of app files + // + $app_file_names = $variant_desc->dockerfile_phys; + foreach ($variant_desc->app_files_phys as $pname) { + $app_file_names .= " $pname"; + } $job_cmds = ''; foreach ($batch_desc->jobs as $job) { $job_cmd = ''; if ($job->cmdline) { $job_cmd .= sprintf('--command_line "%s"', $job->cmdline); } - foreach ($batch_desc->shared_file_phys_names as $x) { + $job_cmd .= " $app_file_names"; + foreach ($batch_desc->shared_files_phys_names as $x) { $job_cmd .= " $x"; } foreach ($job->phys_names as $x) { $job_cmd .= " $x"; } - $job_cmds[] .= "$job_cmd\n"; + $job_cmds .= "$job_cmd\n"; } $cmd = sprintf( - '../../bin/create_work --appname %s --batch %d --stdin --command_line "--dockerfile %s"', - $boinc_app->name, $batch_id, $batch_desc->dockerfile + 'cd ../..; bin/create_work --appname %s --batch %d --stdin --command_line "--dockerfile %s" --wu_template %s --result_template %s', + $boinc_app->name, $batch_id, $variant_desc->dockerfile, + "buda_batches/$batch_dir_name/template_in", + "buda_batches/$batch_dir_name/template_out" ); + $cmd .= sprintf(' > %s 2<&1', "buda_batches/errfile"); $h = popen($cmd, "w"); - if (!$h) error_paage('create_work launch failed'); + if (!$h) error_page('create_work launch failed'); fwrite($h, $job_cmds); $ret = pclose($h); if ($ret) { - error_page('create_work failed'); + echo $cmd; + echo "\n\n"; + echo $job_cmds; + echo "\n\n"; + readfile("../../buda_batches/errfile"); + exit; + error_page("create_work failed: $x"); } } @@ -228,20 +250,20 @@ function file_ref_out($i, $fname) { // create templates and put them in batch dir // -function create_templates($batch_desc, $batch_dir) { +function create_templates($variant_desc, $batch_dir) { // input template // $x = "\n"; - $ninfiles = 1 + count($batch_desc->input_file_names) + count($batch_desc->app_files); + $ninfiles = 1 + count($variant_desc->input_file_names) + count($variant_desc->app_files); for ($i=0; $i<$ninfiles; $i++) { $x .= " \n \n \n"; } $x .= " \n"; - $x .= file_ref_in($batch_desc->dockerfile); - foreach ($batch_desc->app_files as $fname) { + $x .= file_ref_in($variant_desc->dockerfile); + foreach ($variant_desc->app_files as $fname) { $x .= file_ref_in($fname); } - foreach ($batch_desc->input_file_names as $fname) { + foreach ($variant_desc->input_file_names as $fname) { $x .= file_ref_in($fname); } $x .= " \n\n"; @@ -251,12 +273,12 @@ function create_templates($batch_desc, $batch_dir) { // $x = "\n"; $i = 0; - foreach ($batch_desc->output_file_names as $fname) { + foreach ($variant_desc->output_file_names as $fname) { $x .= file_info_out($i++); } $x .= " \n"; $i = 0; - foreach ($batch_desc->output_file_names as $fname) { + foreach ($variant_desc->output_file_names as $fname) { $x .= file_ref_out($i++, $fname); } $x .= " \n\n"; @@ -264,7 +286,7 @@ function create_templates($batch_desc, $batch_dir) { } function handle_submit($user) { - $boinc_app = BoincApp::lookup_name('buda'); + $boinc_app = BoincApp::lookup("name='buda'"); if (!$boinc_app) { error_page("No buda app found"); } @@ -279,20 +301,23 @@ function handle_submit($user) { ); // unzip batch file into temp dir - $batch_dir = unzip_batch_file($user, $batch_file); + $batch_dir_name = unzip_batch_file($user, $batch_file); + $batch_dir = "../../buda_batches/$batch_dir_name"; // scan batch dir; validate and return struct $batch_desc = parse_batch_dir($batch_dir, $variant_desc); - create_templates($batch_desc, $batch_dir); + create_templates($variant_desc, $batch_dir); $batch_id = create_batch($user, count($batch_desc->jobs), $boinc_app); // stage input files and record the physical names // - stage_input_files($batch_desc, $batch_dir, $batch_id); + stage_input_files($batch_dir, $batch_desc, $batch_id); - create_jobs($batch_desc, $batch_id, $boinc_app); + create_jobs( + $variant_desc, $batch_desc, $batch_id, $boinc_app, $batch_dir_name + ); page_head("BUDA jobs submitted"); echo sprintf('Submitted %d jobs to app %s variant %s', diff --git a/tools/create_work.cpp b/tools/create_work.cpp index 0809b37f8a5..fff305cbc79 100644 --- a/tools/create_work.cpp +++ b/tools/create_work.cpp @@ -436,10 +436,6 @@ int main(int argc, char** argv) { strcat(jd.result_template_path, jd.result_template_file); if (use_stdin) { - // clear the WU template name so we'll recognize a job-level one - // - strcpy(jd.wu_template_file, ""); - if (jd.assign_flag) { // if we're doing assignment we can't use the bulk-query method; // create the jobs one at a time. From 160cc3de8eb4cbd36007f60aa73c3bfd9e45730b Mon Sep 17 00:00:00 2001 From: David Anderson Date: Wed, 20 Nov 2024 18:48:12 -0800 Subject: [PATCH 04/12] docker wrapper: it's OK if no config file --- html/user/buda_submit.php | 4 ++++ samples/docker_wrapper/docker_wrapper.cpp | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/html/user/buda_submit.php b/html/user/buda_submit.php index a86b3490de3..053bbfa3719 100644 --- a/html/user/buda_submit.php +++ b/html/user/buda_submit.php @@ -319,6 +319,10 @@ function handle_submit($user) { $variant_desc, $batch_desc, $batch_id, $boinc_app, $batch_dir_name ); + // clean up batch dir + // + //system("rm -rf $batch_dir"); + page_head("BUDA jobs submitted"); echo sprintf('Submitted %d jobs to app %s variant %s', count($batch_desc->jobs), $app, $variant diff --git a/samples/docker_wrapper/docker_wrapper.cpp b/samples/docker_wrapper/docker_wrapper.cpp index 4b7a519393b..02b0ae66a14 100644 --- a/samples/docker_wrapper/docker_wrapper.cpp +++ b/samples/docker_wrapper/docker_wrapper.cpp @@ -124,7 +124,8 @@ DOCKER_CONN docker_conn; int parse_config_file() { std::ifstream ifs(config_file); if (ifs.fail()) { - return -1; + fprintf(stderr, "no job.toml config file\n"); + return 0; } toml::ParseResult r = toml::parse(ifs); if (!r.valid()) { From 415696ba6a979ae448c32493ed243a015ad649ac Mon Sep 17 00:00:00 2001 From: David Anderson Date: Thu, 21 Nov 2024 00:59:13 -0800 Subject: [PATCH 05/12] dockerwrapper: work dir defaults to /app --- html/user/buda_submit.php | 2 +- samples/docker_wrapper/docker_wrapper.cpp | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/html/user/buda_submit.php b/html/user/buda_submit.php index 053bbfa3719..5d65b71ee95 100644 --- a/html/user/buda_submit.php +++ b/html/user/buda_submit.php @@ -191,7 +191,7 @@ function create_jobs( $job_cmds .= "$job_cmd\n"; } $cmd = sprintf( - 'cd ../..; bin/create_work --appname %s --batch %d --stdin --command_line "--dockerfile %s" --wu_template %s --result_template %s', + 'cd ../..; bin/create_work --appname %s --batch %d --stdin --command_line "--dockerfile %s --verbose" --wu_template %s --result_template %s', $boinc_app->name, $batch_id, $variant_desc->dockerfile, "buda_batches/$batch_dir_name/template_in", "buda_batches/$batch_dir_name/template_out" diff --git a/samples/docker_wrapper/docker_wrapper.cpp b/samples/docker_wrapper/docker_wrapper.cpp index 02b0ae66a14..1e970249b07 100644 --- a/samples/docker_wrapper/docker_wrapper.cpp +++ b/samples/docker_wrapper/docker_wrapper.cpp @@ -122,6 +122,9 @@ DOCKER_CONN docker_conn; // parse job config file // int parse_config_file() { + // defaults + config.workdir = "/app"; + std::ifstream ifs(config_file); if (ifs.fail()) { fprintf(stderr, "no job.toml config file\n"); @@ -137,8 +140,6 @@ int parse_config_file() { x = v.find("workdir"); if (x) { config.workdir = x->as(); - } else { - config.workdir = "/app"; } x = v.find("project_dir_mount"); if (x) { From 1b017e173faac03f55076cb13248935b846b246f Mon Sep 17 00:00:00 2001 From: David Anderson Date: Thu, 21 Nov 2024 01:42:22 -0800 Subject: [PATCH 06/12] add buda test files buda: output files are --- html/user/buda_submit.php | 1 + .../test_buda/Dockerfile_worker_1 | 3 +++ samples/docker_wrapper/test_buda/main_2.sh | 3 +++ .../test_buda/worker_3_x86_64-pc-linux-gnu | Bin 0 -> 18536 bytes 4 files changed, 7 insertions(+) create mode 100644 samples/docker_wrapper/test_buda/Dockerfile_worker_1 create mode 100644 samples/docker_wrapper/test_buda/main_2.sh create mode 100644 samples/docker_wrapper/test_buda/worker_3_x86_64-pc-linux-gnu diff --git a/html/user/buda_submit.php b/html/user/buda_submit.php index 5d65b71ee95..02475a2d9da 100644 --- a/html/user/buda_submit.php +++ b/html/user/buda_submit.php @@ -243,6 +243,7 @@ function file_ref_out($i, $fname) { ' %s + ', $i, $fname ); diff --git a/samples/docker_wrapper/test_buda/Dockerfile_worker_1 b/samples/docker_wrapper/test_buda/Dockerfile_worker_1 new file mode 100644 index 00000000000..d897eaff2b1 --- /dev/null +++ b/samples/docker_wrapper/test_buda/Dockerfile_worker_1 @@ -0,0 +1,3 @@ +FROM debian +WORKDIR /app +CMD ./main_2.sh diff --git a/samples/docker_wrapper/test_buda/main_2.sh b/samples/docker_wrapper/test_buda/main_2.sh new file mode 100644 index 00000000000..c09b323cc44 --- /dev/null +++ b/samples/docker_wrapper/test_buda/main_2.sh @@ -0,0 +1,3 @@ +#! /bin/bash + +worker_3_x86_64-pc-linux-gnu --nsecs 20 in out diff --git a/samples/docker_wrapper/test_buda/worker_3_x86_64-pc-linux-gnu b/samples/docker_wrapper/test_buda/worker_3_x86_64-pc-linux-gnu new file mode 100644 index 0000000000000000000000000000000000000000..0aff3a397e1e65ae7aaa5e2acfccb504f223e0e1 GIT binary patch literal 18536 zcmeHPd30OVnZHl+B1^HY#5gz$jUWWFVA;;%BqSkr5(Nbu2#`?71XNj;Y&EhZv}6H7 z-6l{>2z9#9ltLTmlv28du1q_pO$JI}&S?vsw$SZNXq(ms+PYzxu*LKH-9?sO>`v#L z`D6ab&GGy0x8Co5chSA~-IospI@UTYOPJ&omk45gE=6h6jKP&U1JEXxh$=kK6X%Kw z;AN7g_-zV+tI|nr+CXhqIKigg5{Htr!|94F3^X1RmTTTv;OSaLHBbnjBv6mOrq7Wm zpigEDJgEIRrlHXzO1(a<*QfOi9M+B*XzIth@o%HHXNPORV=|DkOdF2px|naSk?3i# zLy?7nM(--<(a(Q0d9~oxy1en?GNAo8P*+zpCE}f}3!4&Mjfr?NzoT)-lGeu7h5l^H zKVR0H_9=Vm6`N#9Y;O}<`g|7tNF%@Qk1ySN`MnQS)f|j%ZLP?5ZvWfrhsiTGWW6k( z=(D^I(k9}MY0f!!PmgU1dzYTrYr2VI6g}@o-dBX1>}+Hv8KekK9*2J3IP}Hi(39iP zyT_q#7>E7{=sx_}6XP(J|Cf(L_l!edGYPu3w! zIMDY;|8XR!f1^1Iqs4+O+m$IlHC{z-pTszC`DD(-wGUjT9X5KKrn_u$HTVlNd{U2N zishMftcYGbj|>*kwP~p`TtwGXm8736qU)(x(oMhM+->SehUpiin{iFL=@+CI_wzfo z{Yucyd_wtwB0A>|lMWQo&0I;|!6JI`@5v)Ybe3V#p(46z1M(g(qPvYS;Nc=V$DT<~ z7ttpeBH*({^y(sdaQ7>&;NJ2T4L%XP{cz4P@@#PTlddNU5+jSgezs4H%=#vtwKLj~ zBi=*)(6JGuS&tHDQx82W@rQ^rbVG+Fejjm$Z0L~0KT4dT8agQPyNEMHLjw~35OH>q zp+1SY)KdWBm@_QUg$`c~L@t#-y5a#maqWK(co`Bi|yr6+^^$8%EWYk35&5Rz?iF8v0z8916M}nJJPK^E8dV+);_+1ZFM(Thr;m=k`UFg(J`}%UCrdiQzEMhDr=$wh zwZ*!-1Fu;x{}8;u$s3+vDd08bJg4ZNbjAiEt3SQhwA z?Y_r>u&@KceP{iUd{}-qXgwR;%hq@YJ(7Ohib8HboS(NlFldb@4YAv$yRQswhSA{e zUBjaG_AjZj#=K2FCU+x`WjlpIFH$#oG@bjd| zFCSZhKDH|#zpnI@kJSYq7a+$*7+QV`_88(^r7}4ey5-WLBj25d>U^Jd!npb%;NXOU zffH{-Z*Tdp>*)HC?{acCl37arPVzUML^l`?_Wy#ehY;eUfk8-$!PQ`&2-eO$jE-A< z8x&5|-gqQdJNs!lKw`E2BekDAyytLk5-PZFgVn!j5C$Gc=`TNtQlq+5PNk;QT9%rC zSy|O*FcLBT+TYuL?YjQw+c&ji?wNI?S57UBgx9q9zq_vgjgC1SFRtK`cb$W&w@{_q z|D5x+{MyV19sO^0^uMvD|F7*MXZDrvW}}g;UgKm$Z#l*dzmAqrx@SEpWwPk+$U(+Ywc2-+v z%QM*<*1jlre|>xW?bbBsGVVP%!4JGSGI9X04PiM9xB-yAJev{Y7XS|fo&+pMt27|^ z&!Q(S0c^u;*9XYNeE^en^VP!I;kBkssB{fjm0r@hc07hM=21YWvE_w0AAHIWpBx!E zMA;he+M0&TYCYRsePZR*Wpfvte;&!yN3)9^u**7_MBOd;yBT$ek=S12y~DA(y225L zN6PlCARh)lP{MC3@b3rxIQUOt02lO~M;sJjng5Ev0d(;wpjcJo{fJ{#O~XCTK#gx- z*{Yh^cb5lin(wFx)-2gwxxS_?Q?sPKrn$Xl_Np2m}6M-`kI1_>YpAj(U_U4>kZ-WbG@#akUpgw#yXTdq0 znWC#@-UsjihRN(@rz*7YUSPV$4gd91BPo&&=*=S!c9{ms6f|f5Piww8(|%5|g*nS^ z(0ILu2AH$=v_8)^dsKNq2qCk7H)rc-YsEvlU_*aIJ7)ZL=|Gux5T^Y1XniAxBL`VO zh)WWV93+0Hwuhq!!G691BRksfRZIe+JBZW4pm$=!*YQ!#6an&=s4k;RPDD zYPeRzYc-5%Xt)1A;pY9&rK?w8?3=x*GoQ@meT)5z{LPIE@)B#gt!0V7dEp#I6&Jyx zYFt&~UAkp%8YglOpeLz5=CUqB>y+_8@I_b{OVRN$NH|_t0uiUH@)UvP>V;*eqw*&N zWtC1yIIW75B$l58P*zrsSA9?@UjVa??~`+PoNmrR!Ond$`z5fP1I(7YDqNqEmMgYF zw32yO7jiF@y*>+4Ra-$Qd!rLiQS~}Ke*0q-sq(U{cbNB99VGo-<{PTKP%1mcyszpm z%8W2S+kGjXWx^uT>|PF6$}EX2aW5jrA(1wB2i40Y5_F$UyX6wu;C_)}6%x75{UIVQ zi*In==RU<^+*T4uk9!q$CRm*iNW05fj#nZ%_jMGTWTnB`;jSiUvb7dSpZii5znC_`Pht)d-!(-9M$H3#4y{-3#c>BI_;)JSiM}5mjFKd5C#9bCuoadI`b} zWd~W`sv}fH+RNbr%DCD>n&skPqq@n#9`T`R$-!V=tf zs|26^sss=GSb~FpmEdz0`a`|z^VJeOJVSyntd!u98zuPSE(yN$F$uo>FA@y5%3@zn zNI&2BnIxY4s|0VZ!emkJdaF!=w`WT5&KAxflQ-c}HS%F-oh@r_Sq)W|HBVZ3TG@O- z8T-95_6KFEGzp9gq*Ij zydv^t&oq~N*$ZGY-c=3|+{-&5B&KJgy~4eM`7+`7B_!OJJPMgCr6b zMvy98oAH!=D2dF}!ULQf2s3;aN_7W)-b)ESz}ydT?s?0}+~gpj0Q-;SE>r%I6vKr6fw{m_utZ1T()z{vlSEV{eqNjL+Oa z_BT1raZK9TK0g@lYFQl)R>>DW-m-FkUWVlxq_Uu?GRH`v6s@dN{?U8ZJq^kF7+&AH#$GunHBDUtl!tD%?>2$ zJ*}StQtw$Tr&iArIkkG0vg_A-E|ycPXStl?JeSC+)zc=YR?liVwR+absnrvZQ>*7v zIkkF%a!&PJCg)Vo<#JB-bjUf?vtG`ro-5>>>e(RYRL@nMQzx)pF0ax*W0vsVf@rZi|A{b9I7pYh-@s$SgX|&RT@-j&3cN^xuS$V$QQ&J7s0}gOAp7EM4}`5{lvt!! z`|<@AaO?gFpv?I;Oeg`z{*F0MI*74rI`+5C#g!>*Ii(!?-#3>xu?*HTH!dtWSvvM# zH;+Gm2HS#;#Rb+yH?xTO{B;~gRhR+_px`Yq*@){}5Kz{yoAVl&2HP+*DVbFpWcXI&kphx3KUa|!t9(K?F`QeF!xh6(|tDlZTOt~$$EDnOrKk9xN|vl@b&jqT{Kaony{^}c3*e(qnq;|PE}uf>bL&ArZ9xV z^9wE&^k)^M%9YFoql6Dzh!;&==ohF;pKQ(kB>2lkqC5o=vH zqVq;WPxPWuGfCNb3T7H%rby54*Wos$eR37TDc(Z&b%-=OvGsOMHJ%pof_PIto8evC zrtV0j372d`tqb*y+wNq(X>n6L8A;^3qC(1PrCJhrjm+)2_@bGM`4jvG8XR!2fs*)fjrfu5b_Yq#e1WQy1X+VPvjco$xG-F7hq<_ za9(KTU7LlY%vq7c(^0XXPuKPGdF}hJZ+`uFx9gs!nU@@YbsK0_MXus|j=6P?d39d_ zcnyDFam=rK2=L$Wmq(tO&ctV&QwFqr-gW#z*B@QYc?kS%zU%n)SY28c?z2Xd+e9ea zlgi{e^ReQnh1Oje;@#I!j`iW?RzxF*btoB-4Mme(q8ArZlaeAYzDgSIrzZ2b^SVR8 zd{;OZWe7zmmI!xe#r8-~I0KWBRC;G99}&3v%3C8@_<&d^rR3%-mKJO*z%nfWQ@AyY zd$)-MZoe|3H{;fbq9wXC1S_4Aw^;aF(A$|yZHIaInhkG*3vf1;NQHT=7i`?7?e@t> zKADYoC!<}s?vjIdxm0hbs$n>?HJ=W3hO<%8l?sJJ-SKYR;!UK|B9Y+Q5qEs)4)4b# zQ@l*>TRm@H%R+TV;cM*8<&)7B-BDC29swH8M0!@JtLnbSZeL>zS79Ps8`GIoE{ak! zji@1<5?S04X2>&Pcue;*(Qub?qca<&6%o#K%L~W&i9*D7EPO{;6n`Q+AQYI`yS-B3E*MPY#X3M^1yp#r!O z9g;V(S*vY1-h&RMVzF#A$7rAjZ(gi>)P^N=groVI=8$OWN%cmXI#ZaQ!d<;8)07SO zrW4U@le&~7)a@sE1)Hie-s~=NP{?k9KFkZ@#&9T|$Y;_1p_}q?j1R@r*TiKn6`@q3 z%lK!sM4_i~2%tW>o0H*QT<6Ym;899No~GU?e7= zm6EY6FsJ>a3@jzk3;#AUN>Mq!M{%IN4%ym~;U#_8Mn6~6&3hSTMTo`7SonS=sN=x$ znAYOY8YBL_pwqs2&qGErz~r z(6?whrb?Nj3W%QqdNE!$_xHRbpD*Dxe3^zygmLqw0--L#4brMF)fjSap(_? zL;vPDbpCEJK6Zb82D@=WgFLYq5U7M8BU<4|9Fb^H22C~(CZ zxr8h+fVK0Q>#t~Eziu_;ia7EDI!IWrhXO%uA-Lvh5xTVF%2n+hp)1#}y(X|Rw6T3v zM}X@d?Dis|NH#C4f<@2x-{oPIjD>JpTk&^%`dh(LqFhdwl0s2zl)~y`!%~j?=Fmps z7l);!)4nFOvuR#^GRRdVK0quL(BCSS673%vhPtw;P!IMc39MSN`cvBit_H_le;37% zjevM|X0zH4x#E@|S(X;dk0(nhTw#||)OVJpv``42Nt)ikb{g}e$kMMdOD&*6SnZb5 zgg?8p7aI;htiC1eF*z)-qnWhuC$Wd~x35~)m(fH;jU_;2<%1*j{GLns+1 zqmR=;nXbhXQ-49<+_y3CIVDzj+T(X4IJ~=J{5Sh01252;#(yJc;8yT?7slXbUuB?A z>)ZX;WWlp~#?MUjIE>f0Z_9_Gt#1`$5#7j6b{nJ)kir^pDB>t{R9H zQ%SP7e?Pdz`iJ$t&%i?^+FX;UWZ=hb`euJEoPw#ULZCw4@;e|wf=*6 zUs#;%vQx&elfJU zC>38=p!5&v`OF8j1%Iaf&F>HA2h;Y*7Sf%|RQ5eKrYYA5PDI8gB3TdX_b8Xx6l|#Y EZ@Z>zMF0Q* literal 0 HcmV?d00001 From 881a4c16b6ec778b1ae45b21a554afbf6be7656f Mon Sep 17 00:00:00 2001 From: David Anderson Date: Thu, 21 Nov 2024 12:31:47 -0800 Subject: [PATCH 07/12] BUDA job submission now passes the basic end-to-end test - woo hoo! When you run a shell script on Unix, and it has Windows line endings (CRLF), it fails with a misleading 'file not found' error message. This can cause problems with BUDA apps, which can involve shell scripts, and all files go through the user sandbox. For example: if you put the script (with Unix endings) into Github and check it out on a Win machine, all of a sudden it has Win endings! If you upload it to your sandbox, it won't work. So I added a sandbox feature where you can add a file by pasting text into a web form. Surprisingly, even this changed the LF to a CRLF! I changed the form handler to convert CRLF to LF, and now it works. How many man-years have been wasted on this line-ending BS? I'm guessing the blame goes to Microsoft. --- html/user/sandbox.php | 35 ++++++++++++++++++++++ samples/docker_wrapper/test_buda/main_2.sh | 2 +- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/html/user/sandbox.php b/html/user/sandbox.php index 293606968c0..07fa4d71904 100644 --- a/html/user/sandbox.php +++ b/html/user/sandbox.php @@ -42,11 +42,28 @@ function list_files($user, $notice) { echo "

Upload files

+

+ NOTE: if you upload text files from Windows, + they will have CRLF line endings. + If they are shell scripts, they won't work on Linux. + Add shell scripts using Add File. +

+

+
+

Add file

+ "; + form_start('sandbox.php', 'post'); + form_input_hidden('action', 'add_file'); + form_input_text('Name', 'name'); + form_input_textarea('Contents', 'contents'); + form_submit('OK'); + form_end(); + echo "

Sandbox contents

"; @@ -116,6 +133,23 @@ function upload_file($user) { list_files($user, $notice); } +function add_file($user) { + $dir = sandbox_dir($user); + $name = post_str('name'); + if (file_exists("$dir/$name")) { + error_page("file $name exists"); + } + $contents = post_str('contents'); + $contents = str_replace("\r\n", "\n", $contents); + file_put_contents("$dir/$name", $contents); + + [$md5, $size] = get_file_info("$dir/$name"); + write_info_file("$dir/.md5/$name", $md5, $size); + + $notice = "Uploaded file $name
"; + list_files($user, $notice); +} + // delete a sandbox file. // function delete_file($user) { @@ -154,6 +188,7 @@ function view_file($user) { switch ($action) { case '': list_files($user,""); break; case 'upload_file': upload_file($user); break; +case 'add_file': add_file($user); break; case 'delete_file': delete_file($user); break; case 'download_file': download_file($user); break; case 'view_file': view_file($user); break; diff --git a/samples/docker_wrapper/test_buda/main_2.sh b/samples/docker_wrapper/test_buda/main_2.sh index c09b323cc44..f07fd7a2a69 100644 --- a/samples/docker_wrapper/test_buda/main_2.sh +++ b/samples/docker_wrapper/test_buda/main_2.sh @@ -1,3 +1,3 @@ #! /bin/bash -worker_3_x86_64-pc-linux-gnu --nsecs 20 in out +./worker_3_x86_64-pc-linux-gnu --nsecs 20 in out From dc47826575050f4731096a841b0350f242ac5eaa Mon Sep 17 00:00:00 2001 From: David Anderson Date: Fri, 22 Nov 2024 17:12:22 -0800 Subject: [PATCH 08/12] improve BUDA web interface For BUDA batches, description is ' ()' Move job submission admin functions to their own page Lay the groundwork for unifying output file handling for remote job submission. --- html/user/buda.php | 45 ++++++++--- html/user/buda_submit.php | 39 ++++++---- html/user/submit.php | 141 ++++++++++++++++++++--------------- sched/sample_assimilator.cpp | 2 +- sched/validate_util.cpp | 26 ++++--- sched/validate_util.h | 3 +- tools/sample_assimilate.py | 3 +- 7 files changed, 159 insertions(+), 100 deletions(-) diff --git a/html/user/buda.php b/html/user/buda.php index 2bc4392ec25..b3066bd0c9a 100644 --- a/html/user/buda.php +++ b/html/user/buda.php @@ -37,17 +37,35 @@ function app_list($notice=null) { if (!is_dir($buda_root)) { mkdir($buda_root); } - page_head('Docker apps'); + page_head('BUDA science apps'); if ($notice) { echo "$notice

\n"; } + text_start(); + echo " +

BUDA (BOINC Universal Docker App) + lets you submit Docker jobs through a web interface; + you don't need to log into the BOINC server. +

+ To use BUDA, you set up a 'science app' and one or more 'variants'. + Each variant includes a Dockerfile, + a main program to run within the container, + and any other files that are needed. +

+ Typically there is a variant named 'cpu' that uses one CPU. + The names of other variants are plan class names; + these can be versions that use a GPU or multiple CPUs. + "; + + echo "

Science apps

"; $dirs = scandir($buda_root); foreach ($dirs as $dir) { if ($dir[0] == '.') continue; show_app($dir); } echo '
'; - show_button_small('buda.php?action=app_form', 'Add app'); + show_button_small('buda.php?action=app_form', 'Add science app'); + text_end(); page_tail(); } @@ -55,23 +73,24 @@ function show_app($dir) { global $buda_root; $indent = "        "; echo "
$dir\n"; - echo "

$indent Variants (click for details):

'; - echo $indent; - show_button_small("buda.php?action=variant_form&app=$dir", 'Add variant'); + end_table(); echo "

"; + show_button("buda.php?action=variant_form&app=$dir", 'Add variant'); echo "

"; show_button_small( - "buda.php?action=app_delete&app=$dir", "Delete app '$dir'" + "buda.php?action=app_delete&app=$dir", "Delete science app '$dir'" ); } @@ -93,6 +112,10 @@ function variant_view() { ); } end_table(); + show_button_small( + "buda.php?action=variant_delete&app=$dir&variant=$variant", + 'Delete variant' + ); page_tail(); } diff --git a/html/user/buda_submit.php b/html/user/buda_submit.php index 02475a2d9da..86cb70c3d0b 100644 --- a/html/user/buda_submit.php +++ b/html/user/buda_submit.php @@ -32,13 +32,19 @@ function submit_form($user) { $app = get_str('app'); $variant = get_str('variant'); + $desc = "
+ A zipped directory with one subdirectory per job, + containing the input file(s) for that job + and an optional file cmdline + containing command-line arguments. + See more details. + "; page_head("Submit jobs to $app ($variant)"); form_start('buda_submit.php'); form_input_hidden('action', 'submit'); form_input_hidden('app', $app); form_input_hidden('variant', $variant); - form_input_text('Batch name', 'batch_name'); - form_select('Batch file', 'batch_file', $sbitems_zip); + form_select("Batch zip file $desc", 'batch_file', $sbitems_zip); form_submit('OK'); form_end(); page_tail(); @@ -131,14 +137,16 @@ function parse_batch_dir($batch_dir, $variant_desc) { return $batch_desc; } -function create_batch($user, $njobs, $boinc_app) { +function create_batch($user, $njobs, $boinc_app, $app, $variant) { $now = time(); $batch_name = sprintf('buda_%d_%d', $user->id, $now); + $description = "$app ($variant)"; $batch_id = BoincBatch::insert(sprintf( - "(user_id, create_time, logical_start_time, logical_end_time, est_completion_time, njobs, fraction_done, nerror_jobs, state, completion_time, credit_estimate, credit_canonical, credit_total, name, app_id, project_state, description, expire_time) values (%d, %d, 0, 0, 0, %d, 0, 0, %d, 0, 0, 0, 0, '%s', %d, 0, '', 0)", - $user->id, $now, $njobs, BATCH_STATE_INIT, $batch_name, $boinc_app->id + "(user_id, create_time, logical_start_time, logical_end_time, est_completion_time, njobs, fraction_done, nerror_jobs, state, completion_time, credit_estimate, credit_canonical, credit_total, name, app_id, project_state, description, expire_time) values (%d, %d, 0, 0, 0, %d, 0, 0, %d, 0, 0, 0, 0, '%s', %d, 0, '%s', 0)", + $user->id, $now, $njobs, BATCH_STATE_INIT, $batch_name, $boinc_app->id, + $description )); - return $batch_id; + return BoincBatch::lookup_id($batch_id); } function stage_input_files($batch_dir, $batch_desc, $batch_id) { @@ -293,7 +301,6 @@ function handle_submit($user) { } $app = get_str('app'); $variant = get_str('variant'); - $batch_name = get_str('batch_name', true); $batch_file = get_str('batch_file'); $variant_dir = "../../buda_apps/$app/$variant"; @@ -310,25 +317,27 @@ function handle_submit($user) { create_templates($variant_desc, $batch_dir); - $batch_id = create_batch($user, count($batch_desc->jobs), $boinc_app); + $batch = create_batch( + $user, count($batch_desc->jobs), $boinc_app, $app, $variant + ); // stage input files and record the physical names // - stage_input_files($batch_dir, $batch_desc, $batch_id); + stage_input_files($batch_dir, $batch_desc, $batch->id); create_jobs( - $variant_desc, $batch_desc, $batch_id, $boinc_app, $batch_dir_name + $variant_desc, $batch_desc, $batch->id, $boinc_app, $batch_dir_name ); + // mark batch as in progress + // + $batch->update(sprintf('state=%d', BATCH_STATE_IN_PROGRESS)); + // clean up batch dir // //system("rm -rf $batch_dir"); - page_head("BUDA jobs submitted"); - echo sprintf('Submitted %d jobs to app %s variant %s', - count($batch_desc->jobs), $app, $variant - ); - page_tail(); + header("Location: submit.php?action=query_batch&batch_id=$batch->id"); } $user = get_logged_in_user(); diff --git a/html/user/submit.php b/html/user/submit.php index 37b2384e404..0af700404b0 100644 --- a/html/user/submit.php +++ b/html/user/submit.php @@ -202,7 +202,7 @@ function handle_main($user) { error_page("Ask the project admins for permission to submit jobs"); } - page_head("Job submission and control"); + page_head("Job submission"); if (isset($submit_urls)) { // show links to per-app job submission pages @@ -254,40 +254,8 @@ function handle_main($user) { } } if ($user_submit->manage_all || $app_admin) { - echo "

Administrative functions

\n"; + echo "

Administer job submission

\n"; + show_button('submit.php?action=admin', 'Administer'); } $batches = BoincBatch::enum("user_id = $user->id order by id desc"); @@ -318,20 +286,59 @@ function check_admin_access($user, $app_id) { } } -function handle_admin($user) { - $app_id = get_int("app_id"); - check_admin_access($user, $app_id); - if ($app_id) { - $app = BoincApp::lookup_id($app_id); - if (!$app) error_page("no such app"); - page_head("Administer batches for $app->user_friendly_name"); - $batches = BoincBatch::enum("app_id = $app_id order by id desc"); - show_batches($batches, PAGE_SIZE, null, $app); +function handle_admin() { + page_head("Administer job submission"); + if ($user_submit->manage_all) { + echo "
  • All applications
    + + "; + $apps = BoincApp::enum("deprecated=0"); + foreach ($apps as $app) { + echo " +
  • $app->user_friendly_name
    + + "; + } } else { - page_head("Administer batches (all apps)"); - $batches = BoincBatch::enum("true order by id desc"); - show_batches($batches, PAGE_SIZE, null, null); + foreach ($usas as $usa) { + $app = BoincApp::lookup_id($usa->app_id); + echo "
  • $app->user_friendly_name
    + id>Batches + "; + if ($usa->manage) { + echo "· + id&action=app_version_form>Versions + "; + } + } } + echo "\n"; + page_tail(); +} + +function handle_admin_app($user) { + $app_id = get_int("app_id"); + check_admin_access($user, $app_id); + $app = BoincApp::lookup_id($app_id); + if (!$app) error_page("no such app"); + + page_head("Administer batches for $app->user_friendly_name"); + $batches = BoincBatch::enum("app_id = $app_id order by id desc"); + show_batches($batches, PAGE_SIZE, null, $app); + page_tail(); +} +function handle_admin_all($user) { + page_head("Administer batches (all apps)"); + $batches = BoincBatch::enum("true order by id desc"); + show_batches($batches, PAGE_SIZE, null, null); page_tail(); } @@ -378,7 +385,8 @@ function handle_batch_stats($user) { page_tail(); return; } - start_table(); + text_start(); + start_table('table-striped'); row2("qualifying results", $n); row2("mean WSS", size_string($wss_sum/$n)); row2("max WSS", size_string($wss_max)); @@ -387,7 +395,7 @@ function handle_batch_stats($user) { row2("mean disk usage", size_string($disk_sum/$n)); row2("max disk usage", size_string($disk_max)); end_table(); - + text_end(); page_tail(); } @@ -438,10 +446,21 @@ function handle_query_batch($user) { $app = BoincApp::lookup_id($batch->app_id); $wus = BoincWorkunit::enum("batch = $batch->id"); $batch = get_batch_params($batch, $wus); + if ($batch->user_id == $user->id) { + $owner = $user; + } else { + $owner = BoincUser::lookup_id($batch->user_id); + } page_head("Batch $batch_id"); start_table(); row2("name", $batch->name); + if ($batch->description) { + row2('description', $batch->description); + } + if ($owner) { + row2('submitter', $owner->name); + } row2("application", $app?$app->name:'---'); row2("state", batch_state_string($batch->state)); //row2("# jobs", $batch->njobs); @@ -459,10 +478,11 @@ function handle_query_batch($user) { row2("Output File Size", size_string(batch_output_file_size($batch->id))); end_table(); $url = "get_output2.php?cmd=batch&batch_id=$batch->id"; + echo "

    "; show_button($url, "Get zipped output files"); + echo "

    "; switch ($batch->state) { case BATCH_STATE_IN_PROGRESS: - echo "

    "; show_button( "submit.php?action=abort_batch_confirm&batch_id=$batch_id", "Abort batch" @@ -470,13 +490,13 @@ function handle_query_batch($user) { break; case BATCH_STATE_COMPLETE: case BATCH_STATE_ABORTED: - echo "

    "; show_button( "submit.php?action=retire_batch_confirm&batch_id=$batch_id", "Retire batch" ); break; } + echo "

    "; show_button("submit.php?action=batch_stats&batch_id=$batch_id", "Show memory/disk usage statistics" ); @@ -484,7 +504,8 @@ function handle_query_batch($user) { echo "

    Jobs

    \n"; start_table(); table_header( - "Job ID and name
    click for details or to get output files", + "ID
    click for details or to get output files", + "Name", "status", "Canonical instance
    click to see result page on BOINC server", "Download Results" @@ -504,13 +525,13 @@ function handle_query_batch($user) { $y = "in progress"; } } - echo " - id>$wu->id · $wu->name - $y - $x - $text - - "; + row_array([ + "id>$wu->id", + $wu->name, + $y, + $x, + $text + ]); } end_table(); echo "

    Return to job control page\n"; @@ -714,6 +735,8 @@ function handle_show_all($user) { case 'abort_batch': handle_abort_batch($user); break; case 'abort_batch_confirm': handle_abort_batch_confirm(); break; case 'admin': handle_admin($user); break; +case 'admin_app': handle_admin_app($user); break; +case 'admin_all': handle_admin_all($user); break; case 'batch_stats': handle_batch_stats($user); break; case 'query_batch': handle_query_batch($user); break; case 'query_job': handle_query_job($user); break; diff --git a/sched/sample_assimilator.cpp b/sched/sample_assimilator.cpp index eda8ce6cc62..3c7fb70a263 100644 --- a/sched/sample_assimilator.cpp +++ b/sched/sample_assimilator.cpp @@ -17,7 +17,7 @@ // A sample assimilator that: // 1) if success, copy the output file(s) to a directory -// ../sample_results/batchid +// ../results/batchid // If 1 output file, its name is the WU name // If >1 files, file i is named _i // 2) if failure, write a message to _error diff --git a/sched/validate_util.cpp b/sched/validate_util.cpp index 55ef5594dc1..accd0507831 100644 --- a/sched/validate_util.cpp +++ b/sched/validate_util.cpp @@ -44,16 +44,18 @@ bool standalone = false; ////////// functions for locating output files /////////////// int OUTPUT_FILE_INFO::parse(XML_PARSER& xp) { - bool found=false; optional = false; no_validate = false; while (!xp.get_tag()) { if (!xp.is_tag) continue; if (xp.match_tag("/file_ref")) { - return found?0:ERR_XML_PARSE; + if (phys_name.empty()) return ERR_XML_PARSE; + if (logical_name.empty()) return ERR_XML_PARSE; } - if (xp.parse_string("file_name", name)) { - found = true; + if (xp.parse_string("file_name", phys_name)) { + continue; + } + if (xp.parse_string("open_name", logical_name)) { continue; } if (xp.parse_bool("optional", optional)) continue; @@ -74,13 +76,13 @@ int get_output_file_info(RESULT const& result, OUTPUT_FILE_INFO& fi) { int retval = fi.parse(xp); if (retval) return retval; if (standalone) { - safe_strcpy(path, fi.name.c_str()); - if (!path_to_filename(fi.name, name)) { - fi.name = name; + safe_strcpy(path, fi.phys_name.c_str()); + if (!path_to_filename(fi.phys_name, name)) { + fi.phys_name = name; } } else { dir_hier_path( - fi.name.c_str(), config.upload_dir, + fi.phys_name.c_str(), config.upload_dir, config.uldl_dir_fanout, path ); } @@ -105,13 +107,13 @@ int get_output_file_infos(RESULT const& result, vector& fis) { int retval = fi.parse(xp); if (retval) return retval; if (standalone) { - safe_strcpy(path, fi.name.c_str()); - if (!path_to_filename(fi.name, name)) { - fi.name = name; + safe_strcpy(path, fi.phys_name.c_str()); + if (!path_to_filename(fi.phys_name, name)) { + fi.phys_name = name; } } else { dir_hier_path( - fi.name.c_str(), config.upload_dir, + fi.phys_name.c_str(), config.upload_dir, config.uldl_dir_fanout, path ); } diff --git a/sched/validate_util.h b/sched/validate_util.h index 221c2d6cad5..f133dc05edc 100644 --- a/sched/validate_util.h +++ b/sched/validate_util.h @@ -27,8 +27,9 @@ // of result.xml_doc_in // struct OUTPUT_FILE_INFO { - std::string name; + std::string phys_name; std::string path; + std::string logical_name; bool optional; // sample_bitwise_validator: not an error if this file is missing bool no_validate; diff --git a/tools/sample_assimilate.py b/tools/sample_assimilate.py index 766c4d9c914..d4c4f267e14 100755 --- a/tools/sample_assimilate.py +++ b/tools/sample_assimilate.py @@ -12,7 +12,8 @@ # sample_assimilator.py --error error_code wu_name wu_id batch_id # # in the 1st case, move the output files from the upload hierarchy -# to results//_i +# to results//_file= +# where is the file's logical name (from template) # in the 2nd case, write the error code # to results//_error From f8879a3a0e6dc90444e9f07b2f638a7bf06f7bf4 Mon Sep 17 00:00:00 2001 From: David Anderson Date: Sat, 23 Nov 2024 23:35:40 -0800 Subject: [PATCH 09/12] web job submission, script assimilator: add new output file model old model: no assimilator output files live in upload hierarchy w/ physical names WUs are marked as assimulated when batch is retired; file_deleter deletes output files after that. new model: assimilator (e.g. sample_assimilate.py and sample_assimilator.cpp) moves output files of canonical results to project/results/, with names that include the batch name and the logical name. WU is marked as assimilated; file_deleter deletes output files of non-canonical results. advantages of new model: can see all output files of a batch on cmdline with ls can zip output files of a batch without copying them unified naming scheme for output files that encodes the batch, the job (e.g. the BUDA job dir name) and the logical name of the file. ------------ script assimilator: pass logical names to the script Support both models. Choice of model is per app. The project.inc file says which app uses which model. --- html/inc/result.inc | 2 +- html/inc/submit_util.inc | 6 +- html/user/buda_submit.php | 4 +- html/user/get_output3.php | 61 +++++++++++++++++++ html/user/submit.php | 111 +++++++++++++++++++++++------------ sched/assimilator.cpp | 3 +- sched/file_deleter.cpp | 2 + sched/sample_assimilator.cpp | 8 +-- sched/script_assimilator.cpp | 57 +++++++++++------- sched/validate_util.cpp | 17 +++++- tools/create_work.cpp | 9 +++ tools/query_job | 10 ++-- tools/sample_assimilate.py | 34 +++++------ 13 files changed, 229 insertions(+), 95 deletions(-) create mode 100644 html/user/get_output3.php diff --git a/html/inc/result.inc b/html/inc/result.inc index 1dae1c5c764..1b3d84f1832 100644 --- a/html/inc/result.inc +++ b/html/inc/result.inc @@ -723,7 +723,7 @@ function show_result($result, $show_outfile_links=false) { } if ($show_outfile_links && $result->outcome == 1) { $fanout = parse_config(get_config(), ""); - $names = get_outfile_names($result); + $names = get_outfile_phys_names($result); $i = 0; $x = ""; foreach ($names as $name) { diff --git a/html/inc/submit_util.inc b/html/inc/submit_util.inc index bce125d5c39..2cba300adbe 100644 --- a/html/inc/submit_util.inc +++ b/html/inc/submit_util.inc @@ -164,7 +164,9 @@ function wus_nsent($wus) { return $n; } -function get_outfile_names($result) { +// get the physical names of a result's output files. +/ +function get_outfile_phys_names($result) { $names = []; $xml = "".$result->xml_doc_out.""; $r = simplexml_load_string($xml); @@ -261,7 +263,7 @@ function batch_output_file_size($batchid) { foreach ($wus as $wu) { if (!$wu->canonical_resultid) continue; $result = BoincResult::lookup_id($wu->canonical_resultid); - $names = get_outfile_names($result); + $names = get_outfile_phys_names($result); foreach ($names as $name) { $path = dir_hier_path($name, $upload_dir, $fanout); if (is_file($path)) { diff --git a/html/user/buda_submit.php b/html/user/buda_submit.php index 86cb70c3d0b..f797caeb837 100644 --- a/html/user/buda_submit.php +++ b/html/user/buda_submit.php @@ -185,9 +185,9 @@ function create_jobs( } $job_cmds = ''; foreach ($batch_desc->jobs as $job) { - $job_cmd = ''; + $job_cmd = sprintf('--wu_name batch_%d__job_%s', $batch_id, $job->dir); if ($job->cmdline) { - $job_cmd .= sprintf('--command_line "%s"', $job->cmdline); + $job_cmd .= sprintf(' --command_line "%s"', $job->cmdline); } $job_cmd .= " $app_file_names"; foreach ($batch_desc->shared_files_phys_names as $x) { diff --git a/html/user/get_output3.php b/html/user/get_output3.php new file mode 100644 index 00000000000..2da95847926 --- /dev/null +++ b/html/user/get_output3.php @@ -0,0 +1,61 @@ +. + +// get output files, individually or zipped groups +// Assumes the layout used by sample_assimilator.cpp and sample_assimilate.py: +// /results/ +// / (0 if not in a batch) +// + +require_once("../inc/util.inc"); + +// show or download a single output file, +// identified by result ID and file index +// +function get_file() { + $path = get_str('path'); + if (strstr($path, '.')) error_page('bad path'); + $path = "../../$path"; + + $download = get_str('download', true); + if ($download) { + do_download($path); + } else { + readfile($path); + } +} + +// download a zip of the given directory +// +function get_batch() { + $batch_id = get_str('batch_id'); + $dir = "../../results/$batch_id"; + $name = "batch_$batch_id.zip"; + $cmd = "cd $dir; rm -f $name; zip $name *"; + system($cmd); + do_download("$dir/$name"); +} + +$action = get_str('action'); +switch ($action) { +case 'get_file': get_file(); break; +case 'get_batch': get_batch(); break; +} + +?> diff --git a/html/user/submit.php b/html/user/submit.php index 0af700404b0..9e04c3981ee 100644 --- a/html/user/submit.php +++ b/html/user/submit.php @@ -196,7 +196,7 @@ function show_batches($batches, $limit, $user, $app) { // and a button for creating a new batch // function handle_main($user) { - global $submit_urls; + global $web_apps; $user_submit = BoincUserSubmit::lookup_userid($user->id); if (!$user_submit) { error_page("Ask the project admins for permission to submit jobs"); @@ -204,19 +204,19 @@ function handle_main($user) { page_head("Job submission"); - if (isset($submit_urls)) { + if (isset($web_apps) && $web_apps) { // show links to per-app job submission pages // echo "

    Submit jobs

      "; - foreach ($submit_urls as $appname=>$submit_url) { + foreach ($web_apps as $appname => $web_app) { $appname = BoincDb::escape_string($appname); $app = BoincApp::lookup("name='$appname'"); - if (!$app) error_page("bad submit_url name: $appname"); + if (!$app) error_page("bad web app name: $appname"); $usa = BoincUserSubmitApp::lookup("user_id=$user->id and app_id=$app->id"); if ($usa || $user_submit->submit_all) { - echo "
    • $app->user_friendly_name "; + echo "
    • submit_url> $app->user_friendly_name "; } } echo "
    \n"; @@ -441,6 +441,8 @@ function progress_bar($batch, $wus, $width) { // show the details of an existing batch // function handle_query_batch($user) { + global $web_apps; + $batch_id = get_int('batch_id'); $batch = BoincBatch::lookup_id($batch_id); $app = BoincApp::lookup_id($batch->app_id); @@ -452,6 +454,8 @@ function handle_query_batch($user) { $owner = BoincUser::lookup_id($batch->user_id); } + $web_app = $web_apps[$app->name]; + page_head("Batch $batch_id"); start_table(); row2("name", $batch->name); @@ -477,7 +481,12 @@ function handle_query_batch($user) { row2("GFLOP/hours, actual", number_format(credit_to_gflop_hours($batch->credit_canonical), 2)); row2("Output File Size", size_string(batch_output_file_size($batch->id))); end_table(); - $url = "get_output2.php?cmd=batch&batch_id=$batch->id"; + + if ($web_app->assim_move) { + $url = "get_output3.php?action=get_batch&batch_id=$batch->id"; + } else { + $url = "get_output2.php?cmd=batch&batch_id=$batch->id"; + } echo "

    "; show_button($url, "Get zipped output files"); echo "

    "; @@ -503,13 +512,16 @@ function handle_query_batch($user) { echo "

    Jobs

    \n"; start_table(); - table_header( + $x = [ "ID
    click for details or to get output files", "Name", "status", - "Canonical instance
    click to see result page on BOINC server", - "Download Results" - ); + "Canonical instance
    click for details" + ]; + if (!$web_app->assim_move) { + $x[] = "Download Results"; + } + row_heading_array($x); foreach($wus as $wu) { $resultid = $wu->canonical_resultid; if ($resultid) { @@ -525,13 +537,16 @@ function handle_query_batch($user) { $y = "in progress"; } } - row_array([ + $x = [ "id>$wu->id", $wu->name, $y, - $x, - $text - ]); + $x + ]; + if (!$web_app->assim_move) { + $x[] = $text; + } + row_array($x); } end_table(); echo "

    Return to job control page\n"; @@ -541,14 +556,20 @@ function handle_query_batch($user) { // show the details of a job, including links to see the output files // function handle_query_job($user) { + global $web_apps; + $wuid = get_int('wuid'); $wu = BoincWorkunit::lookup_id($wuid); if (!$wu) error_page("no such job"); + $app = BoincApp::lookup_id($wu->appid); + $web_app = $web_apps[$app->name]; + page_head("Job $wuid"); echo " - Workunit details · + Workunit details +

    batch>Batch $wu->batch "; @@ -558,7 +579,7 @@ function handle_query_job($user) { $x = "".$wu->xml_doc.""; $x = simplexml_load_string($x); start_table(); - table_header("Logical name
    (click to view)", + table_header("Name
    (click to view)", "Size (bytes)", "MD5" ); foreach ($x->workunit->file_ref as $fr) { @@ -577,41 +598,53 @@ function handle_query_job($user) { } end_table(); - echo "

    Instances

    \n"; + echo "

    Job instances

    \n"; start_table(); table_header( - "Instance ID
    click for result page", - "State", "Output files
    click to view the file" + "ID
    click for result page", + "State", + "Output files" ); $results = BoincResult::enum("workunitid=$wuid"); $upload_dir = parse_config(get_config(), ""); $fanout = parse_config(get_config(), ""); foreach($results as $result) { - echo " - id>$result->id · $result->name - ".state_string($result)." - -"; + $x = [ + "id>$result->id", + state_string($result) + ]; $i = 0; - if ($result->server_state == 5) { + if ($result->server_state == RESULT_SERVER_STATE_OVER) { $phys_names = get_outfile_names($result); $log_names = get_outfile_log_names($result); for ($i=0; $iid, $i - ); - $path = dir_hier_path($phys_names[$i], $upload_dir, $fanout); - $s = stat($path); - $size = $s['size']; - echo sprintf('%s (%s bytes)
    ', - $url, - $log_names[$i], - number_format($size) - ); + if ($web_app->assim_move) { + // file is in + // project/results//__file_ + $path = sprintf('results/%s/%s__file_%s', + $wu->batch, $wu->name, $log_names[$i] + ); + $x[] = "view · download"; + } else { + // file is in upload hier + $url = sprintf( + 'get_output2.php?cmd=result&result_id=%d&file_num=%d', + $result->id, $i + ); + $path = dir_hier_path($phys_names[$i], $upload_dir, $fanout); + $s = stat($path); + $size = $s['size']; + $x[] = sprintf('%s (%s bytes)
    ', + $url, + $log_names[$i], + number_format($size) + ); + } } + } else { + $x[] = '---'; } - echo "\n"; + row_array($x); } end_table(); echo "

    Return to job control page\n"; @@ -743,7 +776,7 @@ function handle_show_all($user) { case 'retire_batch': handle_retire_batch($user); break; case 'retire_batch_confirm': handle_retire_batch_confirm(); break; case 'show_all': handle_show_all($user); break; -case 'toggle_loc': handle_toggle_loc($user); +case 'toggle_loc': handle_toggle_loc($user); break; default: error_page("no such action $action"); } diff --git a/sched/assimilator.cpp b/sched/assimilator.cpp index 3f62321003d..ca9c56db8b6 100644 --- a/sched/assimilator.cpp +++ b/sched/assimilator.cpp @@ -160,7 +160,8 @@ bool do_pass(APP& app) { retval = assimilate_handler(wu, results, canonical_result); if (retval && retval != DEFER_ASSIMILATION) { log_messages.printf(MSG_CRITICAL, - "[%s] handler error: %s; exiting\n", wu.name, boincerror(retval) + "assimilator.cpp [%s] handler error %d: %s; exiting\n", + wu.name, retval, boincerror(retval) ); exit(retval); } diff --git a/sched/file_deleter.cpp b/sched/file_deleter.cpp index a495a918775..7d5f7243bf1 100644 --- a/sched/file_deleter.cpp +++ b/sched/file_deleter.cpp @@ -17,6 +17,8 @@ // file deleter. See usage() below for usage. +// skips WUs with 'nodelete' in the name +// skips files with in the // enum sizes. RESULT_PER_ENUM is three times larger on the // assumption of 3-fold average redundancy. diff --git a/sched/sample_assimilator.cpp b/sched/sample_assimilator.cpp index 3c7fb70a263..f25f4450770 100644 --- a/sched/sample_assimilator.cpp +++ b/sched/sample_assimilator.cpp @@ -89,11 +89,9 @@ int assimilate_handler( bool file_copied = false; for (i=0; i +// of output files of the canonical result // wu_id workunit ID // wu_name workunit name // result_id ID of the canonical result @@ -56,20 +59,22 @@ using std::vector; using std::string; -vector script; +// scriptname, followed by arguments +vector script_args; int assimilate_handler_init(int argc, char** argv) { // handle project specific arguments here for (int i=1; i paths; - retval = get_output_file_paths(canonical_result, paths); + sprintf(cmd, "../bin/%s", script_args[0].c_str()); + vector fis; + retval = get_output_file_infos(canonical_result, fis); if (retval) return retval; - for (i=1; i& fis) { if (xp.match_tag("file_ref")) { OUTPUT_FILE_INFO fi; int retval = fi.parse(xp); - if (retval) return retval; + if (retval) { + log_messages.printf(MSG_CRITICAL, + "get_output_file_infos(): error parsing %s\n", + result.xml_doc_in + ); + return retval; + } if (standalone) { safe_strcpy(path, fi.phys_name.c_str()); if (!path_to_filename(fi.phys_name, name)) { diff --git a/tools/create_work.cpp b/tools/create_work.cpp index fff305cbc79..3680943b675 100644 --- a/tools/create_work.cpp +++ b/tools/create_work.cpp @@ -25,6 +25,15 @@ // - to create a single job, with everything passed on the cmdline // - to create multiple jobs, where per-job info is passed via stdin, // one line per job +// available options here: +// --wu_name X +// --wu_template F +// --result_template F +// --remote_file url nbytes md5 +// --target_host ID +// --target_user ID +// --priority N +// phys_name1 ... // // The input files must already be staged (i.e. in the download hierarchy). diff --git a/tools/query_job b/tools/query_job index 65a2b860202..9f56272f2f0 100755 --- a/tools/query_job +++ b/tools/query_job @@ -34,17 +34,17 @@ function show_wu($wu, $dir) { ." Host $host->id ($host->os_name, $host->p_vendor)\n" ." User $user->id ($user->name)\n" ; + $xmlin = simplexml_load_string( + sprintf("%s", $result->xml_doc_in) + ); $xmlout = simplexml_load_string( sprintf("%s", $result->xml_doc_out) ); $ofs = $xmlout->file_info; + $ifs = $xmlin->result->file_ref; $nofs = $ofs->count(); for ($i=0; $i<$nofs; $i++) { - if ($nofs == 1) { - $path = "$dir/$wu->name"; - } else { - $path = sprintf("$dir/%s_%d", $wu->name, $i); - } + $path = sprintf("$dir/%s__file_%s", $wu->name, $ifs[$i]->open_name); if (!is_file($path)) { die("output file $i is missing: $path\n"); } diff --git a/tools/sample_assimilate.py b/tools/sample_assimilate.py index d4c4f267e14..2f48d22bc16 100755 --- a/tools/sample_assimilate.py +++ b/tools/sample_assimilate.py @@ -1,21 +1,21 @@ #! /usr/bin/env python3 # Sample script for the script-based assimilator (sched/script_assimilator.cpp) -# Moves output files into a results/ dir hierarchy +# Moves output files into a results/ hierarchy # # Use with a config.xml command of the form -# script_assimilator -d 3 --app worker --script "sample_assimilate.py wu_name batch_id files" +# script_assimilator -d 3 --app worker --script "sample_assimilate.py wu_name batch_id files2" # With this command, this script will be invoked either as -# sample_assimilate.py wu_name batch_id outfile_path1 ... +# sample_assimilate.py wu_name batch_id outfile_path1 logical_name1 ... # or -# sample_assimilator.py --error error_code wu_name wu_id batch_id +# sample_assimilator.py --error error_code wu_name wu_id batch_id # # in the 1st case, move the output files from the upload hierarchy -# to results//_file= -# where is the file's logical name (from template) +# to results//__file_ +# where is the file's logical name # in the 2nd case, write the error code -# to results//_error +# to results//_error import sys, os @@ -38,15 +38,13 @@ if os.system(cmd): raise Exception('%s failed'%(cmd)) - nfiles = len(sys.argv) - 3 - if nfiles == 1: - outfile_path = sys.argv[3] - cmd = 'mv %s %s/%s'%(outfile_path, outdir, wu_name) + nfiles = (len(sys.argv) - 3)//2 + for i in range(nfiles): + outfile_path = sys.argv[2*i+3] + logical_name = sys.argv[2*i+4] + cmd = 'mv %s %s/%s__file_%s'%( + outfile_path, outdir, wu_name, logical_name + ) if os.system(cmd): - raise Exception('%s failed'%(cmd)) - else: - for i in range(nfiles): - outfile_path = sys.argv[i+3] - cmd = 'mv %s %s/%s_%d'%(outfile_path, outdir, wu_name, i) - if os.system(cmd): - raise Exception('%s failed'%(cmd)) + #raise Exception('%s failed'%(cmd)) + sys.stderr.write('%s failed\n'%(cmd)) From 3858f356adf33b77a79250f08da06bd35a71440a Mon Sep 17 00:00:00 2001 From: David Anderson Date: Sun, 24 Nov 2024 11:12:36 -0800 Subject: [PATCH 10/12] job submission web: user friendlier Principle: when showing jobs or files, use names that the user provided, rather than DB IDs or names that BOINC generated. --- html/user/submit.php | 71 +++++++++++++++--------------- sched/sample_dummy_assimilator.cpp | 3 ++ 2 files changed, 38 insertions(+), 36 deletions(-) diff --git a/html/user/submit.php b/html/user/submit.php index 9e04c3981ee..75ee59f5c16 100644 --- a/html/user/submit.php +++ b/html/user/submit.php @@ -457,6 +457,7 @@ function handle_query_batch($user) { $web_app = $web_apps[$app->name]; page_head("Batch $batch_id"); + text_start(); start_table(); row2("name", $batch->name); if ($batch->description) { @@ -513,10 +514,8 @@ function handle_query_batch($user) { echo "

    Jobs

    \n"; start_table(); $x = [ - "ID
    click for details or to get output files", - "Name", - "status", - "Canonical instance
    click for details" + "Name
    click for details", + "status" ]; if (!$web_app->assim_move) { $x[] = "Download Results"; @@ -525,11 +524,9 @@ function handle_query_batch($user) { foreach($wus as $wu) { $resultid = $wu->canonical_resultid; if ($resultid) { - $x = "$resultid"; $y = 'completed'; $text = "id>Download output files"; } else { - $x = "---"; $text = "---"; if ($batch->state == BATCH_STATE_COMPLETE) { $y = 'failed'; @@ -538,10 +535,8 @@ function handle_query_batch($user) { } } $x = [ - "id>$wu->id", - $wu->name, + "id>$wu->name", $y, - $x ]; if (!$web_app->assim_move) { $x[] = $text; @@ -550,6 +545,7 @@ function handle_query_batch($user) { } end_table(); echo "

    Return to job control page\n"; + text_end(); page_tail(); } @@ -565,7 +561,8 @@ function handle_query_job($user) { $app = BoincApp::lookup_id($wu->appid); $web_app = $web_apps[$app->name]; - page_head("Job $wuid"); + page_head("Job $wu->name"); + text_start(); echo " Workunit details @@ -573,32 +570,7 @@ function handle_query_job($user) { batch>Batch $wu->batch "; - // show input files - // - echo "

    Input files

    \n"; - $x = "".$wu->xml_doc.""; - $x = simplexml_load_string($x); - start_table(); - table_header("Name
    (click to view)", - "Size (bytes)", "MD5" - ); - foreach ($x->workunit->file_ref as $fr) { - $pname = (string)$fr->file_name; - $lname = (string)$fr->open_name; - foreach ($x->file_info as $fi) { - if ((string)$fi->name == $pname) { - table_row( - "url>$lname", - $fi->nbytes, - $fi->md5_cksum - ); - break; - } - } - } - end_table(); - - echo "

    Job instances

    \n"; + echo "

    Instances

    \n"; start_table(); table_header( "ID
    click for result page", @@ -647,6 +619,33 @@ function handle_query_job($user) { row_array($x); } end_table(); + + // show input files + // + echo "

    Input files

    \n"; + $x = "".$wu->xml_doc.""; + $x = simplexml_load_string($x); + start_table(); + table_header("Name
    (click to view)", + "Size (bytes)", "MD5" + ); + foreach ($x->workunit->file_ref as $fr) { + $pname = (string)$fr->file_name; + $lname = (string)$fr->open_name; + foreach ($x->file_info as $fi) { + if ((string)$fi->name == $pname) { + table_row( + "url>$lname", + $fi->nbytes, + $fi->md5_cksum + ); + break; + } + } + } + + end_table(); + text_end(); echo "

    Return to job control page\n"; page_tail(); } diff --git a/sched/sample_dummy_assimilator.cpp b/sched/sample_dummy_assimilator.cpp index b6f18635296..a9629a9a99b 100644 --- a/sched/sample_dummy_assimilator.cpp +++ b/sched/sample_dummy_assimilator.cpp @@ -16,6 +16,9 @@ // along with BOINC. If not, see . // A sample assimilator that only writes a log message. +// But WUs are marked as assimilated, which means file deleter +// will delete output files unless you mark them as no_delete, +// or include 'no_delete' in the WU name #include "config.h" #include From b6654b894f3994047f1e10473890c242b2db58db Mon Sep 17 00:00:00 2001 From: David Anderson Date: Mon, 25 Nov 2024 14:49:36 -0800 Subject: [PATCH 11/12] web (BUDA) - validate URL args that are used as filenames; prevent ../ stuff. Do this by checking for '/'; is that sufficient? - add 'delete app' function - remove binary test file --- html/inc/util_basic.inc | 15 +++-- html/user/buda.php | 58 ++++++++++++++++-- .../test_buda/worker_3_x86_64-pc-linux-gnu | Bin 18536 -> 0 bytes 3 files changed, 60 insertions(+), 13 deletions(-) delete mode 100644 samples/docker_wrapper/test_buda/worker_3_x86_64-pc-linux-gnu diff --git a/html/inc/util_basic.inc b/html/inc/util_basic.inc index 39e5fe3ec40..8785c24b112 100644 --- a/html/inc/util_basic.inc +++ b/html/inc/util_basic.inc @@ -56,14 +56,6 @@ function sched_stopped() { return file_exists("$d/stop_sched"); } -function show_page($x, $y) { - echo " - $x -

    $x

    - $y - "; -} - function xml_error($num=-1, $msg=null, $file=null, $line=null) { global $xml_outer_tag; if (!$msg) { @@ -205,4 +197,11 @@ function dtime() { return microtime(true); } +// is $x a valid file (or dir) name? +// +function is_valid_filename($x) { + if (strstr($x, '/')) return false; + return true; +} + ?> diff --git a/html/user/buda.php b/html/user/buda.php index b3066bd0c9a..7a8048844c5 100644 --- a/html/user/buda.php +++ b/html/user/buda.php @@ -71,7 +71,6 @@ function app_list($notice=null) { function show_app($dir) { global $buda_root; - $indent = "        "; echo "
    $dir\n"; start_table('table-striped'); table_header('Variant name (click for details)', 'Submit jobs'); @@ -97,7 +96,9 @@ function show_app($dir) { function variant_view() { global $buda_root; $app = get_str('app'); + if (!is_valid_filename($app)) die('bad arg'); $variant = get_str('variant'); + if (!is_valid_filename($variant)) die('bad arg'); page_head("App $app variant $variant"); $dir = "$buda_root/$app/$variant"; start_table(); @@ -131,6 +132,7 @@ function variant_view() { function variant_form($user) { $sbitems = sandbox_select_items($user); $app = get_str('app'); + if (!is_valid_filename($app)) die('bad arg'); page_head("Create variant of Docker app $app"); form_start('buda.php'); @@ -165,11 +167,23 @@ function copy_and_stage_file($user, $fname, $dir, $app, $variant) { function variant_action($user) { global $buda_root; $variant = get_str('variant'); + if (!is_valid_filename($variant)) die('bad arg'); $app = get_str('app'); + if (!is_valid_filename($app)) die('bad arg'); $dockerfile = get_str('dockerfile'); + if (!is_valid_filename($dockerfile)) die('bad arg'); $app_files = get_array('app_files'); + foreach ($app_files as $fname) { + if (!is_valid_filename($fname)) die('bad arg'); + } $input_file_names = explode(' ', get_str('input_file_names')); $output_file_names = explode(' ', get_str('output_file_names')); + foreach ($input_file_names as $fname) { + if (!is_valid_filename($fname)) die('bad arg'); + } + foreach ($output_file_names as $fname) { + if (!is_valid_filename($fname)) die('bad arg'); + } if (file_exists("$buda_root/$app/$variant")) { error_page("Variant '$variant' already exists."); @@ -210,10 +224,13 @@ function variant_action($user) { function variant_delete() { global $buda_root; $app = get_str('app'); + if (!is_valid_filename($app)) die('bad arg'); $variant = get_str('variant'); + if (!is_valid_filename($variant)) die('bad arg'); $confirmed = get_str('confirmed', true); if ($confirmed) { $dir = "$buda_root/$app/$variant"; + if (!file_exists($dir)) error_page('no such variant'); // delete staged files // foreach (scandir("$dir/.md5") as $fname) { @@ -232,9 +249,7 @@ function variant_delete() { app_list($notice); } else { page_head("Confirm"); - echo "Are you sure want to delete variant $variant of app $app? -

    - "; + echo "Are you sure you want to delete variant $variant of app $app?

    "; show_button( "buda.php?action=variant_delete&app=$app&variant=$variant&confirmed=yes", "Yes" @@ -243,8 +258,37 @@ function variant_delete() { } } +function app_delete() { + global $buda_root; + $app = get_str('app'); + if (!is_valid_filename($app)) die('bad arg'); + $confirmed = get_str('confirmed', true); + if ($confirmed) { + $dir = "$buda_root/$app"; + if (!file_exists($dir)) error_page('no such app'); + foreach (scandir($dir) as $fname) { + if ($fname[0] == '.') continue; + error_page("You must delete all variants first."); + } + system("rmdir $buda_root/$app", $ret); + if ($ret) { + error_page('delete failed'); + } + $notice = "App $app removed."; + app_list($notice); + } else { + page_head('Confirm'); + echo "Are you sure you want to delete app $app?

    "; + show_button( + "buda.php?action=app_delete&app=$app&confirmed=yes", + "Yes" + ); + page_tail(); + } +} + function app_form() { - page_head("Create Docker app"); + page_head('Create Docker app'); form_start(); form_input_text('Name', 'name'); form_submit('OK'); @@ -255,6 +299,7 @@ function app_form() { function app_action() { global $buda_root; $name = get_str('name'); + if (!is_valid_filename($name)) die('bad arg'); $dir = "$buda_root/$name"; if (file_exists($dir)) { error_page("App $name already exists."); @@ -266,8 +311,11 @@ function app_action() { function view_file() { global $buda_root; $app = get_str('app'); + if (!is_valid_filename($app)) die('bad arg'); $variant = get_str('variant'); + if (!is_valid_filename($arg)) die('bad arg'); $fname = get_str('fname'); + if (!is_valid_filename($fname)) die('bad arg'); echo "

    \n";
     readfile("$buda_root/$app/$variant/$fname");
     echo "
    \n"; diff --git a/samples/docker_wrapper/test_buda/worker_3_x86_64-pc-linux-gnu b/samples/docker_wrapper/test_buda/worker_3_x86_64-pc-linux-gnu deleted file mode 100644 index 0aff3a397e1e65ae7aaa5e2acfccb504f223e0e1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 18536 zcmeHPd30OVnZHl+B1^HY#5gz$jUWWFVA;;%BqSkr5(Nbu2#`?71XNj;Y&EhZv}6H7 z-6l{>2z9#9ltLTmlv28du1q_pO$JI}&S?vsw$SZNXq(ms+PYzxu*LKH-9?sO>`v#L z`D6ab&GGy0x8Co5chSA~-IospI@UTYOPJ&omk45gE=6h6jKP&U1JEXxh$=kK6X%Kw z;AN7g_-zV+tI|nr+CXhqIKigg5{Htr!|94F3^X1RmTTTv;OSaLHBbnjBv6mOrq7Wm zpigEDJgEIRrlHXzO1(a<*QfOi9M+B*XzIth@o%HHXNPORV=|DkOdF2px|naSk?3i# zLy?7nM(--<(a(Q0d9~oxy1en?GNAo8P*+zpCE}f}3!4&Mjfr?NzoT)-lGeu7h5l^H zKVR0H_9=Vm6`N#9Y;O}<`g|7tNF%@Qk1ySN`MnQS)f|j%ZLP?5ZvWfrhsiTGWW6k( z=(D^I(k9}MY0f!!PmgU1dzYTrYr2VI6g}@o-dBX1>}+Hv8KekK9*2J3IP}Hi(39iP zyT_q#7>E7{=sx_}6XP(J|Cf(L_l!edGYPu3w! zIMDY;|8XR!f1^1Iqs4+O+m$IlHC{z-pTszC`DD(-wGUjT9X5KKrn_u$HTVlNd{U2N zishMftcYGbj|>*kwP~p`TtwGXm8736qU)(x(oMhM+->SehUpiin{iFL=@+CI_wzfo z{Yucyd_wtwB0A>|lMWQo&0I;|!6JI`@5v)Ybe3V#p(46z1M(g(qPvYS;Nc=V$DT<~ z7ttpeBH*({^y(sdaQ7>&;NJ2T4L%XP{cz4P@@#PTlddNU5+jSgezs4H%=#vtwKLj~ zBi=*)(6JGuS&tHDQx82W@rQ^rbVG+Fejjm$Z0L~0KT4dT8agQPyNEMHLjw~35OH>q zp+1SY)KdWBm@_QUg$`c~L@t#-y5a#maqWK(co`Bi|yr6+^^$8%EWYk35&5Rz?iF8v0z8916M}nJJPK^E8dV+);_+1ZFM(Thr;m=k`UFg(J`}%UCrdiQzEMhDr=$wh zwZ*!-1Fu;x{}8;u$s3+vDd08bJg4ZNbjAiEt3SQhwA z?Y_r>u&@KceP{iUd{}-qXgwR;%hq@YJ(7Ohib8HboS(NlFldb@4YAv$yRQswhSA{e zUBjaG_AjZj#=K2FCU+x`WjlpIFH$#oG@bjd| zFCSZhKDH|#zpnI@kJSYq7a+$*7+QV`_88(^r7}4ey5-WLBj25d>U^Jd!npb%;NXOU zffH{-Z*Tdp>*)HC?{acCl37arPVzUML^l`?_Wy#ehY;eUfk8-$!PQ`&2-eO$jE-A< z8x&5|-gqQdJNs!lKw`E2BekDAyytLk5-PZFgVn!j5C$Gc=`TNtQlq+5PNk;QT9%rC zSy|O*FcLBT+TYuL?YjQw+c&ji?wNI?S57UBgx9q9zq_vgjgC1SFRtK`cb$W&w@{_q z|D5x+{MyV19sO^0^uMvD|F7*MXZDrvW}}g;UgKm$Z#l*dzmAqrx@SEpWwPk+$U(+Ywc2-+v z%QM*<*1jlre|>xW?bbBsGVVP%!4JGSGI9X04PiM9xB-yAJev{Y7XS|fo&+pMt27|^ z&!Q(S0c^u;*9XYNeE^en^VP!I;kBkssB{fjm0r@hc07hM=21YWvE_w0AAHIWpBx!E zMA;he+M0&TYCYRsePZR*Wpfvte;&!yN3)9^u**7_MBOd;yBT$ek=S12y~DA(y225L zN6PlCARh)lP{MC3@b3rxIQUOt02lO~M;sJjng5Ev0d(;wpjcJo{fJ{#O~XCTK#gx- z*{Yh^cb5lin(wFx)-2gwxxS_?Q?sPKrn$Xl_Np2m}6M-`kI1_>YpAj(U_U4>kZ-WbG@#akUpgw#yXTdq0 znWC#@-UsjihRN(@rz*7YUSPV$4gd91BPo&&=*=S!c9{ms6f|f5Piww8(|%5|g*nS^ z(0ILu2AH$=v_8)^dsKNq2qCk7H)rc-YsEvlU_*aIJ7)ZL=|Gux5T^Y1XniAxBL`VO zh)WWV93+0Hwuhq!!G691BRksfRZIe+JBZW4pm$=!*YQ!#6an&=s4k;RPDD zYPeRzYc-5%Xt)1A;pY9&rK?w8?3=x*GoQ@meT)5z{LPIE@)B#gt!0V7dEp#I6&Jyx zYFt&~UAkp%8YglOpeLz5=CUqB>y+_8@I_b{OVRN$NH|_t0uiUH@)UvP>V;*eqw*&N zWtC1yIIW75B$l58P*zrsSA9?@UjVa??~`+PoNmrR!Ond$`z5fP1I(7YDqNqEmMgYF zw32yO7jiF@y*>+4Ra-$Qd!rLiQS~}Ke*0q-sq(U{cbNB99VGo-<{PTKP%1mcyszpm z%8W2S+kGjXWx^uT>|PF6$}EX2aW5jrA(1wB2i40Y5_F$UyX6wu;C_)}6%x75{UIVQ zi*In==RU<^+*T4uk9!q$CRm*iNW05fj#nZ%_jMGTWTnB`;jSiUvb7dSpZii5znC_`Pht)d-!(-9M$H3#4y{-3#c>BI_;)JSiM}5mjFKd5C#9bCuoadI`b} zWd~W`sv}fH+RNbr%DCD>n&skPqq@n#9`T`R$-!V=tf zs|26^sss=GSb~FpmEdz0`a`|z^VJeOJVSyntd!u98zuPSE(yN$F$uo>FA@y5%3@zn zNI&2BnIxY4s|0VZ!emkJdaF!=w`WT5&KAxflQ-c}HS%F-oh@r_Sq)W|HBVZ3TG@O- z8T-95_6KFEGzp9gq*Ij zydv^t&oq~N*$ZGY-c=3|+{-&5B&KJgy~4eM`7+`7B_!OJJPMgCr6b zMvy98oAH!=D2dF}!ULQf2s3;aN_7W)-b)ESz}ydT?s?0}+~gpj0Q-;SE>r%I6vKr6fw{m_utZ1T()z{vlSEV{eqNjL+Oa z_BT1raZK9TK0g@lYFQl)R>>DW-m-FkUWVlxq_Uu?GRH`v6s@dN{?U8ZJq^kF7+&AH#$GunHBDUtl!tD%?>2$ zJ*}StQtw$Tr&iArIkkG0vg_A-E|ycPXStl?JeSC+)zc=YR?liVwR+absnrvZQ>*7v zIkkF%a!&PJCg)Vo<#JB-bjUf?vtG`ro-5>>>e(RYRL@nMQzx)pF0ax*W0vsVf@rZi|A{b9I7pYh-@s$SgX|&RT@-j&3cN^xuS$V$QQ&J7s0}gOAp7EM4}`5{lvt!! z`|<@AaO?gFpv?I;Oeg`z{*F0MI*74rI`+5C#g!>*Ii(!?-#3>xu?*HTH!dtWSvvM# zH;+Gm2HS#;#Rb+yH?xTO{B;~gRhR+_px`Yq*@){}5Kz{yoAVl&2HP+*DVbFpWcXI&kphx3KUa|!t9(K?F`QeF!xh6(|tDlZTOt~$$EDnOrKk9xN|vl@b&jqT{Kaony{^}c3*e(qnq;|PE}uf>bL&ArZ9xV z^9wE&^k)^M%9YFoql6Dzh!;&==ohF;pKQ(kB>2lkqC5o=vH zqVq;WPxPWuGfCNb3T7H%rby54*Wos$eR37TDc(Z&b%-=OvGsOMHJ%pof_PIto8evC zrtV0j372d`tqb*y+wNq(X>n6L8A;^3qC(1PrCJhrjm+)2_@bGM`4jvG8XR!2fs*)fjrfu5b_Yq#e1WQy1X+VPvjco$xG-F7hq<_ za9(KTU7LlY%vq7c(^0XXPuKPGdF}hJZ+`uFx9gs!nU@@YbsK0_MXus|j=6P?d39d_ zcnyDFam=rK2=L$Wmq(tO&ctV&QwFqr-gW#z*B@QYc?kS%zU%n)SY28c?z2Xd+e9ea zlgi{e^ReQnh1Oje;@#I!j`iW?RzxF*btoB-4Mme(q8ArZlaeAYzDgSIrzZ2b^SVR8 zd{;OZWe7zmmI!xe#r8-~I0KWBRC;G99}&3v%3C8@_<&d^rR3%-mKJO*z%nfWQ@AyY zd$)-MZoe|3H{;fbq9wXC1S_4Aw^;aF(A$|yZHIaInhkG*3vf1;NQHT=7i`?7?e@t> zKADYoC!<}s?vjIdxm0hbs$n>?HJ=W3hO<%8l?sJJ-SKYR;!UK|B9Y+Q5qEs)4)4b# zQ@l*>TRm@H%R+TV;cM*8<&)7B-BDC29swH8M0!@JtLnbSZeL>zS79Ps8`GIoE{ak! zji@1<5?S04X2>&Pcue;*(Qub?qca<&6%o#K%L~W&i9*D7EPO{;6n`Q+AQYI`yS-B3E*MPY#X3M^1yp#r!O z9g;V(S*vY1-h&RMVzF#A$7rAjZ(gi>)P^N=groVI=8$OWN%cmXI#ZaQ!d<;8)07SO zrW4U@le&~7)a@sE1)Hie-s~=NP{?k9KFkZ@#&9T|$Y;_1p_}q?j1R@r*TiKn6`@q3 z%lK!sM4_i~2%tW>o0H*QT<6Ym;899No~GU?e7= zm6EY6FsJ>a3@jzk3;#AUN>Mq!M{%IN4%ym~;U#_8Mn6~6&3hSTMTo`7SonS=sN=x$ znAYOY8YBL_pwqs2&qGErz~r z(6?whrb?Nj3W%QqdNE!$_xHRbpD*Dxe3^zygmLqw0--L#4brMF)fjSap(_? zL;vPDbpCEJK6Zb82D@=WgFLYq5U7M8BU<4|9Fb^H22C~(CZ zxr8h+fVK0Q>#t~Eziu_;ia7EDI!IWrhXO%uA-Lvh5xTVF%2n+hp)1#}y(X|Rw6T3v zM}X@d?Dis|NH#C4f<@2x-{oPIjD>JpTk&^%`dh(LqFhdwl0s2zl)~y`!%~j?=Fmps z7l);!)4nFOvuR#^GRRdVK0quL(BCSS673%vhPtw;P!IMc39MSN`cvBit_H_le;37% zjevM|X0zH4x#E@|S(X;dk0(nhTw#||)OVJpv``42Nt)ikb{g}e$kMMdOD&*6SnZb5 zgg?8p7aI;htiC1eF*z)-qnWhuC$Wd~x35~)m(fH;jU_;2<%1*j{GLns+1 zqmR=;nXbhXQ-49<+_y3CIVDzj+T(X4IJ~=J{5Sh01252;#(yJc;8yT?7slXbUuB?A z>)ZX;WWlp~#?MUjIE>f0Z_9_Gt#1`$5#7j6b{nJ)kir^pDB>t{R9H zQ%SP7e?Pdz`iJ$t&%i?^+FX;UWZ=hb`euJEoPw#ULZCw4@;e|wf=*6 zUs#;%vQx&elfJU zC>38=p!5&v`OF8j1%Iaf&F>HA2h;Y*7Sf%|RQ5eKrYYA5PDI8gB3TdX_b8Xx6l|#Y EZ@Z>zMF0Q* From f70ae7f529c6eff715dd9bb1aec6355f6b6983fe Mon Sep 17 00:00:00 2001 From: David Anderson Date: Mon, 25 Nov 2024 14:53:38 -0800 Subject: [PATCH 12/12] check GET args of buda_submit.php --- html/user/buda_submit.php | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/html/user/buda_submit.php b/html/user/buda_submit.php index f797caeb837..2ce3556613f 100644 --- a/html/user/buda_submit.php +++ b/html/user/buda_submit.php @@ -30,7 +30,9 @@ function submit_form($user) { error_page("No .zip files in your sandbox."); } $app = get_str('app'); + if (!is_valid_filename($app)) die('bad arg'); $variant = get_str('variant'); + if (!is_valid_filename($variant)) die('bad arg'); $desc = "
    A zipped directory with one subdirectory per job, @@ -300,8 +302,11 @@ function handle_submit($user) { error_page("No buda app found"); } $app = get_str('app'); + if (!is_valid_filename($app)) die('bad arg'); $variant = get_str('variant'); + if (!is_valid_filename($variant)) die('bad arg'); $batch_file = get_str('batch_file'); + if (!is_valid_filename($batch_file)) die('bad arg'); $variant_dir = "../../buda_apps/$app/$variant"; $variant_desc = json_decode(