diff --git a/.editorconfig b/.editorconfig new file mode 120000 index 000000000000..e7044a352b98 --- /dev/null +++ b/.editorconfig @@ -0,0 +1 @@ +external/os-autoinst-common/.editorconfig \ No newline at end of file diff --git a/container/openqa/entrypoint.sh b/container/openqa/entrypoint.sh index 6edca114e6b4..864d96884f18 100644 --- a/container/openqa/entrypoint.sh +++ b/container/openqa/entrypoint.sh @@ -10,19 +10,19 @@ export LC_ALL="en_US.UTF-8" # First, try to upgrade all container dependencies (or we won't catch bugs # until a new container image is built) -[ "$UPGRADE_FROM_ZYPPER" -eq 1 ] && \ - sudo zypper --gpg-auto-import-keys -n ref --force && \ - sudo zypper up -l -y +[ "$UPGRADE_FROM_ZYPPER" -eq 1 ] \ + && sudo zypper --gpg-auto-import-keys -n ref --force \ + && sudo zypper up -l -y cp -rd /opt/openqa /opt/testing_area cd /opt/testing_area/openqa run_as_normal_user() { if [ "$INSTALL_FROM_CPAN" -eq 1 ]; then - echo ">> Trying to get dependencies from CPAN" - cpanm -M https://cpan.metacpan.org --local-lib=~/perl5 local::lib && cpanm -M https://cpan.metacpan.org -n --installdeps . + echo ">> Trying to get dependencies from CPAN" + cpanm -M https://cpan.metacpan.org --local-lib=~/perl5 local::lib && cpanm -M https://cpan.metacpan.org -n --installdeps . else - cpanm -n --mirror http://no.where/ --installdeps . + cpanm -n --mirror http://no.where/ --installdeps . fi MOJO_TMPDIR=$(mktemp -d) export MOJO_TMPDIR diff --git a/container/openqa_data/scripts/set_openid b/container/openqa_data/scripts/set_openid index 93b113788608..406444f4f36b 100644 --- a/container/openqa_data/scripts/set_openid +++ b/container/openqa_data/scripts/set_openid @@ -6,7 +6,7 @@ read OPENID awk -v openid="$OPENID" ' /^provider/ {$0 = "provider = " openid} {print} -' /data/conf/openqa.ini > /data/conf/openqa.ini.new +' /data/conf/openqa.ini >/data/conf/openqa.ini.new mv /data/conf/openqa.ini.new /data/conf/openqa.ini echo "The file now contains:" diff --git a/container/webui/nginx-entrypoint.sh b/container/webui/nginx-entrypoint.sh index cac5dba20383..ad8dca209842 100755 --- a/container/webui/nginx-entrypoint.sh +++ b/container/webui/nginx-entrypoint.sh @@ -3,11 +3,11 @@ set -e replicas_cfg="" for i in $(seq "${OPENQA_WEBUI_REPLICAS:-2}"); do - replicas_cfg="server webui_webui_$i:9526;$replicas_cfg" + replicas_cfg="server webui_webui_$i:9526;$replicas_cfg" done reg="s/REPLICAS/$replicas_cfg/" -sed "$reg" /etc/nginx/nginx.conf.template > /etc/nginx/nginx.conf +sed "$reg" /etc/nginx/nginx.conf.template >/etc/nginx/nginx.conf echo 'NGINX configuration:' cat /etc/nginx/nginx.conf diff --git a/container/webui/run_openqa.sh b/container/webui/run_openqa.sh index c07cc0952535..c5f0d88a1b94 100644 --- a/container/webui/run_openqa.sh +++ b/container/webui/run_openqa.sh @@ -2,72 +2,72 @@ set -e function wait_for_db_creation() { - echo "Waiting for DB creation" - while ! su geekotest -c 'PGPASSWORD=openqa psql -h db -U openqa --list | grep -qe openqa'; do sleep .1; done + echo "Waiting for DB creation" + while ! su geekotest -c 'PGPASSWORD=openqa psql -h db -U openqa --list | grep -qe openqa'; do sleep .1; done } function upgradedb() { - wait_for_db_creation - su geekotest -c '/usr/share/openqa/script/upgradedb --upgrade_database' + wait_for_db_creation + su geekotest -c '/usr/share/openqa/script/upgradedb --upgrade_database' } function scheduler() { - su geekotest -c /usr/share/openqa/script/openqa-scheduler-daemon + su geekotest -c /usr/share/openqa/script/openqa-scheduler-daemon } function websockets() { - su geekotest -c /usr/share/openqa/script/openqa-websockets-daemon + su geekotest -c /usr/share/openqa/script/openqa-websockets-daemon } function gru() { - wait_for_db_creation - su geekotest -c /usr/share/openqa/script/openqa-gru + wait_for_db_creation + su geekotest -c /usr/share/openqa/script/openqa-gru } function livehandler() { - wait_for_db_creation - su geekotest -c /usr/share/openqa/script/openqa-livehandler-daemon + wait_for_db_creation + su geekotest -c /usr/share/openqa/script/openqa-livehandler-daemon } function webui() { - wait_for_db_creation - su geekotest -c /usr/share/openqa/script/openqa-webui-daemon + wait_for_db_creation + su geekotest -c /usr/share/openqa/script/openqa-webui-daemon } function all_together_apache() { - # use certificate that comes with Mojolicious if none configured by the user (by making one available via `-v`) - local mojo_resources=$(perl -e 'use Mojolicious; print(Mojolicious->new->home->child("Mojo/IOLoop/resources"))') - cp --no-clobber "$mojo_resources"/server.crt /etc/apache2/ssl.crt/server.crt ||: - cp --no-clobber "$mojo_resources"/server.key /etc/apache2/ssl.key/server.key ||: - cp --no-clobber "$mojo_resources"/server.crt /etc/apache2/ssl.crt/ca.crt ||: + # use certificate that comes with Mojolicious if none configured by the user (by making one available via `-v`) + local mojo_resources=$(perl -e 'use Mojolicious; print(Mojolicious->new->home->child("Mojo/IOLoop/resources"))') + cp --no-clobber "$mojo_resources"/server.crt /etc/apache2/ssl.crt/server.crt || : + cp --no-clobber "$mojo_resources"/server.key /etc/apache2/ssl.key/server.key || : + cp --no-clobber "$mojo_resources"/server.crt /etc/apache2/ssl.crt/ca.crt || : - # run the database within the container if no database is configured by the user (by making one available via `-v`) - if [[ ! -e /data/conf/database.ini ]]; then - mkdir -p /data/conf - echo -e "[production]\ndsn = DBI:Pg:dbname=openqa" > /data/conf/database.ini - chown -R postgres:postgres /var/lib/pgsql # ensure right ownership in case `/var/lib/pgsql` is from host via `-v` - su postgres -c '/usr/share/postgresql/postgresql-script start' - su postgres -c '/usr/bin/openqa-setup-db' - fi + # run the database within the container if no database is configured by the user (by making one available via `-v`) + if [[ ! -e /data/conf/database.ini ]]; then + mkdir -p /data/conf + echo -e "[production]\ndsn = DBI:Pg:dbname=openqa" >/data/conf/database.ini + chown -R postgres:postgres /var/lib/pgsql # ensure right ownership in case `/var/lib/pgsql` is from host via `-v` + su postgres -c '/usr/share/postgresql/postgresql-script start' + su postgres -c '/usr/bin/openqa-setup-db' + fi - # run services and apache2 - su geekotest -c /usr/share/openqa/script/openqa-scheduler-daemon & - su geekotest -c /usr/share/openqa/script/openqa-websockets-daemon & - su geekotest -c /usr/share/openqa/script/openqa-gru & - su geekotest -c /usr/share/openqa/script/openqa-livehandler-daemon & - apache2ctl start - su geekotest -c /usr/share/openqa/script/openqa-webui-daemon + # run services and apache2 + su geekotest -c /usr/share/openqa/script/openqa-scheduler-daemon & + su geekotest -c /usr/share/openqa/script/openqa-websockets-daemon & + su geekotest -c /usr/share/openqa/script/openqa-gru & + su geekotest -c /usr/share/openqa/script/openqa-livehandler-daemon & + apache2ctl start + su geekotest -c /usr/share/openqa/script/openqa-webui-daemon } usermod --shell /bin/sh geekotest # run services case "$MODE" in - upgradedb ) upgradedb;; - scheduler ) scheduler;; - websockets ) websockets;; - gru ) gru;; - livehandler ) livehandler;; - webui ) webui;; - * ) all_together_apache;; +upgradedb) upgradedb ;; +scheduler) scheduler ;; +websockets) websockets ;; +gru) gru ;; +livehandler) livehandler ;; +webui) webui ;; +*) all_together_apache ;; esac diff --git a/container/worker/kvm-mknod.sh b/container/worker/kvm-mknod.sh index 7d2321414a58..24e790cae0bd 100644 --- a/container/worker/kvm-mknod.sh +++ b/container/worker/kvm-mknod.sh @@ -2,14 +2,14 @@ # If possible, create the /dev/kvm device node. set -e -kvm=$({ [[ -f /proc/config.gz ]] && test "$(zgrep CONFIG_KVM=y /proc/config.gz)" ; } || true) -$kvm || lsmod | grep '\' > /dev/null || { - echo >&2 "KVM module not loaded; software emulation will be used" - exit 1 +kvm=$({ [[ -f /proc/config.gz ]] && test "$(zgrep CONFIG_KVM=y /proc/config.gz)"; } || true) +$kvm || lsmod | grep '\' >/dev/null || { + echo >&2 "KVM module not loaded; software emulation will be used" + exit 1 } [[ -c /dev/kvm ]] || mknod /dev/kvm c 10 "$(grep '\' /proc/misc | cut -f 1 -d' ')" || { - echo >&2 "Unable to make /dev/kvm node; software emulation will be used" - echo >&2 "(This can happen if the container is run without -privileged)" - exit 1 + echo >&2 "Unable to make /dev/kvm node; software emulation will be used" + echo >&2 "(This can happen if the container is run without -privileged)" + exit 1 } diff --git a/container/worker/launch_workers_pool.sh b/container/worker/launch_workers_pool.sh index ca5bb4e78f90..26ad10d88c80 100755 --- a/container/worker/launch_workers_pool.sh +++ b/container/worker/launch_workers_pool.sh @@ -5,7 +5,7 @@ size=1 cre="${cre:-"podman"}" usage() { - cat << EOF + cat < /dev/null; then - /root/qemu/kvm-mknod.sh + if [ -e "/dev/kvm" ] && getent group kvm >/dev/null; then + /root/qemu/kvm-mknod.sh - group=$(ls -lhn /dev/kvm | cut -d ' ' -f 4) - groupmod -g "$group" --non-unique kvm - usermod -a -G kvm _openqa-worker - else - echo "Warning: /dev/kvm doesn't exist. If you want to use KVM, run the container with --device=/dev/kvm" - fi + group=$(ls -lhn /dev/kvm | cut -d ' ' -f 4) + groupmod -g "$group" --non-unique kvm + usermod -a -G kvm _openqa-worker + else + echo "Warning: /dev/kvm doesn't exist. If you want to use KVM, run the container with --device=/dev/kvm" + fi fi qemu-system-x86_64 -S & kill $! # Install test distribution dependencies -if [[ -z $TEST_DISTRI_DEPS ]] ; then +if [[ -z $TEST_DISTRI_DEPS ]]; then find -L "/var/lib/openqa/share/tests" -maxdepth 2 -type f -executable -name 'install_deps.*' -exec {} \; else zypper -n --gpg-auto-import-keys install $TEST_DISTRI_DEPS diff --git a/dependencies.yaml b/dependencies.yaml index be050498222e..60b3b122a1cb 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -84,6 +84,7 @@ devel_no_selenium_requires: devel_requires: '%devel_no_selenium_requires': chromedriver: + shfmt: main_requires: '%assetpack_requires': diff --git a/script/configure-web-proxy b/script/configure-web-proxy index 78b7ad39f859..c438e8fc6f83 100755 --- a/script/configure-web-proxy +++ b/script/configure-web-proxy @@ -1,7 +1,7 @@ #!/bin/bash -e usage() { - cat << EOF + cat < /etc/nginx/vhosts.d/openqa.conf - sed -i -e "s/\(^[^#]*server_name localhost;\)/#\1/" /etc/nginx/nginx.conf -elif [[ $web_proxy == "apache" || $web_proxy == "apache2" ]] ; then - echo "Setting up apache" - for i in headers proxy proxy_http proxy_wstunnel rewrite ; do a2enmod $i ; done - sed "s/#ServerName.*$/ServerName $(hostname)/" /etc/apache2/vhosts.d/openqa.conf.template > /etc/apache2/vhosts.d/openqa.conf +if [[ $web_proxy == "nginx" ]]; then + echo "Setting up nginx" + sed "s/openqa.example.com/$(hostname)/" /etc/nginx/vhosts.d/openqa.conf.template >/etc/nginx/vhosts.d/openqa.conf + sed -i -e "s/\(^[^#]*server_name localhost;\)/#\1/" /etc/nginx/nginx.conf +elif [[ $web_proxy == "apache" || $web_proxy == "apache2" ]]; then + echo "Setting up apache" + for i in headers proxy proxy_http proxy_wstunnel rewrite; do a2enmod $i; done + sed "s/#ServerName.*$/ServerName $(hostname)/" /etc/apache2/vhosts.d/openqa.conf.template >/etc/apache2/vhosts.d/openqa.conf else - echo "No supported proxy: $web_proxy" - exit 1 + echo "No supported proxy: $web_proxy" + exit 1 fi diff --git a/script/fetchneedles b/script/fetchneedles index 2c44c8f2638b..343c0e79705f 100755 --- a/script/fetchneedles +++ b/script/fetchneedles @@ -18,9 +18,9 @@ : "${updateall:="0"}" : "${force:="0"}" -if [ "$1" = "-h" ] || [ "$1" = "--help" ] ; then +if [ "$1" = "-h" ] || [ "$1" = "--help" ]; then printf "Get %s tests and needles from:\n\t\t%s\n" "${dist_name}" "${giturl}" - if [ "$needles_separate" = 1 ] ; then + if [ "$needles_separate" = 1 ]; then printf "\tand\t%s\n" "${needles_giturl}" fi printf "\n(environment variables mentioned in '%s' can be set to change these values)\n" "$0" @@ -66,7 +66,7 @@ needlesdir() { deal_with_stale_lockfile() { lock_path=.git/index.lock - if [ -e "$lock_path" ] && ! fuser --silent "$lock_path"; then # fuser returns 0 if one or more processes hold the lock + if [ -e "$lock_path" ] && ! fuser --silent "$lock_path"; then # fuser returns 0 if one or more processes hold the lock echo "removing stale lock $PWD/$lock_path" rm "$lock_path" fi diff --git a/script/openqa-auto-update b/script/openqa-auto-update index c3cb89d9d9f6..ed59179d0668 100755 --- a/script/openqa-auto-update +++ b/script/openqa-auto-update @@ -3,7 +3,7 @@ set -e set -o pipefail usage() { - cat << EOF + cat </dev/null || (command -v rebootmgrctl >/dev/null && rebootmgrctl reboot ||:) +needs-restarting --reboothint >/dev/null || (command -v rebootmgrctl >/dev/null && rebootmgrctl reboot || :) diff --git a/script/openqa-bootstrap b/script/openqa-bootstrap index e48cd6a67d24..6986422d0655 100755 --- a/script/openqa-bootstrap +++ b/script/openqa-bootstrap @@ -22,24 +22,23 @@ if [ "$NAME" = "openSUSE Leap" ]; then repobase=https://download.opensuse.org/repositories/devel:/openQA zypper -n addrepo -p 95 "$repobase/${VERSION}" 'devel:openQA' zypper -n addrepo -p 90 "$repobase:/Leap:/${VERSION}/${VERSION}" "devel:openQA:Leap:${VERSION}" - zypper -n --gpg-auto-import-keys refresh + zypper -n --gpg-auto-import-keys refresh fi - # install packages pkgs=(qemu-arm qemu-ppc qemu-x86 qemu-tools sudo iputils os-autoinst-distri-opensuse-deps) if [ "$(uname -m)" = "aarch64" ]; then pkgs+=(qemu-uefi-aarch64) fi -if [[ $setup_web_proxy == "nginx" ]] ; then +if [[ $setup_web_proxy == "nginx" ]]; then pkgs+=(openQA-single-instance-nginx) else pkgs+=(openQA-single-instance) fi # this was split into a separate package on newer dist versions - so install it if available -if zypper -n search -x qemu-hw-display-virtio-gpu-pci ; then +if zypper -n search -x qemu-hw-display-virtio-gpu-pci; then pkgs+=(qemu-hw-display-virtio-gpu qemu-hw-display-virtio-gpu-pci) fi zypper -n install --no-recommends "${pkgs[@]}" @@ -57,17 +56,16 @@ su postgres -c "/usr/share/openqa/script/setup-db" "$dbuser" "$dbname" proxy_args="" [[ -n "$setup_web_proxy" ]] && proxy_args="--proxy=$setup_web_proxy" setup=/usr/share/openqa/script/configure-web-proxy -if command -v $setup ; then +if command -v $setup; then bash -ex $setup "$proxy_args" else curl -s https://raw.githubusercontent.com/os-autoinst/openQA/master/script/configure-web-proxy | bash -ex -s -- "$proxy_args" fi sed -i -e 's/#*.*method.*=.*$/method = Fake/' /etc/openqa/openqa.ini - -if [ -z "$skip_suse_specifics" ] && ping -c1 download.suse.de. && (! rpm -q ca-certificates-suse) ; then +if [ -z "$skip_suse_specifics" ] && ping -c1 download.suse.de. && (! rpm -q ca-certificates-suse); then # add internal CA if executed within suse network - if ! zypper info ca-certificates-suse | grep -q ':' ; then + if ! zypper info ca-certificates-suse | grep -q ':'; then # add suse ca repo if needed # use this way of adding the repo to be distro agnostic if [ "$NAME" = "openSUSE Leap" ]; then @@ -85,16 +83,16 @@ fi # fetch tests and needles if [ -z "$skip_suse_tests" ]; then - if ping -c1 gitlab.suse.de. ; then + if ping -c1 gitlab.suse.de.; then # use faster local mirror if run from within SUSE network export needles_giturl="https://gitlab.suse.de/openqa/os-autoinst-needles-opensuse-mirror.git" fi /usr/share/openqa/script/fetchneedles - if [ ! -e /var/lib/openqa/tests/sle ] ; then + if [ ! -e /var/lib/openqa/tests/sle ]; then ln -s opensuse /var/lib/openqa/tests/sle fi - if ping -c1 gitlab.suse.de. ; then + if ping -c1 gitlab.suse.de.; then sles_needles_giturl="https://gitlab.suse.de/openqa/os-autoinst-needles-sles.git" sles_needles_directory="/var/lib/openqa/tests/opensuse/products/sle/needles" # clone SLE needles if run from within SUSE network @@ -106,10 +104,8 @@ if [ -z "$skip_suse_tests" ]; then fi fi - # ensure that the hostname is mapped to 127.0.0.1 (needed for livehandler) -grep -q "$(hostname)" /etc/hosts || echo "127.0.0.1 $(hostname)" >> /etc/hosts - +grep -q "$(hostname)" /etc/hosts || echo "127.0.0.1 $(hostname)" >>/etc/hosts # start daemons if [ -z "$running_systemd" ]; then @@ -117,14 +113,14 @@ if [ -z "$running_systemd" ]; then pgrep -f openqa-websockets-daemon >/dev/null || su geekotest -c /usr/share/openqa/script/openqa-websockets-daemon & pgrep -f openqa-gru >/dev/null || su geekotest -c /usr/share/openqa/script/openqa-gru & pgrep -f openqa-livehandler-daemon >/dev/null || su geekotest -c /usr/share/openqa/script/openqa-livehandler-daemon & - if [[ $setup_web_proxy == "nginx" ]] ; then + if [[ $setup_web_proxy == "nginx" ]]; then nginx else /usr/sbin/start_apache2 -k start fi pgrep -f openqa-webui-daemon >/dev/null || su geekotest -c /usr/share/openqa/script/openqa-webui-daemon & else - if [[ $setup_web_proxy == "nginx" ]] ; then + if [[ $setup_web_proxy == "nginx" ]]; then systemctl enable --now nginx else systemctl enable --now apache2 @@ -134,7 +130,7 @@ else fi # wait for webui to become available -while ! curl -sI http://localhost/ | grep 200 ; do +while ! curl -sI http://localhost/ | grep 200; do sleep 3 done @@ -144,13 +140,12 @@ API_KEY=$(hexdump -n 8 -e '2/4 "%08X" 1 "\n"' /dev/random) API_SECRET=$(hexdump -n 8 -e '2/4 "%08X" 1 "\n"' /dev/random) echo "INSERT INTO api_keys (key, secret, user_id, t_created, t_updated) VALUES ('${API_KEY}', '${API_SECRET}', 2, NOW(), NOW());" | su postgres -c "psql $dbname" -cat >> /etc/openqa/client.conf <>/etc/openqa/client.conf <&1 ; do systemctl restart systemd-nspawn-openqa@$CONTAINER_NAME.service || true ; sleep 3 ; done +while ! timeout -s9 2 systemd-run -qPM $CONTAINER_NAME /bin/bash -c whoami /dev/null 2>&1; do + systemctl restart systemd-nspawn-openqa@$CONTAINER_NAME.service || true + sleep 3 +done systemd-run "${systemd_run_params[@]}" /bin/bash -c 'update-ca-certificates' systemd-run "${systemd_run_params[@]}" /bin/bash -c '/usr/share/openqa/script/openqa-bootstrap' -echo -e "$(tput setaf 2;tput bold)Your openQA container has been created. Run 'systemd-run -tM $CONTAINER_NAME /bin/bash' to get a shell in the container$(tput sgr0)" +echo -e "$( + tput setaf 2 + tput bold +)Your openQA container has been created. Run 'systemd-run -tM $CONTAINER_NAME /bin/bash' to get a shell in the container$(tput sgr0)" diff --git a/script/openqa-check-devel-repo b/script/openqa-check-devel-repo index 5ccab1055da8..af6633beb8fe 100755 --- a/script/openqa-check-devel-repo +++ b/script/openqa-check-devel-repo @@ -3,7 +3,7 @@ set -e set -o pipefail usage() { - cat << EOF + cat < [CUSTOM_TEST_VAR_1=foo] [CUSTOM_TEST_VAR_2=bar] ... openqa-clone-custom-git-refspec [CUSTOM_TEST_VAR_1=foo] [CUSTOM_TEST_VAR_2=bar] ... @@ -49,7 +49,7 @@ throw_json_error() { } extract_urls_from_pr() { - urls=$(echo "$pr_content" | jq -r '.body | capture("^(?.*@openqa: Clone )(?http.*[0-9]*)") | .url') + urls=$(echo "$pr_content" | jq -r '.body | capture("^(?.*@openqa: Clone )(?http.*[0-9]*)") | .url') if [[ $urls == *"http"* ]]; then echo "$urls" fi @@ -58,14 +58,26 @@ extract_urls_from_pr() { opts=$(getopt -o vhnc: --long verbose,dry-run,help,clone-job-args: -n "$0" -- "$@") || usage 1 eval set -- "$opts" while true; do - case "$1" in - -v | --verbose ) set -x; shift ;; - -h | --help ) usage 0 ;; - -n | --dry-run ) dry_run=true; shift ;; - -c | --clone-job-args ) clone_args="$2 $clone_args"; shift 2 ;; - -- ) shift; break ;; - * ) break ;; - esac + case "$1" in + -v | --verbose) + set -x + shift + ;; + -h | --help) usage 0 ;; + -n | --dry-run) + dry_run=true + shift + ;; + -c | --clone-job-args) + clone_args="$2 $clone_args" + shift 2 + ;; + --) + shift + break + ;; + *) break ;; + esac done job_list="${job:-"$2"}" @@ -76,7 +88,7 @@ if [[ -z "$repo_name" ]] || [[ -z "$pr" ]]; then target_repo_part=${pr_url%%/pull*} pr="${pr_url##*pull/}" pr="${pr%%/*}" - elif [[ $first_arg == *"tree"* ]]; then + elif [[ $first_arg == *"tree"* ]]; then #maybe a branch_url is given branch_url=$first_arg forked_repo_part=${branch_url%%/tree*} @@ -115,8 +127,8 @@ clone_job() { local json_url=${host}/tests/${job}/file/vars.json local json_data json_data=$(eval "${curl_openqa} -s ${json_url}") - echo "$json_data" | jq . &>/dev/null || \ - fail "Unreadable openQA job or no valid JSON data encountered. \ + echo "$json_data" | jq . &>/dev/null \ + || fail "Unreadable openQA job or no valid JSON data encountered. \ Please try 'curl $json_url' or select another job, e.g. in the same scenario: $host/t$job#next_previous" local testsuite="${testsuite:-"$(echo "$json_data" | jq -r '.TEST')"}" || throw_json_error "$json_url" "$json_data" local old_productdir @@ -130,7 +142,7 @@ Please try 'curl $json_url' or select another job, e.g. in the same scenario: $h else local productdir="${productdir:-"${repo_name##*/}/${old_productdir#*"${old_casedir##*/}"}"}" fi - productdir="${productdir/\/\//\/}" # avoid consecutive slashes + productdir="${productdir/\/\//\/}" # avoid consecutive slashes local old_needledir old_needledir=$(echo "$json_data" | jq -r '.NEEDLES_DIR | select (.!=null)') || throw_json_error "$json_url" "$json_data" local needles_dir="${needles_dir:-"$old_needledir"}" @@ -158,6 +170,7 @@ if [[ -z "$host" ]] && [[ -z "$job_list" ]]; then exit 1 fi args=("${@:3}") -IFS=',' ; for i in $job_list; do +IFS=',' +for i in $job_list; do clone_job "$i" done diff --git a/script/openqa-continuous-update b/script/openqa-continuous-update index ee29de641f21..90d294c695ff 100755 --- a/script/openqa-continuous-update +++ b/script/openqa-continuous-update @@ -3,7 +3,7 @@ set -e set -o pipefail usage() { - cat << EOF + cat </dev/null 2>&1 && pwd )" - -echo "openSUSE-Leap-15.1-DVD-x86_64-Build470.2-Media.iso" > "$thisdir"/files_iso.lst -echo "Build469.1" > "$thisdir"/Media1_ftp_ftp.lst +thisdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +echo "openSUSE-Leap-15.1-DVD-x86_64-Build470.2-Media.iso" >"$thisdir"/files_iso.lst +echo "Build469.1" >"$thisdir"/Media1_ftp_ftp.lst diff --git a/t/data/openqa-trigger-from-obs/BatchedProj/Batch2/read_files.sh b/t/data/openqa-trigger-from-obs/BatchedProj/Batch2/read_files.sh index c371d12b6e35..f26b1e7c8d2f 100644 --- a/t/data/openqa-trigger-from-obs/BatchedProj/Batch2/read_files.sh +++ b/t/data/openqa-trigger-from-obs/BatchedProj/Batch2/read_files.sh @@ -1,4 +1,4 @@ -thisdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +thisdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" -echo "openSUSE-Leap-15.1-DVD-x86_64-Snapshot4704-Media.iso" > "$thisdir"/files_iso.lst -echo "openSUSE-Leap-15.1-DVD-x86_64-Snapshot4703-Media.iso" >> "$thisdir"/files_iso.lst +echo "openSUSE-Leap-15.1-DVD-x86_64-Snapshot4704-Media.iso" >"$thisdir"/files_iso.lst +echo "openSUSE-Leap-15.1-DVD-x86_64-Snapshot4703-Media.iso" >>"$thisdir"/files_iso.lst diff --git a/t/data/openqa-trigger-from-obs/Proj1/read_files.sh b/t/data/openqa-trigger-from-obs/Proj1/read_files.sh index 7a869a5026b6..ac296ee78920 100644 --- a/t/data/openqa-trigger-from-obs/Proj1/read_files.sh +++ b/t/data/openqa-trigger-from-obs/Proj1/read_files.sh @@ -1,4 +1,3 @@ -thisdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - -echo "openSUSE-Leap-15.1-DVD-x86_64-Build470.1-Media.iso" > "$thisdir"/files_iso.lst +thisdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +echo "openSUSE-Leap-15.1-DVD-x86_64-Build470.1-Media.iso" >"$thisdir"/files_iso.lst diff --git a/t/data/openqa-trigger-from-obs/script/rsync.sh b/t/data/openqa-trigger-from-obs/script/rsync.sh index e2e1bb0e5f75..87a109f29d09 100755 --- a/t/data/openqa-trigger-from-obs/script/rsync.sh +++ b/t/data/openqa-trigger-from-obs/script/rsync.sh @@ -1,6 +1,6 @@ #!/bin/bash -thisdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +thisdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" function sleep_until_file_created { for ((i = 0; i < 20; i += 1)); do @@ -11,5 +11,8 @@ function sleep_until_file_created { } [[ "$1" =~ MockProjectLongProcessing* ]] && { sleep_until_file_created "$1" || exit 1; } -[ "$1" != MockProjectError ] || { >&2 echo "Mock Error"; exit 1; } +[ "$1" != MockProjectError ] || { + echo >&2 "Mock Error" + exit 1 +} echo MOCK OK $1 diff --git a/t/dummy-isotovideo.sh b/t/dummy-isotovideo.sh index f0e50f8b1b31..41cc4659fdc0 100644 --- a/t/dummy-isotovideo.sh +++ b/t/dummy-isotovideo.sh @@ -2,16 +2,16 @@ set -euo pipefail echo "dummy isotovideo started with arguments: $*" -echo "arguments: $*" > autoinst-log.txt +echo "arguments: $*" >autoinst-log.txt # create fake test results so the web UI thinks we ran at least one test module successfully and considers # the test as passed mkdir -p testresults -echo '[{"category":"dummy","name":"fake","script":"none","flags":{}}]' > testresults/test_order.json -echo '{"dents":0,"details":[],"result":"ok"}' > testresults/result-fake.json -echo '{"current_test":"fake","status":"finished","test_execution_paused":0}' > autoinst-status.json +echo '[{"category":"dummy","name":"fake","script":"none","flags":{}}]' >testresults/test_order.json +echo '{"dents":0,"details":[],"result":"ok"}' >testresults/result-fake.json +echo '{"current_test":"fake","status":"finished","test_execution_paused":0}' >autoinst-status.json # also add a vars.json file for less distracting error messages in the test log -echo '{"ARCH":"x86_64","NAME":"fake-test"}' > vars.json +echo '{"ARCH":"x86_64","NAME":"fake-test"}' >vars.json exit 0 diff --git a/t/test_postgresql b/t/test_postgresql index fa57eea1d9af..7c3f101b675e 100755 --- a/t/test_postgresql +++ b/t/test_postgresql @@ -7,16 +7,18 @@ else DIR=$(readlink -f "$DIR") fi if test -e "$DIR"/postmaster.pid; then - pg_ctl -D "$DIR" stop + pg_ctl -D "$DIR" stop fi if test -d "$DIR"; then - rm -r "$DIR" + rm -r "$DIR" fi initdb --auth-local=peer -N "$DIR" -U "$(id -u -n)" -(echo "listen_addresses=''" -echo "unix_socket_directories='$DIR'" -echo "fsync=off" -echo "full_page_writes=off") >> "$DIR"/postgresql.conf +( + echo "listen_addresses=''" + echo "unix_socket_directories='$DIR'" + echo "fsync=off" + echo "full_page_writes=off" +) >>"$DIR"/postgresql.conf LOGDIR="$DIR/log" LOGFILE="${LOGFILE:-"$LOGDIR/postgresql-openqa-test.log"}" diff --git a/tools/ci/build_autoinst.sh b/tools/ci/build_autoinst.sh index b6e1467161a4..68e9bbafcd21 100755 --- a/tools/ci/build_autoinst.sh +++ b/tools/ci/build_autoinst.sh @@ -9,13 +9,14 @@ destdir=${1:-../os-autoinst} sha=${2:-$(cat tools/ci/autoinst.sha)} [[ -d $destdir ]] || { - sudo mkdir "$destdir" - sudo chown -R $USER "$destdir" + sudo mkdir "$destdir" + sudo chown -R $USER "$destdir" } echo Building os-autoinst $destdir $sha git clone https://github.com/os-autoinst/os-autoinst.git "$destdir" -( cd "$destdir" -git checkout $sha -cmake -G Ninja -DCMAKE_BUILD_TYPE=Release . -ninja symlinks +( + cd "$destdir" + git checkout $sha + cmake -G Ninja -DCMAKE_BUILD_TYPE=Release . + ninja symlinks ) diff --git a/tools/ci/build_cache.sh b/tools/ci/build_cache.sh index 5ae812bb9895..15824d44c87c 100755 --- a/tools/ci/build_cache.sh +++ b/tools/ci/build_cache.sh @@ -8,6 +8,6 @@ set -ex sudo zypper ar -f -p 90 https://download.opensuse.org/repositories/devel:/openQA:/Leap:/15.5/15.5 openQA sudo zypper ar -f -p 95 http://download.opensuse.org/repositories/devel:openQA/15.5 devel tools/retry sudo zypper --gpg-auto-import-keys ref -sudo zypper -n install --download-only $(sed -e 's/\r//' < tools/ci/ci-packages.txt) +sudo zypper -n install --download-only $(sed -e 's/\r//' /dev/null 2>&1 && pwd )" +thisdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" CI_PACKAGES=$thisdir/ci-packages.txt DEPS_BEFORE="${DEPS_BEFORE:-"$(mktemp)"}" @@ -15,15 +15,15 @@ listdeps() { rpm -qa --qf "%{NAME}-%{VERSION}\n" | grep -v gpg-pubkey | grep -v openQA | grep -v os-autoinst | sort } -listdeps > "$DEPS_BEFORE" +listdeps >"$DEPS_BEFORE" sudo zypper ar --priority 91 -f https://download.opensuse.org/repositories/devel:openQA/15.5 devel_openQA sudo zypper ar --priority 90 -f https://download.opensuse.org/repositories/devel:/openQA:/Leap:/15.5/15.5 devel_openQA_Leap tools/retry sudo sh -c 'zypper --gpg-auto-import-keys ref && sudo zypper --no-refresh -n install openQA-devel perl-TAP-Harness-JUnit' -listdeps > "$DEPS_AFTER" +listdeps >"$DEPS_AFTER" -comm -13 "$DEPS_BEFORE" "$DEPS_AFTER" > "$CI_PACKAGES" +comm -13 "$DEPS_BEFORE" "$DEPS_AFTER" >"$CI_PACKAGES" # let's tidy if Tidy version changes newtidyver="$(git diff "$CI_PACKAGES" | grep perl-Perl-Tidy | grep '^+' | grep -o '[0-9.]*' || :)" diff --git a/tools/ci/build_local_container.sh b/tools/ci/build_local_container.sh index 6acabbf59514..d3d360eab331 100755 --- a/tools/ci/build_local_container.sh +++ b/tools/ci/build_local_container.sh @@ -5,16 +5,16 @@ set -e -thisdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +thisdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" cre="${cre:-"podman"}" -$cre build -t localtest -f- "$thisdir"< tools/ci/autoinst.sha + curl -s https://api.github.com/repos/os-autoinst/os-autoinst/commits | grep sha | head -n 1 | grep -o -E '[0-9a-f]{40}' >tools/ci/autoinst.sha git add tools/ci/ci-packages.txt tools/ci/autoinst.sha # these might have changed by tidy in tools/ci/build_dependencies.sh git add cpanfile dbicdh dependencies.yaml tools/* script/* lib/* t/* diff --git a/tools/generate-documentation b/tools/generate-documentation index f63e91d75879..64266a36767f 100755 --- a/tools/generate-documentation +++ b/tools/generate-documentation @@ -41,25 +41,25 @@ set -eo pipefail # exit status 0 if any changes detected anything_changed() { - echo '*.pdf -diff' >> .gitattributes + echo '*.pdf -diff' >>.gitattributes git --no-pager diff --shortstat local lines_changed_in_index lines_changed_in_api other_files_changed # it looks it is a challenge to ignore lines in git diff - lines_changed_in_index="$(git --no-pager diff -U0 -- index.html | grep -v -e 'Last updated ' -e '^\@\@' -e '+++ b' -e '^index ' -e '--- a' | tail -n +2 | wc -l)" + lines_changed_in_index="$(git --no-pager diff -U0 -- index.html | grep -v -e 'Last updated ' -e '^\@\@' -e '+++ b' -e '^index ' -e '--- a' | tail -n +2 | wc -l)" lines_changed_in_api="$(git --no-pager diff -U0 -- api/testapi.html | grep -v -e 'Last updated ' -e '^\@\@' -e '+++ b' -e '^index ' -e '--- a' | tail -n +2 | wc -l)" other_files_changed="$(git --no-pager diff --name-only -- '(:!index.html)' '(:!api/testapi.html)' '(:!current.pdf)' | wc -l)" # if any other file changed || any line besides containing 'Last Updated' in .html - test "$other_files_changed" -gt 0 || test "$((lines_changed_in_index+lines_changed_in_api))" -gt 0 + test "$other_files_changed" -gt 0 || test "$((lines_changed_in_index + lines_changed_in_api))" -gt 0 } check_branch_exists() { local repo=$1 local branch=$2 - if git ls-remote --exit-code -- "$repo" "$branch" ; then + if git ls-remote --exit-code -- "$repo" "$branch"; then return fi @@ -68,8 +68,8 @@ check_branch_exists() { } update_docs() { - mkdir out || exit 1 - ( + mkdir out || exit 1 + ( cd out # shellcheck disable=SC2030 if [[ -z "${PULL_REQUEST_USER}" ]]; then @@ -99,9 +99,9 @@ update_docs() { git add docs/index.html docs/current.pdf docs/api/testapi.html git add docs/api/schema topic="Update documentation to commit ${shortref}" - echo "$topic" > last.commit - echo "" >> last.commit - [[ -z "${CIRCLE_SHA1}" ]] || (cd .. && git log --pretty=fuller "${CIRCLE_SHA1}" -1 >> out/last.commit) + echo "$topic" >last.commit + echo "" >>last.commit + [[ -z "${CIRCLE_SHA1}" ]] || (cd .. && git log --pretty=fuller "${CIRCLE_SHA1}" -1 >>out/last.commit) git commit -F last.commit if [[ -n "${PUBLISH}" ]] && [[ "${PUBLISH}" != 0 ]]; then git push "$PUSH_REMOTE_URL" "$PUSH_BRANCH" @@ -118,35 +118,34 @@ update_docs() { else echo Documentation is up to date fi - ) - [[ $KEEP_OUTPUT ]] || rm -rf out + ) + [[ $KEEP_OUTPUT ]] || rm -rf out } update_api() { - mkdir -p api/src - response=$(curl -o api/src/testapi.pm -sS -w "%{http_code}\n" https://raw.githubusercontent.com/os-autoinst/os-autoinst/master/testapi.pm) - [[ $response != 200 ]] && echo "Download of testapi.pm failed" && exit 1 - cd api - "${scriptroot}"/generate-documentation-genapi - - find . -name '*.asciidoc' -not -name 'header' -exec "${asciidoctor_bin}" {} \; - while IFS= read -r -d '' file - do - header_template "$file" > "$file.tmp" - cat "$file" >> "$file.tmp" - mv "$file.tmp" "$file" - done < <(find . -name '*.html' -print0) - cd .. + mkdir -p api/src + response=$(curl -o api/src/testapi.pm -sS -w "%{http_code}\n" https://raw.githubusercontent.com/os-autoinst/os-autoinst/master/testapi.pm) + [[ $response != 200 ]] && echo "Download of testapi.pm failed" && exit 1 + cd api + "${scriptroot}"/generate-documentation-genapi + + find . -name '*.asciidoc' -not -name 'header' -exec "${asciidoctor_bin}" {} \; + while IFS= read -r -d '' file; do + header_template "$file" >"$file.tmp" + cat "$file" >>"$file.tmp" + mv "$file.tmp" "$file" + done < <(find . -name '*.html' -print0) + cd .. } update_schemas() { - mkdir -p api/schema - cp "$scriptroot"/../public/schema/* api/schema + mkdir -p api/schema + cp "$scriptroot"/../public/schema/* api/schema } header_template() { -filename=$( basename -s .html "$1" ) -cat </dev/null || true # we don't care if the directory already exists + mkdir "${tmpwd}"/output 2>/dev/null || true # we don't care if the directory already exists cp -r images "${tmpwd}"/output [[ ${formats[pdf]} ]] \ @@ -217,7 +219,7 @@ call_asciidoctor() { [[ ${formats[html]} ]] \ && "${asciidoctor_bin}" -o "${tmpwd}"/output/openqa-documentation-"${verbose_doc_name}".html index.asciidoc -d book - echo -e "${green}The output has been generated at ${tmpwd}/output" + echo -e "${green}The output has been generated at ${tmpwd}/output" cd .. } diff --git a/tools/generate-packed-assets b/tools/generate-packed-assets index 0e147d286d28..c57512a2a350 100755 --- a/tools/generate-packed-assets +++ b/tools/generate-packed-assets @@ -36,7 +36,7 @@ generate() { # shellcheck disable=SC2039 for _ in {1..3}; do # shellcheck disable=SC2086 - env MOJO_MODE="$1" perl $BUILD_CACHE_OPTS -e "$BUILD_CACHE" > /dev/null && return 0 + env MOJO_MODE="$1" perl $BUILD_CACHE_OPTS -e "$BUILD_CACHE" >/dev/null && return 0 rc=$? sleep 1 done diff --git a/tools/js-tidy b/tools/js-tidy index 81750eb4c657..36dba5c92402 100755 --- a/tools/js-tidy +++ b/tools/js-tidy @@ -1,7 +1,7 @@ #!/bin/bash usage() { - cat << EOF + cat < /dev/null 2>&1; then +if ! command -v npm >/dev/null 2>&1; then echo "No Node.js found, install it first!" exit 1 fi diff --git a/tools/retry b/tools/retry index c3ca07b49986..0ac5ba41b469 100755 --- a/tools/retry +++ b/tools/retry @@ -28,6 +28,6 @@ else echo "Calling retry hook $HOOK" RETRY_ATTEMPT=$n "$HOOK" fi - n=$((n+1)) + n=$((n + 1)) done fi diff --git a/tools/run-tests-within-container b/tools/run-tests-within-container index b386f5a0d408..744b26c507dc 100755 --- a/tools/run-tests-within-container +++ b/tools/run-tests-within-container @@ -37,7 +37,7 @@ run_fullstack_test() { if ! prove -l "${PROVE_ARGS}" "$@"; then touch tests_failed find '/tmp' -path '*/t/full-stack.d/openqa/testresults/*/autoinst-log.txt' \ - -exec echo 'contents of' {} \; -exec cat {} \; 2> /dev/null > '/opt/openqa/autoinst-log.txt' + -exec echo 'contents of' {} \; -exec cat {} \; 2>/dev/null >'/opt/openqa/autoinst-log.txt' fi } @@ -58,5 +58,5 @@ fi [ -r tests_failed ] && exit 1 if test -n "$TRAVIS"; then - cp -a assets/cache/* /opt/openqa/assets/cache + cp -a assets/cache/* /opt/openqa/assets/cache fi diff --git a/tools/test_containers_compose b/tools/test_containers_compose index a51f10677f44..d6a30bd43aa8 100755 --- a/tools/test_containers_compose +++ b/tools/test_containers_compose @@ -3,56 +3,68 @@ set -euo pipefail wait_until() { - command=$1 - timeout=$2 + command=$1 + timeout=$2 - i=1 - until eval "$command" - do - ((i++)) - if [[ $i -gt $timeout ]]; then - echo "Command '$command' aborted because of timeout" - exit 1 - fi - sleep 1 - done + i=1 + until eval "$command"; do + ((i++)) + if [[ $i -gt $timeout ]]; then + echo "Command '$command' aborted because of timeout" + exit 1 + fi + sleep 1 + done } setup_containers() { - for retry in {2..0}; do sudo docker-compose build && break; echo "Remaining retries $retry"; done - exit_code="" - sudo MOJO_CLIENT_DEBUG=1 docker-compose up -d || exit_code=$? - if [[ -n $exit_code ]]; then - echo "docker-compose exited with non-zero code $exit_code, showing logs:" - docker-compose logs - exit "$exit_code" - fi - (docker-compose ps --services --filter status=stopped | grep "^[[:space:]]*$") || (docker-compose logs; sudo docker-compose ps; exit 1) + for retry in {2..0}; do + sudo docker-compose build && break + echo "Remaining retries $retry" + done + exit_code="" + sudo MOJO_CLIENT_DEBUG=1 docker-compose up -d || exit_code=$? + if [[ -n $exit_code ]]; then + echo "docker-compose exited with non-zero code $exit_code, showing logs:" + docker-compose logs + exit "$exit_code" + fi + (docker-compose ps --services --filter status=stopped | grep "^[[:space:]]*$") || ( + docker-compose logs + sudo docker-compose ps + exit 1 + ) } test_webui() { - ( - workspace=$(mktemp -d) && - trap 'docker-compose down; sudo rm -r "$workspace"' EXIT && - cp -r container/webui "$workspace" && - cd "$workspace/webui" && - sed -i -e "s/method = OpenID/method = Fake/" conf/openqa.ini - printf "[nginx]\nkey = 1234567890ABCDEF\nsecret = 1234567890ABCDEF\n" > conf/client.conf && - setup_containers && - docker-compose exec -T webui openqa-cli api -X POST jobs ISO=foo.iso DISTRI=my-distri FLAVOR=my-flavor VERSION=42 BUILD=42 TEST=my-test \ - --host http://nginx:9526 || (echo "Error executing a job"; exit 1) && - (wait_until 'docker-compose logs webui | grep "GET /api/wakeup" >/dev/null' 10) || (docker-compose logs webui; exit 1) - ) || exit 1; + ( + workspace=$(mktemp -d) \ + && trap 'docker-compose down; sudo rm -r "$workspace"' EXIT \ + && cp -r container/webui "$workspace" \ + && cd "$workspace/webui" \ + && sed -i -e "s/method = OpenID/method = Fake/" conf/openqa.ini + printf "[nginx]\nkey = 1234567890ABCDEF\nsecret = 1234567890ABCDEF\n" >conf/client.conf \ + && setup_containers \ + && docker-compose exec -T webui openqa-cli api -X POST jobs ISO=foo.iso DISTRI=my-distri FLAVOR=my-flavor VERSION=42 BUILD=42 TEST=my-test \ + --host http://nginx:9526 || ( + echo "Error executing a job" + exit 1 + ) \ + && (wait_until 'docker-compose logs webui | grep "GET /api/wakeup" >/dev/null' 10) || ( + docker-compose logs webui + exit 1 + ) + ) || exit 1 } test_worker() { - ( - workspace=$(mktemp -d) && - trap 'docker-compose down; sudo rm -r "$workspace"' EXIT && - cp -r container/worker "$workspace" && - cd "$workspace/worker" && - setup_containers - ) || exit 1; + ( + workspace=$(mktemp -d) \ + && trap 'docker-compose down; sudo rm -r "$workspace"' EXIT \ + && cp -r container/worker "$workspace" \ + && cd "$workspace/worker" \ + && setup_containers + ) || exit 1 } test_webui diff --git a/tools/test_helm_chart b/tools/test_helm_chart index cdc5e5a89e61..8c7a835a98dd 100755 --- a/tools/test_helm_chart +++ b/tools/test_helm_chart @@ -2,7 +2,7 @@ set -euo pipefail which helm >/dev/null || echo "Error: Helm is not installed" -which ct > /dev/null || echo "Error: ct is not installed" +which ct >/dev/null || echo "Error: ct is not installed" cd container/helm diff --git a/tools/tidy b/tools/tidy index f16b5e332df5..8056d2abdf1a 100755 --- a/tools/tidy +++ b/tools/tidy @@ -4,7 +4,7 @@ # perltidy rules can be found in ../.perltidyrc # usage() { - cat << EOF + cat < /dev/null 2>&1; then +if ! command -v perltidy >/dev/null 2>&1; then echo "No perltidy found, install it first!" exit 1 fi