Skip to content

Commit

Permalink
feat: Makefile 移除无用的 bcs-ops-offline-release (#2701)
Browse files Browse the repository at this point in the history
feat: 运行时 增加对 insecure_registry 的配置
feat: 支持curl方式下载离线包(for bkrepo),新增INSTALL_METHOD变量
feat: yq/jq 离线部署支持

fix:
1. 离线文件路径修正为 ${ROOT_DIR}/version-${K8S_VER}
2. docker 离线部署时添加 docker 用户组
3. 离线部署时,kubeadm 不执行 config image pull
4. 增加 sleep 等待,防止kubelet 反复挂起,导致判定失败。
5. 添加node_name域名解析, 减少 kubeadm warning 告警

change: csi取消默认配置 localpv。

style: shell-format
  • Loading branch information
bingoct authored Oct 24, 2023
1 parent 845ef1e commit 965f84d
Show file tree
Hide file tree
Showing 16 changed files with 411 additions and 108 deletions.
4 changes: 1 addition & 3 deletions bcs-ops/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,11 @@ CURRENT_VERSION = release-$(VER)

clean:
-rm ./bcs-ops-script-release-$(VER).tar.gz
-rm ./bcs-ops-offline-release-$(VER).tar.gz
-rm MD5SUMS

build:clean
find . -not -path "*/.git/*" -a -not -path "*/bin/*" -a -not -path "*/image/*" -a -not -path "*/Makefile" -a -not -path "*/functions/*" -type f -print0 | xargs -0 chmod 555
find ./functions/ -not -path "*/.git/*" -a -not -path "*/bin/*" -a -not -path "*/image/*" -a -not -path "*/Makefile" -type f -print0 | xargs -0 chmod 444
tar -czvf bcs-ops-script-release-$(VER).tar.gz --exclude=bin --exclude=image --exclude=Makefile --exclude=\..* --exclude=.*tar.gz ./*
tar -czvf bcs-ops-offline-release-$(VER).tar.gz --exclude=Makefile --exclude=\..* --exclude=.*tar.gz ./*
md5sum bcs-ops-script-release-$(VER).tar.gz >> MD5SUMS
md5sum bcs-ops-offline-release-$(VER).tar.gz >> MD5SUMS
md5sum bcs-ops-offline-release-$(VER).tar.gz >> MD5SUMS
8 changes: 6 additions & 2 deletions bcs-ops/install_master.sh
Original file line number Diff line number Diff line change
Expand Up @@ -55,9 +55,13 @@ safe_source "${ROOT_DIR}/functions/k8s.sh"
safe_source "${ROOT_DIR}/env/bcs.env"

# pull image
kubeadm --config="${ROOT_DIR}/kubeadm-config" config images pull \
|| utils::log "FATAL" "fail to pull k8s image"
if [[ -z ${BCS_OFFLINE:-} ]]; then
kubeadm --config="${ROOT_DIR}/kubeadm-config" config images pull \
|| utils::log "FATAL" "fail to pull k8s image"
fi

# wait to check kubelet start
sleep 30
if [[ -z ${MASTER_JOIN_CMD:-} ]]; then
if systemctl is-active kubelet.service -q; then
utils::log "WARN" "kubelet service is active now, skip kubeadm init"
Expand Down
9 changes: 7 additions & 2 deletions bcs-ops/install_node.sh
Original file line number Diff line number Diff line change
Expand Up @@ -122,16 +122,21 @@ case "${K8S_CSI,,}" in
;;
esac

kubeadm --config="${ROOT_DIR}/kubeadm-config" config images pull \
|| utils::log "FATAL" "fail to pull k8s image"
if [[ -z ${BCS_OFFLINE:-} ]]; then
kubeadm --config="${ROOT_DIR}/kubeadm-config" config images pull \
|| utils::log "FATAL" "fail to pull k8s image"
fi

# wait kubelet to start
sleep 30
if systemctl is-active kubelet.service -q; then
utils::log "WARN" "kubelet service is active now, skip kubeadm join"
else
kubeadm join --config="${ROOT_DIR}/kubeadm-config" -v 11 \
|| utils::log "FATAL" "${LAN_IP} failed to join cluster: ${K8S_CTRL_IP}"
fi


if [[ "${ENABLE_APISERVER_HA}" == "true" ]]; then
if [[ "${APISERVER_HA_MODE}" == "bcs-apiserver-proxy" ]]; then
init_bap_rule
Expand Down
152 changes: 152 additions & 0 deletions bcs-ops/k8s/insecure_registry.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,152 @@
#!/bin/bash
set -euo pipefail

# 通用脚本框架变量
PROGRAM=$(basename "$0")

# 定义需要设置为免证书信任的registry地址
REGISTRIES=()
TIMESTMP=$(date +%s)
CRI_TYPE=""
ACTION=""

usage_and_exit() {
cat <<EOF
免证书信任的registry地址,docker 需要
Usage:
$PROGRAM -c containerd -a docker.example.com docker.example2.com:8080
$PROGRAM -c docker -d docker.example.com docker.example2.com:8080
$PROGRAM [ -h --help -? show usage ]
[ -a, --add add insecure registry]
[ -d, --del remove insecure registry]
[ -c, --cri-type support docker\containerd]
EOF
exit "$1"
}

version() {
echo "$PROGRAM version $VERSION"
}

while (($# > 0)); do
case "$1" in
-a | --add)
shift
if [[ -z $ACTION ]]; then
ACTION="add"
else
echo "ACTION already define: ${ACTION}"
usage_and_exit 1
fi
while (($# > 0)) && [[ "$1" != -* ]]; do
REGISTRIES+=("$1")
shift
done
continue
;;
-d | --del)
shift
if [[ -z $ACTION ]]; then
ACTION="del"
else
echo "ACTION already define: ${ACTION}"
usage_and_exit 1
fi
while (($# > 0)) && [[ "$1" != -* ]]; do
REGISTRIES+=("$1")
shift
done
continue
;;
-c | --cri-type)
shift
CRI_TYPE=$1
;;
--help | -h | '-?')
usage_and_exit 0
;;
-*)
error "不可识别的参数: $1"
;;
*)
break
;;
esac
(($# > 0)) && shift
done

add_docker() {
# 获取docker配置文件路径
DOCKER_CONFIG_PATH="/etc/docker/daemon.json"

# 文件不存在,则需要创建
if [[ ! -f "$DOCKER_CONFIG_PATH" ]]; then
echo "{}" >"$DOCKER_CONFIG_PATH"
fi

cp $DOCKER_CONFIG_PATH $DOCKER_CONFIG_PATH.registry.tmp

registries=$(printf '"%s",' "${REGISTRIES[@]}")
registries="[${registries%,}]"

jq --arg k 'insecure-registries' --argjson v "$registries" '.[$k] as $insecure_registries | if $insecure_registries then reduce $v[] as $r (.; if $insecure_registries | index($r) == null then .[$k] += [$r] else . end) else .[$k] = $v end' $DOCKER_CONFIG_PATH >/tmp/docker_daemon-"${TIMESTMP}".tmp

cp "$DOCKER_CONFIG_PATH" "$DOCKER_CONFIG_PATH.${TIMESTMP}.bak"
mv /tmp/docker_daemon-"${TIMESTMP}".tmp "$DOCKER_CONFIG_PATH"
cat "$DOCKER_CONFIG_PATH"

# 重启docker服务
systemctl reload docker
}

del_docker() {
DOCKER_CONFIG_PATH="/etc/docker/daemon.json"

if [[ ! -f "$DOCKER_CONFIG_PATH" ]]; then
echo "{}" >"$DOCKER_CONFIG_PATH"
cat $DOCKER_CONFIG_PATH
return 0
fi

registries=$(printf '"%s",' "${REGISTRIES[@]}")
registries="[${registries%,}]"

jq --arg k 'insecure-registries' --argjson v "$registries" '.[$k] as $insecure_registries | if $insecure_registries then reduce $v[] as $r (.; if $insecure_registries | index($r) != null then .[$k] -= [$r] else . end) else .[$k] = $v end' $DOCKER_CONFIG_PATH >/tmp/docker_daemon-"${TIMESTMP}".tmp

cp "$DOCKER_CONFIG_PATH" "$DOCKER_CONFIG_PATH.$TIMESTMP.bak"
mv /tmp/docker_daemon-"${TIMESTMP}".tmp "$DOCKER_CONFIG_PATH"
cat "$DOCKER_CONFIG_PATH"

systemctl reload docker
}

add_containerd() {
local registry
for registry in "${REGISTRIES[@]}"; do
CONTAINERD_HOST_DIR="/etc/containerd/certs.d/${registry}"
mkdir -p "$CONTAINERD_HOST_DIR"
if [[ -f $CONTAINERD_HOST_DIR/hosts.toml ]]; then
cp "$CONTAINERD_HOST_DIR/hosts.toml" "$CONTAINERD_HOST_DIR/hosts.toml.${TIMESTMP}.bak"
fi
cat <<EOF >"$CONTAINERD_HOST_DIR/hosts.toml"
[host."https://$registry"]
capabilities = ["pull", "resolve", "push"]
skip_verify = true
EOF
done
}

del_containerd() {
local registry
for registry in "${REGISTRIES[@]}"; do
CONTAINERD_HOST_DIR="/etc/containerd/certs.d/${registry}"
if [[ -f $CONTAINERD_HOST_DIR/host.toml ]]; then
if grep -q "skip_verify = true" "$CONTAINERD_HOST_DIR"/host.toml; then
cp "$CONTAINERD_HOST_DIR/hosts.toml" "$CONTAINERD_HOST_DIR/hosts.toml.${TIMESTMP}.bak"
sed -i '/skip_verify = true/d' "$CONTAINERD_HOST_DIR"/host.toml
fi
fi
done
}

"${ACTION}_${CRI_TYPE}"
50 changes: 46 additions & 4 deletions bcs-ops/k8s/install_containerd
Original file line number Diff line number Diff line change
Expand Up @@ -68,9 +68,37 @@ _yum_containerd() {
return 0
}

_curl_containerd() {
local bin_path name ver file url
bin_path=${ROOT_DIR}/version-${K8S_VER}/bin-tools/
mkdir -p "$bin_path"

name="containerd"
ver=$(awk '/version: \"'"${K8S_VER}"'\"/{f=1;next} f && /'"${name}"':/{gsub("\"","",$2);print $2;exit}' "${ROOT_DIR}"/env/offline-manifest.yaml)
file="${name}-${ver}.tgz"
url=${REPO_URL}/${file}
if curl -sSfL "${url}" -o "${bin_path}/${file}" -m "360"; then
utils::log "INFO" "Downloaded ${url}"
else
utils::log "ERROR" "fail to download ${url}"
fi

name="runc"
ver=$(awk '/version: \"'"${K8S_VER}"'\"/{f=1;next} f && /'"${name}"':/{gsub("\"","",$2);print $2;exit}' "${ROOT_DIR}"/env/offline-manifest.yaml)
file="${name}-${ver}.tgz"
url="${REPO_URL}/${file}"
if curl -sSfL "${url}" -o "${bin_path}/${file}" -m "360"; then
utils::log "INFO" "Downloaded ${url}"
else
utils::log "ERROR" "fail to download ${url}"
fi

_offline_containerd
}

_offline_containerd() {
local bin_path tar_name
bin_path=${ROOT_DIR}/version-${VERSION}/bin-tools/
bin_path=${ROOT_DIR}/version-${K8S_VER}/bin-tools/

tar_name=$(find "$bin_path" -iname "containerd-*.tgz" -type f | head -1)
if [[ -z ${tar_name} ]]; then
Expand Down Expand Up @@ -165,9 +193,19 @@ main() {
utils::log "WARN" "containerd installed, $(ctr -v)"
else
if [[ -n ${BCS_OFFLINE:-} ]]; then
_offline_containerd
_offline_containerd
else
_yum_containerd
case ${INSTALL_METHOD} in
"yum")
_yum_containerd
;;
"curl")
_curl_containerd
;;
*)
utils::log "ERROR" "unkown ${INSTALL_METHOD} to exec download containerd"
;;
esac
fi
fi

Expand All @@ -181,6 +219,11 @@ main() {
utils::log "ERROR" "Did containerd get installed?"
fi

# add insecure_registry
if [[ -n ${INSECURE_REGISTRY:-} ]]; then
"${ROOT_DIR}"/k8s/insecure_registry.sh -c containerd -a "${INSECURE_REGISTRY}"
fi

if [[ -n ${BCS_OFFLINE:-} ]]; then
find "${ROOT_DIR}"/version-"${VERSION}"/images -name '*.tar' -type f -print0 \
| xargs -0 -I {} ctr -n k8s.io image import {}
Expand All @@ -191,7 +234,6 @@ main() {
test_img_url=${BK_PUBLIC_REPO:-"docker.io"}/library/hello-world:latest
utils::log "DEBUG" "hello-world: ${test_img_url}"


if ! (ctr -n k8s.io i pull --hosts-dir "/etc/containerd/certs.d" "$test_img_url" \
&& ctr -n k8s.io run --rm "$test_img_url" hello-world."$(date +%s)"); then
utils::log "ERROR" "Could not get containerd to run ${test_img_url}"
Expand Down
52 changes: 44 additions & 8 deletions bcs-ops/k8s/install_docker
Original file line number Diff line number Diff line change
Expand Up @@ -74,9 +74,27 @@ _yum_docker() {
return 0
}

_curl_docker() {
local bin_path name ver file url
bin_path=${ROOT_DIR}/version-${K8S_VER}/bin-tools/
mkdir -p "$bin_path"

name="docker"
ver=$(awk '/version: \"'"${K8S_VER}"'\"/{f=1;next} f && /'"${name}"':/{gsub("\"","",$2);print $2;exit}' "${ROOT_DIR}"/env/offline-manifest.yaml)
file="${name}-${ver}.tgz"
url=${REPO_URL}/${file}
if curl -sSfL "${url}" -o "${bin_path}/${file}" -m "360"; then
utils::log "INFO" "Downloaded ${url}"
else
utils::log "ERROR" "fail to download ${url}"
fi

_offline_docker
}

_offline_docker() {
local bin_path tar_name
bin_path=${ROOT_DIR}/version-${VERSION}/bin-tools/
bin_path=${ROOT_DIR}/version-${K8S_VER}/bin-tools/
tar_name=$(find "$bin_path" -iname "docker-*.tgz" -type f | head -1)

if [[ -z ${tar_name} ]]; then
Expand All @@ -85,6 +103,14 @@ _offline_docker() {
tar xvzf "${tar_name}" -C /usr/bin/ --strip-components=1 bin/
tar xvzf "${tar_name}" -C /etc/systemd/system/ --strip-components=1 systemd/
fi

# docker.sock need docker group
if getent group docker >/dev/null 2>&1; then
utils::log "INFO" "docker group existed"
else
utils::log "INFO" "creating docker group"
groupadd docker
fi
}

# ToDo: config separte
Expand Down Expand Up @@ -150,7 +176,17 @@ main() {
if [[ -n ${BCS_OFFLINE:-} ]]; then
_offline_docker
else
_yum_docker
case ${INSTALL_METHOD} in
"yum")
_yum_docker
;;
"curl")
_curl_docker
;;
*)
utils::log "ERROR" "unkown ${INSTALL_METHOD} to exec download docker"
;;
esac
fi
fi

Expand All @@ -164,7 +200,12 @@ main() {
utils::log "ERROR" "Did docker get installed?"
fi

# load image
# add insecure_registry
if [[ -n ${INSECURE_REGISTRY:-} ]]; then
"${ROOT_DIR}"/k8s/insecure_registry.sh -c docker -a "${INSECURE_REGISTRY}"
fi

# bcs_offline load image
if [[ -n ${BCS_OFFLINE:-} ]]; then
find "${ROOT_DIR}"/version-"${VERSION}"/images -name '*.tar' -type f -print0 \
| xargs -0 -I {} docker load -i {}
Expand All @@ -175,11 +216,6 @@ main() {
test_img_url=${BK_PUBLIC_REPO:-"docker.io"}/library/hello-world:latest
utils::log "DEBUG" "hello-world: ${test_img_url}"

if [[ -n ${BCS_OFFLINE:-} ]]; then
# ToDo hello world image offline install
true
fi

if ! docker run --rm "${test_img_url}"; then
utils::log "ERROR" "Count not get docker to run ${test_img_url}"
fi
Expand Down
Loading

0 comments on commit 965f84d

Please sign in to comment.