TencentBlueKing · abstractmj · Oct 24, 2023 · Oct 19, 2023
diff --git a/bcs-ops/Makefile b/bcs-ops/Makefile
@@ -4,13 +4,11 @@ CURRENT_VERSION = release-$(VER)
 
 clean:
 	-rm ./bcs-ops-script-release-$(VER).tar.gz
-	-rm ./bcs-ops-offline-release-$(VER).tar.gz
 	-rm MD5SUMS
 
 build:clean
 	find . -not -path "*/.git/*" -a -not -path "*/bin/*" -a -not -path "*/image/*" -a -not -path "*/Makefile" -a -not -path "*/functions/*" -type f -print0 | xargs -0 chmod 555
 	find ./functions/ -not -path "*/.git/*" -a -not -path "*/bin/*" -a -not -path "*/image/*" -a -not -path "*/Makefile" -type f -print0 | xargs -0 chmod 444
 	tar -czvf bcs-ops-script-release-$(VER).tar.gz  --exclude=bin --exclude=image --exclude=Makefile --exclude=\..* --exclude=.*tar.gz ./*
-	tar -czvf bcs-ops-offline-release-$(VER).tar.gz  --exclude=Makefile --exclude=\..* --exclude=.*tar.gz ./*
 	md5sum bcs-ops-script-release-$(VER).tar.gz >> MD5SUMS
-	md5sum bcs-ops-offline-release-$(VER).tar.gz >> MD5SUMS
+	md5sum bcs-ops-offline-release-$(VER).tar.gz >> MD5SUMS
diff --git a/bcs-ops/install_master.sh b/bcs-ops/install_master.sh
@@ -55,9 +55,13 @@ safe_source "${ROOT_DIR}/functions/k8s.sh"
 safe_source "${ROOT_DIR}/env/bcs.env"
 
 # pull image
-kubeadm --config="${ROOT_DIR}/kubeadm-config" config images pull \
-  || utils::log "FATAL" "fail to pull k8s image"
+if [[ -z ${BCS_OFFLINE:-} ]]; then
+  kubeadm --config="${ROOT_DIR}/kubeadm-config" config images pull \
+    || utils::log "FATAL" "fail to pull k8s image"
+fi
 
+# wait to check kubelet start
+sleep 30
 if [[ -z ${MASTER_JOIN_CMD:-} ]]; then
   if systemctl is-active kubelet.service -q; then
     utils::log "WARN" "kubelet service is active now, skip kubeadm init"

diff --git a/bcs-ops/install_node.sh b/bcs-ops/install_node.sh
@@ -122,16 +122,21 @@ case "${K8S_CSI,,}" in
     ;;
 esac
 
-kubeadm --config="${ROOT_DIR}/kubeadm-config" config images pull \
-  || utils::log "FATAL" "fail to pull k8s image"
+if [[ -z ${BCS_OFFLINE:-} ]]; then
+  kubeadm --config="${ROOT_DIR}/kubeadm-config" config images pull \
+    || utils::log "FATAL" "fail to pull k8s image"
+fi
 
+# wait kubelet to start
+sleep 30
 if systemctl is-active kubelet.service -q; then
   utils::log "WARN" "kubelet service is active now, skip kubeadm join"
 else
   kubeadm join --config="${ROOT_DIR}/kubeadm-config" -v 11 \
     || utils::log "FATAL" "${LAN_IP} failed to join cluster: ${K8S_CTRL_IP}"
 fi
 
+
 if [[ "${ENABLE_APISERVER_HA}" == "true" ]]; then
   if [[ "${APISERVER_HA_MODE}" == "bcs-apiserver-proxy" ]]; then
     init_bap_rule

diff --git a/bcs-ops/k8s/insecure_registry.sh b/bcs-ops/k8s/insecure_registry.sh
@@ -0,0 +1,152 @@
+#!/bin/bash
+set -euo pipefail
+
+# 通用脚本框架变量
+PROGRAM=$(basename "$0")
+
+# 定义需要设置为免证书信任的registry地址
+REGISTRIES=()
+TIMESTMP=$(date +%s)
+CRI_TYPE=""
+ACTION=""
+
+usage_and_exit() {
+    cat <<EOF
+	免证书信任的registry地址，docker 需要
+Usage:
+	$PROGRAM -c containerd -a docker.example.com docker.example2.com:8080
+	$PROGRAM -c docker -d docker.example.com docker.example2.com:8080
+    $PROGRAM [ -h --help -?  show usage ]
+			 [ -a, --add add insecure registry]
+			 [ -d, --del remove insecure registry]
+             [ -c, --cri-type  support docker\containerd]
+EOF
+    exit "$1"
+}
+
+version() {
+    echo "$PROGRAM version $VERSION"
+}
+
+while (($# > 0)); do
+    case "$1" in
+        -a | --add)
+            shift
+            if [[ -z $ACTION ]]; then
+                ACTION="add"
+            else
+                echo "ACTION already define: ${ACTION}"
+                usage_and_exit 1
+            fi
+            while (($# > 0)) && [[ "$1" != -* ]]; do
+                REGISTRIES+=("$1")
+                shift
+            done
+            continue
+            ;;
+        -d | --del)
+            shift
+            if [[ -z $ACTION ]]; then
+                ACTION="del"
+            else
+                echo "ACTION already define: ${ACTION}"
+                usage_and_exit 1
+            fi
+            while (($# > 0)) && [[ "$1" != -* ]]; do
+                REGISTRIES+=("$1")
+                shift
+            done
+            continue
+            ;;
+        -c | --cri-type)
+            shift
+            CRI_TYPE=$1
+            ;;
+        --help | -h | '-?')
+            usage_and_exit 0
+            ;;
+        -*)
+            error "不可识别的参数: $1"
+            ;;
+        *)
+            break
+            ;;
+    esac
+    (($# > 0)) && shift
+done
+
+add_docker() {
+    # 获取docker配置文件路径
+    DOCKER_CONFIG_PATH="/etc/docker/daemon.json"
+
+    # 文件不存在，则需要创建
+    if [[ ! -f "$DOCKER_CONFIG_PATH" ]]; then
+        echo "{}" >"$DOCKER_CONFIG_PATH"
+    fi
+
+    cp $DOCKER_CONFIG_PATH $DOCKER_CONFIG_PATH.registry.tmp
+
+    registries=$(printf '"%s",' "${REGISTRIES[@]}")
+    registries="[${registries%,}]"
+
+    jq --arg k 'insecure-registries' --argjson v "$registries" '.[$k] as $insecure_registries | if $insecure_registries then reduce $v[] as $r (.; if $insecure_registries | index($r) == null then .[$k] += [$r] else . end) else .[$k] = $v end' $DOCKER_CONFIG_PATH >/tmp/docker_daemon-"${TIMESTMP}".tmp
+
+    cp "$DOCKER_CONFIG_PATH" "$DOCKER_CONFIG_PATH.${TIMESTMP}.bak"
+    mv /tmp/docker_daemon-"${TIMESTMP}".tmp "$DOCKER_CONFIG_PATH"
+    cat "$DOCKER_CONFIG_PATH"
+
+    # 重启docker服务
+    systemctl reload docker
+}
+
+del_docker() {
+    DOCKER_CONFIG_PATH="/etc/docker/daemon.json"
+
+    if [[ ! -f "$DOCKER_CONFIG_PATH" ]]; then
+        echo "{}" >"$DOCKER_CONFIG_PATH"
+        cat $DOCKER_CONFIG_PATH
+        return 0
+    fi
+
+    registries=$(printf '"%s",' "${REGISTRIES[@]}")
+    registries="[${registries%,}]"
+
+    jq --arg k 'insecure-registries' --argjson v "$registries" '.[$k] as $insecure_registries | if $insecure_registries then reduce $v[] as $r (.; if $insecure_registries | index($r) != null then .[$k] -= [$r] else . end) else .[$k] = $v end' $DOCKER_CONFIG_PATH >/tmp/docker_daemon-"${TIMESTMP}".tmp
+
+    cp "$DOCKER_CONFIG_PATH" "$DOCKER_CONFIG_PATH.$TIMESTMP.bak"
+    mv /tmp/docker_daemon-"${TIMESTMP}".tmp "$DOCKER_CONFIG_PATH"
+    cat "$DOCKER_CONFIG_PATH"
+
+    systemctl reload docker
+}
+
+add_containerd() {
+    local registry
+    for registry in "${REGISTRIES[@]}"; do
+        CONTAINERD_HOST_DIR="/etc/containerd/certs.d/${registry}"
+        mkdir -p "$CONTAINERD_HOST_DIR"
+        if [[ -f $CONTAINERD_HOST_DIR/hosts.toml ]]; then
+            cp "$CONTAINERD_HOST_DIR/hosts.toml" "$CONTAINERD_HOST_DIR/hosts.toml.${TIMESTMP}.bak"
+        fi
+        cat <<EOF >"$CONTAINERD_HOST_DIR/hosts.toml"
+[host."https://$registry"]
+  capabilities = ["pull", "resolve", "push"]
+  skip_verify = true
+EOF
+    done
+}
+
+del_containerd() {
+    local registry
+    for registry in "${REGISTRIES[@]}"; do
+        CONTAINERD_HOST_DIR="/etc/containerd/certs.d/${registry}"
+        if [[ -f $CONTAINERD_HOST_DIR/host.toml ]]; then
+            if grep -q "skip_verify = true" "$CONTAINERD_HOST_DIR"/host.toml; then
+                cp "$CONTAINERD_HOST_DIR/hosts.toml" "$CONTAINERD_HOST_DIR/hosts.toml.${TIMESTMP}.bak"
+                sed -i '/skip_verify = true/d' "$CONTAINERD_HOST_DIR"/host.toml
+            fi
+        fi
+    done
+}
+
+"${ACTION}_${CRI_TYPE}"
diff --git a/bcs-ops/k8s/install_containerd b/bcs-ops/k8s/install_containerd
@@ -68,9 +68,37 @@ _yum_containerd() {
   return 0
 }
 
+_curl_containerd() {
+  local bin_path name ver file url
+  bin_path=${ROOT_DIR}/version-${K8S_VER}/bin-tools/
+  mkdir -p "$bin_path"
+
+  name="containerd"
+  ver=$(awk '/version: \"'"${K8S_VER}"'\"/{f=1;next} f && /'"${name}"':/{gsub("\"","",$2);print $2;exit}' "${ROOT_DIR}"/env/offline-manifest.yaml)
+  file="${name}-${ver}.tgz"
+  url=${REPO_URL}/${file}
+  if curl -sSfL "${url}" -o "${bin_path}/${file}" -m "360"; then
+    utils::log "INFO" "Downloaded ${url}"
+  else
+    utils::log "ERROR" "fail to download ${url}"
+  fi
+
+  name="runc"
+  ver=$(awk '/version: \"'"${K8S_VER}"'\"/{f=1;next} f && /'"${name}"':/{gsub("\"","",$2);print $2;exit}' "${ROOT_DIR}"/env/offline-manifest.yaml)
+  file="${name}-${ver}.tgz"
+  url="${REPO_URL}/${file}"
+  if curl -sSfL "${url}" -o "${bin_path}/${file}" -m "360"; then
+    utils::log "INFO" "Downloaded ${url}"
+  else
+    utils::log "ERROR" "fail to download ${url}"
+  fi
+
+  _offline_containerd
+}
+
 _offline_containerd() {
   local bin_path tar_name
-  bin_path=${ROOT_DIR}/version-${VERSION}/bin-tools/
+  bin_path=${ROOT_DIR}/version-${K8S_VER}/bin-tools/
 
   tar_name=$(find "$bin_path" -iname "containerd-*.tgz" -type f | head -1)
   if [[ -z ${tar_name} ]]; then
@@ -165,9 +193,19 @@ main() {
     utils::log "WARN" "containerd installed, $(ctr -v)"
   else
     if [[ -n ${BCS_OFFLINE:-} ]]; then
-	  _offline_containerd
+      _offline_containerd
     else
-      _yum_containerd
+      case ${INSTALL_METHOD} in
+        "yum")
+          _yum_containerd
+          ;;
+        "curl")
+          _curl_containerd
+          ;;
+        *)
+          utils::log "ERROR" "unkown ${INSTALL_METHOD} to exec download containerd"
+          ;;
+      esac
     fi
   fi
 
@@ -181,6 +219,11 @@ main() {
     utils::log "ERROR" "Did containerd get installed?"
   fi
 
+  # add insecure_registry
+  if [[ -n ${INSECURE_REGISTRY:-} ]]; then
+    "${ROOT_DIR}"/k8s/insecure_registry.sh -c containerd -a "${INSECURE_REGISTRY}"
+  fi
+
   if [[ -n ${BCS_OFFLINE:-} ]]; then
     find "${ROOT_DIR}"/version-"${VERSION}"/images -name '*.tar' -type f -print0 \
       | xargs -0 -I {} ctr -n k8s.io image import {}
@@ -191,7 +234,6 @@ main() {
   test_img_url=${BK_PUBLIC_REPO:-"docker.io"}/library/hello-world:latest
   utils::log "DEBUG" "hello-world: ${test_img_url}"
 
-
   if ! (ctr -n k8s.io i pull --hosts-dir "/etc/containerd/certs.d" "$test_img_url" \
     && ctr -n k8s.io run --rm "$test_img_url" hello-world."$(date +%s)"); then
     utils::log "ERROR" "Could not get containerd to run ${test_img_url}"

diff --git a/bcs-ops/k8s/install_docker b/bcs-ops/k8s/install_docker
@@ -74,9 +74,27 @@ _yum_docker() {
   return 0
 }
 
+_curl_docker() {
+  local bin_path name ver file url
+  bin_path=${ROOT_DIR}/version-${K8S_VER}/bin-tools/
+  mkdir -p "$bin_path"
+
+  name="docker"
+  ver=$(awk '/version: \"'"${K8S_VER}"'\"/{f=1;next} f && /'"${name}"':/{gsub("\"","",$2);print $2;exit}' "${ROOT_DIR}"/env/offline-manifest.yaml)
+  file="${name}-${ver}.tgz"
+  url=${REPO_URL}/${file}
+  if curl -sSfL "${url}" -o "${bin_path}/${file}" -m "360"; then
+    utils::log "INFO" "Downloaded ${url}"
+  else
+    utils::log "ERROR" "fail to download ${url}"
+  fi
+
+  _offline_docker
+}
+
 _offline_docker() {
   local bin_path tar_name
-  bin_path=${ROOT_DIR}/version-${VERSION}/bin-tools/
+  bin_path=${ROOT_DIR}/version-${K8S_VER}/bin-tools/
   tar_name=$(find "$bin_path" -iname "docker-*.tgz" -type f | head -1)
 
   if [[ -z ${tar_name} ]]; then
@@ -85,6 +103,14 @@ _offline_docker() {
     tar xvzf "${tar_name}" -C /usr/bin/ --strip-components=1 bin/
     tar xvzf "${tar_name}" -C /etc/systemd/system/ --strip-components=1 systemd/
   fi
+
+  # docker.sock need docker group
+  if getent group docker >/dev/null 2>&1; then
+    utils::log "INFO" "docker group existed"
+  else
+    utils::log "INFO" "creating docker group"
+    groupadd docker
+  fi
 }
 
 # ToDo: config separte
@@ -150,7 +176,17 @@ main() {
     if [[ -n ${BCS_OFFLINE:-} ]]; then
       _offline_docker
     else
-      _yum_docker
+      case ${INSTALL_METHOD} in
+        "yum")
+          _yum_docker
+          ;;
+        "curl")
+          _curl_docker
+          ;;
+        *)
+          utils::log "ERROR" "unkown ${INSTALL_METHOD} to exec download docker"
+          ;;
+      esac
     fi
   fi
 
@@ -164,7 +200,12 @@ main() {
     utils::log "ERROR" "Did docker get installed?"
   fi
 
-  # load image
+  # add insecure_registry
+  if [[ -n ${INSECURE_REGISTRY:-} ]]; then
+    "${ROOT_DIR}"/k8s/insecure_registry.sh -c docker -a "${INSECURE_REGISTRY}"
+  fi
+
+  # bcs_offline load image
   if [[ -n ${BCS_OFFLINE:-} ]]; then
     find "${ROOT_DIR}"/version-"${VERSION}"/images -name '*.tar' -type f -print0 \
       | xargs -0 -I {} docker load -i {}
@@ -175,11 +216,6 @@ main() {
   test_img_url=${BK_PUBLIC_REPO:-"docker.io"}/library/hello-world:latest
   utils::log "DEBUG" "hello-world: ${test_img_url}"
 
-  if [[ -n ${BCS_OFFLINE:-} ]]; then
-    # ToDo hello world image offline install
-    true
-  fi
-
   if ! docker run --rm "${test_img_url}"; then
     utils::log "ERROR" "Count not get docker to run ${test_img_url}"
   fi