Merge branch 'master' into docker-updates-crate-2023-09-14-123939

GoogleCloudPlatform · Sep 19, 2023 · ae67375 · ae67375
2 parents ef53173 + d656920
commit ae67375
Show file tree

Hide file tree

Showing 8 changed files with 751 additions and 4 deletions.
diff --git a/docker/airflow/2/debian11/2.7/Dockerfile b/docker/airflow/2/debian11/2.7/Dockerfile
@@ -0,0 +1,265 @@
+ARG AIRFLOW_VERSION="2.7.1"
+ARG AIRFLOW_EXTRAS="amazon,async,celery,cncf.kubernetes,dask,docker,elasticsearch,ftp,google,google_auth,grpc,hashicorp,http,ldap,microsoft.azure,mysql,odbc,pandas,postgres,redis,sendgrid,sftp,slack,ssh,statsd,virtualenv"
+
+ARG AIRFLOW_HOME=/opt/airflow
+ARG AIRFLOW_UID="50000"
+ARG AIRFLOW_USER_HOME_DIR=/home/airflow
+ARG AIRFLOW_PIP_VERSION="22.3.1"
+
+FROM marketplace.gcr.io/google/c2d-debian11 as airflow-build-image
+
+SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "nounset", "-o", "nolog", "-c"]
+ENV DEBIAN_FRONTEND=noninteractive LANGUAGE=C.UTF-8 LANG=C.UTF-8 LC_ALL=C.UTF-8 \
+    LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8
+
+ENV DEV_APT_DEPS="\
+     apt-transport-https \
+     apt-utils \
+     build-essential \
+     ca-certificates \
+     dirmngr \
+     freetds-bin \
+     freetds-dev \
+     gosu \
+     krb5-user \
+     ldap-utils \
+     libffi-dev \
+     libkrb5-dev \
+     libldap2-dev \
+     libpq-dev \
+     libsasl2-2 \
+     libsasl2-dev \
+     libsasl2-modules \
+     libssl-dev \
+     locales  \
+     lsb-release \
+     nodejs \
+     openssh-client \
+     pkg-config \
+     postgresql-client \
+     python3 \
+     python3-pip \
+     python3-dev \
+     sasl2-bin \
+     software-properties-common \
+     sqlite3 \
+     sudo \
+     unixodbc \
+     unixodbc-dev \
+     yarn"
+
+ENV DEV_APT_COMMAND="\
+    curl --silent --fail --location https://deb.nodesource.com/setup_14.x | \
+    bash -o pipefail -o errexit -o nolog - \
+    && curl --silent https://dl.yarnpkg.com/debian/pubkey.gpg | \
+    apt-key add - >/dev/null 2>&1\
+    && echo 'deb https://dl.yarnpkg.com/debian/ stable main' > /etc/apt/sources.list.d/yarn.list"
+
+RUN apt-get update \
+    && apt-get install --no-install-recommends -yqq apt-utils >/dev/null 2>&1 \
+    && apt-get install -y --no-install-recommends curl gnupg2 \
+    && mkdir -pv /usr/share/man/man1 \
+    && mkdir -pv /usr/share/man/man7 \
+    && bash -o pipefail -o errexit -o nounset -o nolog -c "${DEV_APT_COMMAND}" \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends ${DEV_APT_DEPS} \
+    && apt-get autoremove -yqq --purge \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+ARG AIRFLOW_EXTRAS
+ARG AIRFLOW_VERSION
+ARG AIRFLOW_PIP_VERSION
+ARG AIRFLOW_HOME
+ARG AIRFLOW_USER_HOME_DIR
+ARG AIRFLOW_UID
+
+ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \
+    AIRFLOW_VERSION=${AIRFLOW_VERSION} \
+    AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS} \
+    AIRFLOW_PRE_CACHED_PIP_PACKAGES="false" \
+    INSTALL_PROVIDERS_FROM_SOURCES="false" \
+    AIRFLOW_INSTALLATION_METHOD="apache-airflow" \
+    PATH=${PATH}:${AIRFLOW_USER_HOME_DIR}/.local/bin \
+    AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \
+    AIRFLOW_USER_HOME_DIR=${AIRFLOW_USER_HOME_DIR} \
+    AIRFLOW_HOME=${AIRFLOW_HOME} \
+    AIRFLOW_UID=${AIRFLOW_UID} \
+    INSTALL_MYSQL_CLIENT="true" \
+    INSTALL_MSSQL_CLIENT="true" \
+    PIP_USER="true"
+
+COPY scripts/docker/install_mysql.sh scripts/docker/install_mssql.sh /scripts/docker/
+
+RUN /scripts/docker/install_mysql.sh dev && /scripts/docker/install_mssql.sh
+ENV PATH=${PATH}:/opt/mssql-tools/bin
+
+RUN adduser --gecos "First Last,RoomNumber,WorkPhone,HomePhone" --disabled-password \
+       --quiet "airflow" --uid "${AIRFLOW_UID}" --gid "0" --home "${AIRFLOW_USER_HOME_DIR}" && \
+    mkdir -p ${AIRFLOW_HOME} && chown -R "airflow:0" "${AIRFLOW_USER_HOME_DIR}" ${AIRFLOW_HOME}
+
+USER airflow
+
+COPY --chown=airflow:0 scripts/docker/install_pip_version.sh /scripts/docker/
+
+RUN /scripts/docker/install_pip_version.sh
+
+ENV INSTALL_FROM_PYPI="true" \
+    EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS="dill<0.3.3 certifi<2021.0.0 google-ads<14.0.1"
+
+WORKDIR /opt/airflow
+
+COPY --chown=airflow:0 scripts/docker/install_airflow.sh /scripts/docker/
+
+RUN /scripts/docker/install_airflow.sh \
+    && find "${AIRFLOW_USER_HOME_DIR}/.local/" -name '*.pyc' -print0 | xargs -0 rm -f || true \
+    && find "${AIRFLOW_USER_HOME_DIR}/.local/" -type d -name '__pycache__' -print0 | xargs -0 rm -rf || true \
+    && find "${AIRFLOW_USER_HOME_DIR}/.local" -executable -print0 | xargs --null chmod g+x \
+    && find "${AIRFLOW_USER_HOME_DIR}/.local" -print0 | xargs --null chmod g+rw
+
+FROM marketplace.gcr.io/google/c2d-debian11 as main
+
+SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "nounset", "-o", "nolog", "-c"]
+
+ARG AIRFLOW_USER_HOME_DIR
+ARG AIRFLOW_HOME
+ARG AIRFLOW_UID
+ARG AIRFLOW_PIP_VERSION
+ARG AIRFLOW_VERSION
+
+ENV AIRFLOW_VERSION=${AIRFLOW_VERSION} \
+    DEBIAN_FRONTEND=noninteractive LANGUAGE=C.UTF-8 LANG=C.UTF-8 LC_ALL=C.UTF-8 \
+    LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8 \
+    AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION}
+
+ENV RUNTIME_APT_DEPS="\
+       apt-transport-https \
+       apt-utils \
+       ca-certificates \
+       curl \
+       dumb-init \
+       freetds-bin \
+       krb5-user \
+       ldap-utils \
+       libffi7 \
+       libldap-2.4-2 \
+       libsasl2-2 \
+       libsasl2-modules \
+       libssl1.1 \
+       locales  \
+       lsb-release \
+       netcat \
+       openssh-client \
+       pkg-config \
+       postgresql-client \
+       python2 \
+       python3 \
+       python3-pip \
+       rsync \
+       sasl2-bin \
+       sqlite3 \
+       sudo \
+       unixodbc"
+
+ENV RUNTIME_APT_COMMAND="echo" \
+    INSTALL_MYSQL_CLIENT="true" \
+    INSTALL_MSSQL_CLIENT="true" \
+    AIRFLOW_INSTALLATION_METHOD="apache-airflow" \
+    AIRFLOW_UID=${AIRFLOW_UID} \
+    AIRFLOW__CORE__LOAD_EXAMPLES="false" \
+    AIRFLOW_USER_HOME_DIR=${AIRFLOW_USER_HOME_DIR} \
+    AIRFLOW_HOME=${AIRFLOW_HOME} \
+    PATH="${AIRFLOW_USER_HOME_DIR}/.local/bin:${PATH}" \
+    GUNICORN_CMD_ARGS="--worker-tmp-dir /dev/shm" \
+    PIP_USER="true"
+
+RUN apt-get update \
+    && apt-get install --no-install-recommends -yqq apt-utils >/dev/null 2>&1 \
+    && apt-get install -y --no-install-recommends curl gnupg2 \
+    && mkdir -pv /usr/share/man/man1 \
+    && mkdir -pv /usr/share/man/man7 \
+    && bash -o pipefail -o errexit -o nounset -o nolog -c "${RUNTIME_APT_COMMAND}" \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends ${RUNTIME_APT_DEPS} \
+    && apt-get autoremove -yqq --purge \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* \
+    && rm -rf /var/log/* \
+    && ln -s /usr/bin/python3 /usr/bin/python
+
+# Install Gosu
+# /usr/sbin/gosu
+ENV GOSU_VERSION 1.16
+RUN set -eux; \
+	apt-get update; \
+	DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+		ca-certificates \
+		gpg \
+		gpgv \
+		libjemalloc2 \
+		pwgen \
+		tzdata \
+		xz-utils \
+		zstd ; \
+	savedAptMark="$(apt-mark showmanual)"; \
+	apt-get install -y --no-install-recommends \
+		dirmngr \
+		gpg-agent \
+		wget; \
+	rm -rf /var/lib/apt/lists/*; \
+	dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')"; \
+	wget -q -O /usr/sbin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch"; \
+	wget -q -O /usr/sbin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch.asc"; \
+	GNUPGHOME="$(mktemp -d)"; \
+	export GNUPGHOME; \
+	gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4; \
+	gpg --batch --verify /usr/sbin/gosu.asc /usr/sbin/gosu; \
+	gpgconf --kill all; \
+	rm -rf "$GNUPGHOME" /usr/sbin/gosu.asc; \
+	apt-mark auto '.*' > /dev/null; \
+	[ -z "$savedAptMark" ] ||	apt-mark manual $savedAptMark >/dev/null; \
+	apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \
+	chmod +x /usr/sbin/gosu; \
+	gosu --version; \
+	gosu nobody true
+
+
+COPY scripts/docker/install_mysql.sh /scripts/docker/install_mssql.sh /scripts/docker/
+RUN chmod a+x /scripts/docker/install_mysql.sh /scripts/docker/install_mssql.sh \
+    && sync \
+    && /scripts/docker/install_mysql.sh prod \
+    && /scripts/docker/install_mssql.sh \
+    && adduser --gecos "First Last,RoomNumber,WorkPhone,HomePhone" --disabled-password \
+       --quiet "airflow" --uid "${AIRFLOW_UID}" --gid "0" --home "${AIRFLOW_USER_HOME_DIR}" \
+    && mkdir -pv "${AIRFLOW_HOME}" \
+    && mkdir -pv "${AIRFLOW_HOME}/dags" \
+    && mkdir -pv "${AIRFLOW_HOME}/logs" \
+    && chown -R airflow:0 "${AIRFLOW_USER_HOME_DIR}" "${AIRFLOW_HOME}" \
+    && chmod -R g+rw "${AIRFLOW_USER_HOME_DIR}" "${AIRFLOW_HOME}" \
+    && find "${AIRFLOW_HOME}" -executable -print0 | xargs --null chmod g+x \
+    && find "${AIRFLOW_USER_HOME_DIR}" -executable -print0 | xargs --null chmod g+x
+
+COPY --chown=airflow:0 --from=airflow-build-image \
+     "${AIRFLOW_USER_HOME_DIR}/.local" "${AIRFLOW_USER_HOME_DIR}/.local"
+COPY --chown=airflow:0 scripts/prod/entrypoint_prod.sh /entrypoint
+COPY --chown=airflow:0 scripts/prod/clean-logs.sh /clean-logs
+
+RUN chmod a+x /entrypoint /clean-logs \
+    && chmod g=u /etc/passwd \
+    && chmod g+w "${AIRFLOW_USER_HOME_DIR}/.local" \
+    && usermod -g 0 airflow -G 0
+
+RUN sed --in-place=.bak "s/secure_path=\"/secure_path=\"\/.venv\/bin:/" /etc/sudoers
+
+ENV DUMB_INIT_SETSID="1" \
+    PS1="(airflow)" \
+    LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libstdc++.so.6"
+
+WORKDIR ${AIRFLOW_HOME}
+EXPOSE 8080
+USER ${AIRFLOW_UID}
+
+ENV C2D_RELEASE 2.7.1
+
+ENTRYPOINT ["/usr/bin/dumb-init", "--", "/entrypoint"]
+CMD [""]
diff --git a/docker/airflow/2/debian11/2.7/scripts/docker/install_airflow.sh b/docker/airflow/2/debian11/2.7/scripts/docker/install_airflow.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+: "${AIRFLOW_PIP_VERSION:?Should be set}"
+
+function install_airflow() {
+    # Remove mysql from extras if client is not going to be installed
+    if [[ ${INSTALL_MYSQL_CLIENT} != "true" ]]; then
+        AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS/mysql,}
+    fi
+    echo
+    echo "Installing all packages and upgrade if needed"
+    echo
+    pip install --upgrade --upgrade-strategy only-if-needed "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]==${AIRFLOW_VERSION}" airflow-exporter 
+}
+
+install_airflow
diff --git a/docker/airflow/2/debian11/2.7/scripts/docker/install_mssql.sh b/docker/airflow/2/debian11/2.7/scripts/docker/install_mssql.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+: "${INSTALL_MSSQL_CLIENT:?Should be true or false}"
+
+function install_mssql_client() {
+    echo
+    echo "Installing mssql client"
+    echo
+    curl --silent https://packages.microsoft.com/keys/microsoft.asc | apt-key add - >/dev/null 2>&1
+    curl --silent https://packages.microsoft.com/config/debian/10/prod.list > /etc/apt/sources.list.d/mssql-release.list
+    apt-get update -yqq
+    apt-get upgrade -yqq
+    ACCEPT_EULA=Y apt-get -yqq install -y --no-install-recommends msodbcsql17 mssql-tools
+    rm -rf /var/lib/apt/lists/*
+    apt-get autoremove -yqq --purge
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+}
+
+# Install MsSQL client from Microsoft repositories
+if [[ ${INSTALL_MSSQL_CLIENT:="true"} == "true" ]]; then
+    install_mssql_client "${@}"
+fi
diff --git a/docker/airflow/2/debian11/2.7/scripts/docker/install_mysql.sh b/docker/airflow/2/debian11/2.7/scripts/docker/install_mysql.sh
@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+declare -a packages
+
+MYSQL_VERSION="8.0"
+readonly MYSQL_VERSION
+
+: "${INSTALL_MYSQL_CLIENT:?Should be true or false}"
+
+install_mysql_client() {
+    echo
+    echo "Installing mysql client version ${MYSQL_VERSION}"
+    echo
+
+    if [[ "${1}" == "dev" ]]; then
+        packages=("libmysqlclient-dev" "mysql-client")
+    elif [[ "${1}" == "prod" ]]; then
+        packages=("libmysqlclient21" "mysql-client")
+    else
+        echo
+        echo "Specify either prod or dev"
+        echo
+        exit 1
+    fi
+
+    local key="467B942D3A79BD29"
+    readonly key
+
+    GNUPGHOME="$(mktemp -d)"
+    export GNUPGHOME
+    set +e
+    for keyserver in $(shuf -e ha.pool.sks-keyservers.net hkp://p80.pool.sks-keyservers.net:80 \
+                               keyserver.ubuntu.com hkp://keyserver.ubuntu.com:80)
+    do
+        gpg --keyserver "${keyserver}" --recv-keys "${key}" 2>&1 && break
+    done
+    set -e
+    gpg --export "${key}" > /etc/apt/trusted.gpg.d/mysql.gpg
+    gpgconf --kill all
+    rm -rf "${GNUPGHOME}"
+    unset GNUPGHOME
+    echo "deb http://repo.mysql.com/apt/debian/ buster mysql-${MYSQL_VERSION}" | tee -a /etc/apt/sources.list.d/mysql.list
+    apt-get update
+    apt-get install --no-install-recommends -y "${packages[@]}"
+    apt-get autoremove -yqq --purge
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+}
+
+# Install MySQL client from Oracle repositories (Debian installs mariadb)
+# But only if it is not disabled
+if [[ ${INSTALL_MYSQL_CLIENT:="true"} == "true" ]]; then
+    install_mysql_client "${@}"
+fi
diff --git a/docker/airflow/2/debian11/2.7/scripts/docker/install_pip_version.sh b/docker/airflow/2/debian11/2.7/scripts/docker/install_pip_version.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+: "${AIRFLOW_PIP_VERSION:?Should be set}"
+
+function install_pip_version() {
+    echo
+    echo "Installing pip version ${AIRFLOW_PIP_VERSION}"
+    echo
+    pip install --disable-pip-version-check --no-cache-dir --upgrade "pip==${AIRFLOW_PIP_VERSION}" &&
+        mkdir -p ${HOME}/.local/bin
+}
+
+install_pip_version