diff --git a/.github/jobs/build_docker_image.sh b/.github/jobs/build_docker_image.sh index 79214e9..dd973a5 100755 --- a/.github/jobs/build_docker_image.sh +++ b/.github/jobs/build_docker_image.sh @@ -5,7 +5,7 @@ MET_BASE_TAG=${GITHUB_TAG} DOCKERHUB_TAG_BASE=${DOCKERHUB_BASE_REPO}:${MET_BASE_TAG} -DOCKERFILE_PATH=${GITHUB_WORKSPACE}/Dockerfile.debian +DOCKERFILE_PATH=${GITHUB_WORKSPACE}/Dockerfile CMD_LOGFILE=${GITHUB_WORKSPACE}/docker_build_base_image.log @@ -18,7 +18,7 @@ fi DOCKERHUB_TAG_UNIT_TEST=${DOCKERHUB_UNIT_TEST_REPO}:${GITHUB_TAG} -DOCKERFILE_PATH=${GITHUB_WORKSPACE}/Dockerfile.debian_unit_test_env +DOCKERFILE_PATH=${GITHUB_WORKSPACE}/Dockerfile.unit_test_env CMD_LOGFILE=${GITHUB_WORKSPACE}/docker_build_unit_test_env.log diff --git a/.github/workflows/build_and_push_docker.yml b/.github/workflows/build_and_push_docker.yml index b02ca58..69fcfbf 100644 --- a/.github/workflows/build_and_push_docker.yml +++ b/.github/workflows/build_and_push_docker.yml @@ -15,21 +15,23 @@ jobs: name: Handle Docker Images runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Get tag name id: get_tag_name - run: echo ::set-output name=tag_name::${GITHUB_REF#"refs/tags/"} + uses: devops-actions/action-get-tag@v1.0.2 + with: + strip_v: false - name: Build Docker Image run: .github/jobs/build_docker_image.sh env: - GITHUB_TAG: ${{ steps.get_tag_name.outputs.tag_name }} + GITHUB_TAG: ${{ steps.get_tag_name.outputs.tag }} - name: Push Docker Image run: .github/jobs/push_docker_image.sh env: - GITHUB_TAG: ${{ steps.get_tag_name.outputs.tag_name }} + GITHUB_TAG: ${{ steps.get_tag_name.outputs.tag }} DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} diff --git a/Dockerfile b/Dockerfile index 469f16d..346d116 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,168 +1,81 @@ -FROM centos:7 -MAINTAINER John Halley Gotway +ARG DEBIAN_VERSION=12 +FROM debian:${DEBIAN_VERSION}-slim +MAINTAINER George McCabe # -# Define the compilers. +# Define the compilers # ENV CC /usr/bin/gcc ENV CXX /usr/bin/g++ ENV FC /usr/bin/gfortran ENV F77 /usr/bin/gfortran -# -# Define library versions. -# Match WCOSS2 versions as of October, 2022. -# -ENV HDF5_VER 1_10_6 -ENV NETCDF4C_VER 4.7.4 -ENV NETCDF4CXX_VER 4.3.1 - -# -# Define package URL's. -# -ENV HDF4_URL http://www.hdfgroup.org/ftp/HDF/releases/HDF4.2r3/src/HDF4.2r3.tar.gz -ENV HDFEOS_URL https://dtcenter.ucar.edu/dfiles/code/METplus/MET/docker_data/HDF-EOS2.16v1.00.tar.Z - -ENV HDF5_URL https://github.com/HDFGroup/hdf5/archive/refs/tags/hdf5-${HDF5_VER}.zip -ENV NETCDF4C_URL https://github.com/Unidata/netcdf-c/archive/refs/tags/v${NETCDF4C_VER}.zip -ENV NETCDF4CXX_URL https://github.com/Unidata/netcdf-cxx4/archive/v${NETCDF4CXX_VER}.tar.gz +ENV PYTHON_VER 3.10.4 -ENV BUFRLIB_URL https://dtcenter.ucar.edu/dfiles/code/METplus/MET/docker_data/BUFRLIB_v10-2-3.tar ENV GSFONT_URL https://dtcenter.ucar.edu/dfiles/code/METplus/MET/docker_data/ghostscript-fonts-std-8.11.tar.gz # -# Install the required packages. -# -RUN yum -y update \ - && yum -y install file gcc gcc-gfortran gcc-c++ glibc.i686 libgcc.i686 \ - libpng-devel jasper jasper-devel zlib zlib-devel \ - cairo-devel freetype-devel epel-release \ - hostname m4 make tar tcsh ksh time wget which \ - flex flex-devel bison bison-devel unzip \ - && yum -y install git g2clib-devel gsl-devel \ - && yum -y install sqlite-devel libffi-devel libcurl-devel.x86_64 \ - && yum -y install gv ncview wgrib wgrib2 ImageMagick ps2pdf \ - && yum -y install python3 python3-devel python3-pip - -# Needed to prevent error in subsequent pip command -ENV LD_LIBRARY_PATH /usr/local/lib - -RUN pip3 install --upgrade pip \ - && python3 -m pip install numpy xarray netCDF4 - -# -# Set the working directory. -# -WORKDIR /met - -# -# Setup the environment for interactive bash/csh container shells. +# Set up the environment for interactive bash shell +# Set soft limit to unlimited to prevent GRIB2 seg faults # -RUN echo export MET_BASE=/usr/local/share/met >> /etc/bashrc \ - && echo setenv MET_BASE /usr/local/share/met >> /etc/csh.cshrc \ - && echo export MET_FONT_DIR=/usr/local/share/met/fonts >> /etc/bashrc \ - && echo setenv MET_FONT_DIR /usr/local/share/met/fonts >> /etc/csh.cshrc \ - && echo export RSCRIPTS_BASE=/usr/local/share/met/Rscripts >> /etc/bashrc \ - && echo setenv RSCRIPTS_BASE /usr/local/share/met/Rscripts >> /etc/csh.cshrc +RUN echo export MET_BASE=/usr/local/share/met >> /root/.bashrc \ + && echo export MET_FONT_DIR=/usr/local/share/met/fonts >> /root/.bashrc \ + && echo export RSCRIPTS_BASE=/usr/local/share/met/Rscripts >> /root/.bashrc \ + && echo ulimit -S -s unlimited >> /root/.bashrc ENV MET_FONT_DIR /usr/local/share/met/fonts # -# Download and install BUFRLIB. +# Install required system tools # -RUN mkdir -p /met/logs \ - && mkdir -p /met/external_libs/BUFRLIB \ - && cd /met/external_libs/BUFRLIB \ - && echo "Downloading BUFRLIB from ${BUFRLIB_URL}" \ - && curl -SL ${BUFRLIB_URL} | tar xC /met/external_libs/BUFRLIB \ - && cat preproc.sh | sed 's/cpp /cpp -traditional-cpp /g' > preproc_patch.sh \ - && chmod +x preproc_patch.sh \ - && LOG_FILE=/met/logs/BUFRLIB_build.log \ - && echo "Compiling BUFRLIB and writing log file ${LOG_FILE}" \ - && ./preproc_patch.sh *.F > ${LOG_FILE} \ - && ${CC} -c -DUNDERSCORE *.c >> ${LOG_FILE} \ - && ${FC} -c -fno-second-underscore *.f >> ${LOG_FILE} \ - && ar crv libbufr.a *.o >> ${LOG_FILE} \ - && rm -f /usr/local/lib/libbufr.a \ - && cp *.a /usr/local/lib \ - && cd /met/external_libs \ - && rm -rf BUFRLIB +RUN apt update && apt -y upgrade \ + && apt install -y build-essential gfortran wget unzip curl imagemagick \ + libcurl4-gnutls-dev m4 git automake flex bison libjpeg-dev libpixman-1-dev \ + emacs vim less \ + libreadline-dev libncursesw5-dev libssl-dev tk-dev \ + libgdbm-dev libc6-dev libbz2-dev libffi-dev zlib1g-dev \ + cmake libtiff-dev sqlite3 libsqlite3-dev -# -# Download and install HDF5. -# -RUN mkdir -p /met/external_libs/hdf5 \ - && cd /met/external_libs/hdf5 \ - && echo "Downloading HDF5 from ${HDF5_URL}" \ - && wget ${HDF5_URL} \ - && unzip hdf5-${HDF5_VER}.zip \ - && cd hdf5-hdf5-${HDF5_VER} \ - && LOG_FILE=/met/logs/hdf5-hdf5-${HDF5_VER}_configure.log \ - && echo "Configuring hdf5-hdf5-${HDF5_VER} and writing log file ${LOG_FILE}" \ - && ./configure --prefix=/usr/local --enable-cxx --with-default-api-version=v18 > ${LOG_FILE} \ - && LOG_FILE=/met/logs/hdf5-hdf5-${HDF5_VER}_make_install.log \ - && echo "Compiling hdf5-hdf5-${HDF5_VER} and writing log file ${LOG_FILE}" \ - && make install > ${LOG_FILE} \ - && cd /met/external_libs \ - && rm -rf hdf5 +RUN echo "Downloading GhostScript fonts from ${GSFONT_URL} into /usr/local/share/met" \ + && mkdir -p /usr/local/share/met \ + && curl -SL ${GSFONT_URL} | tar zxC /usr/local/share/met # -# Download and install NetCDF4 (C and C++). +# Fix rules for ghostscript files in convert +# See: https://en.linuxportal.info/tutorials/troubleshooting/how-to-fix-errors-from-imagemagick-imagick-conversion-system-security-policy # -RUN mkdir -p /met/external_libs/netcdf \ - && cd /met/external_libs/netcdf \ - && echo "Downloading netcdf-c-${NETCDF4C_VER} from ${NETCDF4C_URL}" \ - && wget ${NETCDF4C_URL} \ - && unzip v${NETCDF4C_VER}.zip \ - && cd netcdf-c-${NETCDF4C_VER} \ - && LOG_FILE=/met/logs/netcdf-c-${NETCDF4C_VER}_configure.log \ - && echo "Configuring netcdf-c-${NETCDF4C_VER} and writing log file ${LOG_FILE}" \ - && ./configure --enable-shared --enable-netcdf-4 > ${LOG_FILE} \ - && LOG_FILE=/met/logs/netcdf-c-${NETCDF4C_VER}_make_install.log \ - && echo "Compiling netcdf-c-${NETCDF4C_VER} and writing log file ${LOG_FILE}" \ - && make install > ${LOG_FILE} \ - && echo "Downloading from ${NETCDF4CXX_URL}" \ - && cd /met/external_libs/netcdf \ - && wget ${NETCDF4CXX_URL} \ - && tar -xzf v${NETCDF4CXX_VER}.tar.gz \ - && cd netcdf-cxx4-${NETCDF4CXX_VER} \ - && LOG_FILE=/met/logs/netcdf-cxx4-${NETCDF4CXX_VER}_configure.log \ - && echo "Configuring netcdf-cxx4-${NETCDF4CXX_VER} and writing log file ${LOG_FILE}" \ - && ./configure > ${LOG_FILE} \ - && LOG_FILE=/met/logs/netcdf-cxx4-${NETCDF4CXX_VER}_make_install.log \ - && echo "Compiling netcdf-cxx4-${NETCDF4CXX_VER} and writing log file ${LOG_FILE}" \ - && make install > ${LOG_FILE} \ - && cd /met/external_libs \ - && rm -rf netcdf +RUN sed -i 's/policy domain="coder" rights="none" pattern="PS/policy domain="coder" rights="read | write" pattern="PS/g' /etc/ImageMagick-6/policy.xml \ + && sed -i 's/policy domain="coder" rights="none" pattern="EPS"/policy domain="coder" rights="read | write" pattern="EPS"/g' /etc/ImageMagick-6/policy.xml \ + && sed -i 's/policy domain="coder" rights="none" pattern="PDF"/policy domain="coder" rights="read | write" pattern="PDF"/g' /etc/ImageMagick-6/policy.xml \ + && sed -i 's/policy domain="coder" rights="none" pattern="XPS"/policy domain="coder" rights="read | write" pattern="XPS"/g' /etc/ImageMagick-6/policy.xml # -# Download and install HDF4 and HDFEOS. +# Set the working directory # -RUN echo "Downloading HDF4.2r3 from ${HDF4_URL}" \ - && curl -SL ${HDF4_URL} | tar zxC /met/external_libs \ - && cd /met/external_libs/HDF4.2r3 \ - && LOG_FILE=/met/logs/HDF4.2r3_configure.log \ - && echo "Configuring HDF4.2r3 and writing log file ${LOG_FILE}" \ - && ./configure --prefix=/usr/local/hdf --disable-netcdf > ${LOG_FILE} \ - && cat mfhdf/hdiff/Makefile | sed 's/LIBS = -ljpeg -lz/LIBS = -ljpeg -lz -lm/g' > Makefile_NEW \ - && mv -f Makefile_NEW mfhdf/hdiff/Makefile \ - && LOG_FILE=/met/logs/HDF4.2r3_make_install.log \ - && echo "Compiling HDF4.2r3 and writing log file ${LOG_FILE}" \ - && make install > ${LOG_FILE} \ - && echo "Downloading hdfeos from ${HDFEOS_URL}" \ - && curl -SL ${HDFEOS_URL} | tar zxC /met/external_libs \ - && cd /met/external_libs/hdfeos \ - && LOG_FILE=/met/logs/hdfeos_configure.log \ - && echo "Configuring hdfeos and writing log file ${LOG_FILE}" \ - && ./configure --prefix=/usr/local/hdfeos --with-hdf4=/usr/local/hdf CC=/usr/local/hdf/bin/h4cc > ${LOG_FILE} \ - && LOG_FILE=/met/logs/hdfeos_make_install.log \ - && echo "Compiling hdfeos and writing log file ${LOG_FILE}" \ - && make install > ${LOG_FILE} \ - && mkdir /usr/local/hdfeos/include \ - && cp include/*.h /usr/local/hdfeos/include/. \ - && cd /met/external_libs \ - && rm -rf HDF4.2r3 hdfeos +WORKDIR /met -RUN echo "Downloading GhostScript fonts from ${GSFONT_URL} into /usr/local/share/met" \ - && mkdir -p /usr/local/share/met \ - && curl -SL ${GSFONT_URL} | tar zxC /usr/local/share/met +RUN wget https://dtcenter.ucar.edu/dfiles/code/METplus/MET/installation/tar_files.tgz \ + && wget https://raw.githubusercontent.com/dtcenter/MET/develop/internal/scripts/installation/compile_MET_all.sh \ + && wget https://raw.githubusercontent.com/dtcenter/MET/develop/internal/scripts/environment/development.docker \ + && tar -zxf tar_files.tgz \ + && export SKIP_MET=TRUE \ + && chmod +x compile_MET_all.sh \ + && ./compile_MET_all.sh development.docker + +# +# Install Python from source +# +RUN wget https://www.python.org/ftp/python/${PYTHON_VER}/Python-${PYTHON_VER}.tgz \ + && tar xzf Python-${PYTHON_VER}.tgz \ + && cd Python-${PYTHON_VER} \ + && ./configure --enable-optimizations --enable-shared LDFLAGS="-L/usr/local/lib -Wl,-rpath,/usr/local/lib" \ + && make -j `nproc` \ + && make install \ + && ln -s /usr/local/bin/python3 /usr/local/bin/python \ + && BLDOPTS="--force-reinstall --global-option=build_ext --global-option=\"-R/usr/local/lib\" --global-option=\"-L/usr/local/lib\"" \ + && python3 -m pip install --upgrade pip \ + && python3 -m pip install ${BLDOPTS} numpy \ + && python3 -m pip install ${BLDOPTS} xarray \ + && export HDF5_DIR=/usr/local/ \ + && export NETCDF4_DIR=/usr/local/ \ + && python3 -m pip install ${BLDOPTS} netCDF4 diff --git a/Dockerfile.debian b/Dockerfile.debian deleted file mode 100644 index eb5f6c0..0000000 --- a/Dockerfile.debian +++ /dev/null @@ -1,72 +0,0 @@ -ARG DEBIAN_VERSION=10 -FROM debian:${DEBIAN_VERSION}-slim -MAINTAINER George McCabe - -# -# Define the compilers. -# -ENV CC /usr/bin/gcc -ENV CXX /usr/bin/g++ -ENV FC /usr/bin/gfortran -ENV F77 /usr/bin/gfortran - -ENV PYTHON_VER 3.10.4 - -ENV GSFONT_URL https://dtcenter.ucar.edu/dfiles/code/METplus/MET/docker_data/ghostscript-fonts-std-8.11.tar.gz - -# -# Setup the environment for interactive bash shell. -# Set soft limit to unlimited to prevent GRIB2 seg faults -# -RUN echo export MET_BASE=/usr/local/share/met >> /root/.bashrc \ - && echo export MET_FONT_DIR=/usr/local/share/met/fonts >> /root/.bashrc \ - && echo export RSCRIPTS_BASE=/usr/local/share/met/Rscripts >> /root/.bashrc \ - && echo ulimit -S -s unlimited >> /root/.bashrc - -ENV MET_FONT_DIR /usr/local/share/met/fonts - -# Install required system tools -RUN apt update && apt -y upgrade \ - && apt install -y build-essential gfortran wget unzip curl imagemagick \ - libcurl4-gnutls-dev m4 git automake flex bison libjpeg-dev libpixman-1-dev \ - emacs less \ - libreadline-gplv2-dev libncursesw5-dev libssl-dev libsqlite3-dev tk-dev \ - libgdbm-dev libc6-dev libbz2-dev libffi-dev zlib1g-dev - -# -# Install Python -# https://linuxhint.com/install-python-debian-10/ -# -RUN wget https://www.python.org/ftp/python/${PYTHON_VER}/Python-${PYTHON_VER}.tgz \ - && tar xzf Python-${PYTHON_VER}.tgz \ - && cd Python-${PYTHON_VER} \ - && ./configure --enable-optimizations --enable-shared LDFLAGS="-L/usr/local/lib -Wl,-rpath,/usr/local/lib" \ - && make -j `nproc` \ - && make install \ - && python3 -m pip install --upgrade pip \ - && python3 -m pip install numpy xarray netCDF4 - -RUN echo "Downloading GhostScript fonts from ${GSFONT_URL} into /usr/local/share/met" \ - && mkdir -p /usr/local/share/met \ - && curl -SL ${GSFONT_URL} | tar zxC /usr/local/share/met - -# Fix rules for ghostscript files in convert -# See: https://en.linuxportal.info/tutorials/troubleshooting/how-to-fix-errors-from-imagemagick-imagick-conversion-system-security-policy -# -RUN sed -i 's/policy domain="coder" rights="none" pattern="PS/policy domain="coder" rights="read | write" pattern="PS/g' /etc/ImageMagick-6/policy.xml \ - && sed -i 's/policy domain="coder" rights="none" pattern="EPS"/policy domain="coder" rights="read | write" pattern="EPS"/g' /etc/ImageMagick-6/policy.xml \ - && sed -i 's/policy domain="coder" rights="none" pattern="PDF"/policy domain="coder" rights="read | write" pattern="PDF"/g' /etc/ImageMagick-6/policy.xml \ - && sed -i 's/policy domain="coder" rights="none" pattern="XPS"/policy domain="coder" rights="read | write" pattern="XPS"/g' /etc/ImageMagick-6/policy.xml - -# -# Set the working directory. -# -WORKDIR /met - -RUN wget https://dtcenter.ucar.edu/dfiles/code/METplus/MET/installation/tar_files.tgz \ - && wget https://raw.githubusercontent.com/dtcenter/MET/develop/internal/scripts/installation/compile_MET_all.sh \ - && wget https://raw.githubusercontent.com/dtcenter/MET/develop/internal/scripts/environment/development.docker \ - && tar -zxf tar_files.tgz \ - && export SKIP_MET=yes \ - && chmod +x compile_MET_all.sh \ - && ./compile_MET_all.sh development.docker diff --git a/Dockerfile.debian_unit_test_env b/Dockerfile.debian_unit_test_env deleted file mode 100644 index 6938adb..0000000 --- a/Dockerfile.debian_unit_test_env +++ /dev/null @@ -1,26 +0,0 @@ -ARG MET_BASE_TAG - -FROM dtcenter/met-base:${MET_BASE_TAG} - -MAINTAINER John Halley Gotway - -# -# Set the working directory. -# -WORKDIR /met - -# -# Download and install MET and GhostScript fonts. -# Delete the MET source code for tagged releases matching "v"*. -# -RUN echo "Installing tools needed for running MET unit tests..." \ - && echo "Installing Perl XML Parser..." \ - && apt install -y libxml-parser-perl \ - && echo "Installing R..." \ - && apt install -y r-base \ - && echo "Installing R ncdf4 1.21..." \ - && wget https://cran.r-project.org/src/contrib/ncdf4_1.21.tar.gz \ - && R CMD INSTALL --configure-args="--with-nc-config=/usr/local/netcdf/netcdf-c-4.7.4/nc-config" ncdf4_1.21.tar.gz \ - && echo "Installing NCO and netcdf-bin (for ncdiff)..." \ - && apt install -y nco netcdf-bin \ - && echo "Finished installing unit test tools" diff --git a/Dockerfile.unit_test_env b/Dockerfile.unit_test_env index d95255e..6938adb 100644 --- a/Dockerfile.unit_test_env +++ b/Dockerfile.unit_test_env @@ -15,13 +15,12 @@ WORKDIR /met # RUN echo "Installing tools needed for running MET unit tests..." \ && echo "Installing Perl XML Parser..." \ - && yum makecache \ - && yum -y install perl-XML-Parser \ + && apt install -y libxml-parser-perl \ && echo "Installing R..." \ - && yum -y install R \ - && echo "Installing R ncdf4 1.19..." \ - && wget https://cran.r-project.org/src/contrib/ncdf4_1.19.tar.gz \ - && R CMD INSTALL ncdf4_1.19.tar.gz \ - && echo "Installing NCO (for ncdiff)..." \ - && yum -y install nco \ + && apt install -y r-base \ + && echo "Installing R ncdf4 1.21..." \ + && wget https://cran.r-project.org/src/contrib/ncdf4_1.21.tar.gz \ + && R CMD INSTALL --configure-args="--with-nc-config=/usr/local/netcdf/netcdf-c-4.7.4/nc-config" ncdf4_1.21.tar.gz \ + && echo "Installing NCO and netcdf-bin (for ncdiff)..." \ + && apt install -y nco netcdf-bin \ && echo "Finished installing unit test tools" diff --git a/README.md b/README.md index 541aaa0..f40fdee 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,12 @@ Please see the [MET website](https://dtcenter.org/community-code/model-evaluatio Version History =============== +v2.1 +---- + +* Completes transition to debian base image using Debian 12 (bookworm) +* Adds Proj library dependency + v2.0_debian10 -------------