Skip to content

Commit

Permalink
feat: Report available system memory under dcc_free_mem
Browse files Browse the repository at this point in the history
This patch adds a trivial wrapper HTTP server, written in Python, that
intercepts the requests going towards the `--stats-port` of the running
`distccd` server.
It relays the request verbatim at first, and if the response is a valid
statistics message, extends it at the bottom with a line

    dcc_free_mem 16384 MB

where the numeric value is obtained from `free`'s output under
**Available** system memory.
  • Loading branch information
whisperity committed Jun 7, 2024
1 parent dafb1c9 commit c9457f8
Show file tree
Hide file tree
Showing 10 changed files with 529 additions and 88 deletions.
89 changes: 84 additions & 5 deletions .github/workflows/docker-image.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,12 +34,20 @@ jobs:
mv -v ./shellcheck-v*/shellcheck "./shellcheck"
rm -rv ./shellcheck-v*
./shellcheck --version
- name: "Install PyCodeStyle"
run: |
sudo apt-get install -y --no-install-recommends \
python3-pycodestyle
- name: "ShellCheck"
run: |
./shellcheck \
--color=always \
etc/cron.*/* \
usr/local/*bin/*
- name: "PyCodeStyle"
run: |
python3 -m pycodestyle \
.
build:
name: "Build ${{ github.event_name == 'workflow_dispatch' && inputs.release && 'and Deploy' || '' }} Docker Image"
Expand All @@ -54,16 +62,87 @@ jobs:
with:
load: true
tags: "distcc-docker:${{ env.RUNS_ON }}"
- name: "Simple test"
- name: "Test starting the container"
run: |
docker run \
--init \
--rm \
"distcc-docker:${{ env.RUNS_ON }}" \
--jobs $(nproc) \
-- \
bash -c \
"set -x; ps fauxw; /usr/bin/ls -alh .; /usr/bin/ls -alh /var/log; cat /var/log/*; exit;"
- name: "Test that \"dcc_free_mem\" is reported"
run: |
set -ex
docker run \
--detach \
--init \
--name "distcc-1" \
--publish "3632:3632/tcp" \
--publish "3633:3633/tcp" \
--rm \
"distcc-docker:${{ env.RUNS_ON }}" \
--jobs $(nproc) \
-- \
bash -c "set -x; sleep 120; exit;"
sleep 10
STAT_RESULT="$(curl "http://localhost:3633" | grep "dcc_free_mem")"
if [ -z "$STAT_RESULT" ]; then
echo "Server did not report 'dcc_free_mem'!" >&2
exit 1
fi
docker kill "distcc-1"
- name: "Test compilation in the container"
run: |
set -ex
sudo apt-get update -y
sudo apt-get install distcc g++ --no-install-recommends
sudo update-distcc-symlinks
docker run \
--detach \
--init \
--name "distcc-1" \
--publish "3632:3632/tcp" \
--publish "3633:3633/tcp" \
--rm \
"distcc-docker:${{ env.RUNS_ON }}" \
--jobs $(nproc) \
-- \
bash -c \
"set -x; ps fauxw; /usr/bin/ls -alh .; /usr/bin/ls -alh /var/log; cat /var/log/*; exit;"
--jobs $(nproc) \
-- \
bash -c "set -x; sleep 120; exit;"
sleep 10
curl "http://localhost:3633"
echo "int main() { return MY_EXIT_CODE; }" >> main.cpp
DISTCC_HOSTS="127.0.0.1:3632/$(nproc),lzo" \
distcc /usr/bin/g++ \
-D MY_EXIT_CODE=42 \
-c \
./main.cpp \
-o main.o
rm -v main.cpp
curl "http://localhost:3633" | grep "dcc_compile_ok 1"
docker kill "distcc-1"
/usr/bin/g++ main.o -o main
set +e
./main
if [ "$?" -ne 42 ]; then
echo "Unexpected error code obtained!" >&2
exit 1
fi
- name: "Log in to GitHub Container Registry (GHCR)"
if: "github.ref == 'refs/heads/master' && github.event_name == 'workflow_dispatch' && inputs.release"
uses: "docker/login-action@v3"
Expand Down
32 changes: 18 additions & 14 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@ FROM ubuntu:20.04
MAINTAINER Whisperity <whisperity-packages@protonmail.com>


RUN export DEBIAN_FRONTEND=noninteractive && \
RUN \
export DEBIAN_FRONTEND=noninteractive && \
set -x && \
apt-get update -y && \
apt-get install -y --no-install-recommends \
Expand All @@ -15,6 +16,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
lsb-release \
locales \
logrotate \
python3 \
wget \
&& \
apt-get purge -y --auto-remove && \
Expand All @@ -24,7 +26,8 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
mkdir -pv "/var/log/"


RUN export DEBIAN_FRONTEND=noninteractive; \
RUN \
export DEBIAN_FRONTEND=noninteractive; \
sed -i -e "s/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/" "/etc/locale.gen" && \
dpkg-reconfigure --frontend=noninteractive locales && \
update-locale LANG="en_US.UTF-8"
Expand All @@ -34,11 +37,11 @@ ENV \
LC_ALL="en_US.UTF-8"


COPY usr/local/sbin/install-compilers.sh /usr/local/sbin/install-compilers.sh

# Set to non-zero if the compilers should only be installed into the container,
# and not immediately baked into the image itself.
ARG LAZY_COMPILERS=0

COPY usr/local/sbin/install-compilers.sh /usr/local/sbin/install-compilers.sh
RUN \
if [ "x$LAZY_COMPILERS" = "x0" ]; \
then \
Expand All @@ -52,25 +55,26 @@ RUN \
fi


COPY etc/ /etc/
COPY usr/ /usr/


ARG USERNAME="distcc"
RUN echo "Creating service user: $USERNAME ..." >&2 && \
mkdir -pv "/var/lib/distcc/" && \
chmod -Rv 755 "/var/lib/distcc" && \
RUN \
cp -av "/etc/skel/." "/root/" && \
echo "Creating service user: $USERNAME ..." >&2 && \
useradd "$USERNAME" \
--system \
--create-home \
--comment "DistCC service" \
--home-dir "/var/lib/distcc/" \
--shell "/bin/bash" \
--home-dir "/var/lib/distcc/" && \
cp -v "/root/.bashrc" "/root/.profile" "/var/lib/distcc/" && \
--system \
&& \
chown -Rv "$USERNAME":"$USERNAME" "/var/lib/distcc" && \
echo "$USERNAME" > "/var/lib/distcc/distcc.user" && \
chmod -v 444 "/var/lib/distcc/distcc.user"


COPY etc/ /etc/
COPY usr/ /usr/


# Expose the DistCC server's normal job and statistics subservice port.
# Custom ports to be used on the host machine should be managed via Docker.
EXPOSE \
Expand Down
2 changes: 1 addition & 1 deletion etc/logrotate.d/cron
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
/var/log/cron.log {
daily
rotate 30
rotate 7

compress
copytruncate
Expand Down
2 changes: 1 addition & 1 deletion etc/logrotate.d/distcc
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
/var/log/distccd.log {
daily
rotate 30
rotate 7

compress
copytruncate
Expand Down
11 changes: 11 additions & 0 deletions etc/logrotate.d/http
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
/var/log/access.log
/var/log/error.log {
daily
rotate 7

compress
copytruncate
delaycompress
missingok
notifempty
}
10 changes: 10 additions & 0 deletions etc/logrotate.d/syslog
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
/var/log/syslog {
daily
rotate 7

compress
copytruncate
delaycompress
missingok
notifempty
}
7 changes: 7 additions & 0 deletions etc/skel/.config/htop/htoprc
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
hide_kernel_threads=1
hide_userland_threads=1
highlight_base_name=1
highlight_megabytes=1
shadow_other_users=0
show_program_path=0
tree_view=1
Loading

0 comments on commit c9457f8

Please sign in to comment.