From 3c77c442a2ba6c8b55a222df9edba87814ac0461 Mon Sep 17 00:00:00 2001 From: asonix Date: Tue, 15 Oct 2019 14:10:42 -0500 Subject: [PATCH 1/5] Add makefile to initiate build process --- Makefile | 10 ++++ build-images.sh | 36 ------------- container/Dockerfile | 71 +++++++++++++++++++++++++ Dockerfile => container/Dockerfile-deps | 66 +---------------------- container/build-images.sh | 38 +++++++++++++ container/pull-containers.sh | 9 ++++ 6 files changed, 130 insertions(+), 100 deletions(-) create mode 100644 Makefile delete mode 100755 build-images.sh create mode 100644 container/Dockerfile rename Dockerfile => container/Dockerfile-deps (53%) create mode 100755 container/build-images.sh create mode 100755 container/pull-containers.sh diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..3bb0fd453 --- /dev/null +++ b/Makefile @@ -0,0 +1,10 @@ +all: pull-containers build-deps build-images + +pull-containers: + ./container/pull-containers.sh + +build-deps: + ./container/build-images.sh Dockerfile-deps ubuntu-dependencies + +build-images: + ./container/build-images.sh Dockerfile $(git rev-parse --abbrev-ref HEAD) diff --git a/build-images.sh b/build-images.sh deleted file mode 100755 index 2e08cf7b7..000000000 --- a/build-images.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash - -set -xe - -TAG=$1 - -require_input() { - input=$1 - name=$2 - - if [ "$input" == "" ]; then - echo "Input, $name, required but not present" - exit 1; - fi -} - -require_input "$TAG" "container tag" - -architectures=('arm64v8' 'arm32v7' 'amd64') - -for architecture in "${architectures[@]}"; do - sed "s/FROM ubuntu/FROM $architecture\/ubuntu/g" Dockerfile > "Dockerfile-$architecture" - if [ "$architecture" == arm32v7 ]; then - sed -i'' 's/x64/armv7l/g' "Dockerfile-$architecture" # for node - sed -i'' 's/amd64/armhf/g' "Dockerfile-$architecture" # for tini - fi - - if [ "$architecture" == arm64v8 ]; then - sed -i'' 's/x64/arm64/g' "Dockerfile-$architecture" # for node - sed -i'' 's/amd64/arm64/g' "Dockerfile-$architecture" # for tini - fi - - buildah build-using-dockerfile -f "Dockerfile-$architecture" -t "$TAG-$architecture" . - - rm "Dockerfile-$architecture" -done diff --git a/container/Dockerfile b/container/Dockerfile new file mode 100644 index 000000000..6887e3314 --- /dev/null +++ b/container/Dockerfile @@ -0,0 +1,71 @@ +FROM deps-amd64/latest as build-dep + +# Use bash for the shell +SHELL ["bash", "-c"] + +COPY Gemfile* package.json yarn.lock /opt/mastodon/ + +RUN cd /opt/mastodon && \ + bundle install -j$(nproc) --deployment --without development test && \ + yarn install --pure-lockfile + +FROM amd64/ubuntu:18.04 + +# Copy over all the langs needed for runtime +COPY --from=build-dep /opt/node /opt/node +COPY --from=build-dep /opt/ruby /opt/ruby +COPY --from=build-dep /opt/jemalloc /opt/jemalloc +COPY --from=build-dep /tini /tini + +RUN chmod +rx /tini + +# Add more PATHs to the PATH +ENV PATH="${PATH}:/opt/ruby/bin:/opt/node/bin:/opt/mastodon/bin" + +# Create the mastodon user +ARG UID=991 +ARG GID=991 +RUN apt update && \ + echo "Etc/UTC" > /etc/localtime && \ + ln -s /opt/jemalloc/lib/* /usr/lib/ && \ + apt install -y whois wget && \ + addgroup --gid $GID mastodon && \ + useradd -m -u $UID -g $GID -d /opt/mastodon mastodon && \ + echo "mastodon:`head /dev/urandom | tr -dc A-Za-z0-9 | head -c 24 | mkpasswd -s -m sha-256`" | chpasswd + +# Install mastodon runtime deps +RUN apt -y --no-install-recommends install \ + libssl1.1 libpq5 imagemagick ffmpeg \ + libicu60 libprotobuf10 libidn11 libyaml-0-2 \ + file ca-certificates tzdata libreadline7 && \ + apt -y install gcc && \ + ln -s /opt/mastodon /mastodon && \ + gem install bundler + +# Copy over mastodon source, and dependencies from building, and set permissions +COPY --chown=mastodon:mastodon . /opt/mastodon +COPY --from=build-dep --chown=mastodon:mastodon /opt/mastodon /opt/mastodon + +# Run mastodon services in prod mode +ENV RAILS_ENV="production" +ENV NODE_ENV="production" + +# Tell rails to serve static files +ENV RAILS_SERVE_STATIC_FILES="true" +ENV BIND="0.0.0.0" + +# Set the run user +USER mastodon + +# Precompile assets +RUN cd ~ && \ + OTP_SECRET=precompile_placeholder SECRET_KEY_BASE=precompile_placeholder rails assets:precompile && \ + yarn cache clean + +# Remove cached packages, break apt +RUN rm -rf /var/cache && \ + rm -rf /var/lib/apt/lists/* + +# Set the work dir and the container entry point +WORKDIR /opt/mastodon +ENTRYPOINT ["/tini", "--"] diff --git a/Dockerfile b/container/Dockerfile-deps similarity index 53% rename from Dockerfile rename to container/Dockerfile-deps index 41b390dca..102521922 100644 --- a/Dockerfile +++ b/container/Dockerfile-deps @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 as build-dep +FROM ubuntu:18.04 # Use bash for the shell SHELL ["bash", "-c"] @@ -56,42 +56,6 @@ RUN npm install -g yarn && \ apt -y install git libicu-dev libidn11-dev \ libpq-dev libprotobuf-dev protobuf-compiler -COPY Gemfile* package.json yarn.lock /opt/mastodon/ - -RUN cd /opt/mastodon && \ - bundle install -j$(nproc) --deployment --without development test && \ - yarn install --pure-lockfile - -FROM ubuntu:18.04 - -# Copy over all the langs needed for runtime -COPY --from=build-dep /opt/node /opt/node -COPY --from=build-dep /opt/ruby /opt/ruby -COPY --from=build-dep /opt/jemalloc /opt/jemalloc - -# Add more PATHs to the PATH -ENV PATH="${PATH}:/opt/ruby/bin:/opt/node/bin:/opt/mastodon/bin" - -# Create the mastodon user -ARG UID=991 -ARG GID=991 -RUN apt update && \ - echo "Etc/UTC" > /etc/localtime && \ - ln -s /opt/jemalloc/lib/* /usr/lib/ && \ - apt install -y whois wget && \ - addgroup --gid $GID mastodon && \ - useradd -m -u $UID -g $GID -d /opt/mastodon mastodon && \ - echo "mastodon:`head /dev/urandom | tr -dc A-Za-z0-9 | head -c 24 | mkpasswd -s -m sha-256`" | chpasswd - -# Install mastodon runtime deps -RUN apt -y --no-install-recommends install \ - libssl1.1 libpq5 imagemagick ffmpeg \ - libicu60 libprotobuf10 libidn11 libyaml-0-2 \ - file ca-certificates tzdata libreadline7 && \ - apt -y install gcc && \ - ln -s /opt/mastodon /mastodon && \ - gem install bundler - # Add tini ENV TINI_VERSION="0.18.0" ENV TINI_ARCH="amd64" @@ -99,9 +63,7 @@ ADD https://github.com/krallin/tini/releases/download/v${TINI_VERSION}/tini-${TI ADD https://github.com/krallin/tini/releases/download/v${TINI_VERSION}/tini-${TINI_ARCH}.asc /tini.asc RUN apt update && \ - apt -y --no-install-recommends install gpg gpg-agent dirmngr && \ - rm -rf /var/cache && \ - rm -rf /var/lib/apt/lists/* + apt -y --no-install-recommends install gpg gpg-agent dirmngr RUN gpg --batch \ --keyserver hkp://p80.pool.sks-keyservers.net:80 \ @@ -109,27 +71,3 @@ RUN gpg --batch \ gpg --batch --verify /tini.asc /tini RUN chmod +rx /tini - -# Copy over mastodon source, and dependencies from building, and set permissions -COPY --chown=mastodon:mastodon . /opt/mastodon -COPY --from=build-dep --chown=mastodon:mastodon /opt/mastodon /opt/mastodon - -# Run mastodon services in prod mode -ENV RAILS_ENV="production" -ENV NODE_ENV="production" - -# Tell rails to serve static files -ENV RAILS_SERVE_STATIC_FILES="true" -ENV BIND="0.0.0.0" - -# Set the run user -USER mastodon - -# Precompile assets -RUN cd ~ && \ - OTP_SECRET=precompile_placeholder SECRET_KEY_BASE=precompile_placeholder rails assets:precompile && \ - yarn cache clean - -# Set the work dir and the container entry point -WORKDIR /opt/mastodon -ENTRYPOINT ["/tini", "--"] diff --git a/container/build-images.sh b/container/build-images.sh new file mode 100755 index 000000000..1c76078b5 --- /dev/null +++ b/container/build-images.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +set -xe + +SOURCE=$1 +TAG=$2 + +require_input() { + input=$1 + name=$2 + + if [ "$input" == "" ]; then + echo "Input, $name, required but not present" + exit 1; + fi +} + +require_input "$SOURCE" "Dockerfile" +require_input "$TAG" "container tag" + +architectures=('arm64v8' 'arm32v7' 'amd64') + +for architecture in "${architectures[@]}"; do + sed "s/FROM ubuntu/FROM $architecture\/ubuntu/g" "$SOURCE" > ""$SOURCE"-$architecture" + if [ "$architecture" == arm32v7 ]; then + sed -i'' 's/x64/armv7l/g' ""$SOURCE"-$architecture" # for node + sed -i'' 's/amd64/armhf/g' ""$SOURCE"-$architecture" # for tini + fi + + if [ "$architecture" == arm64v8 ]; then + sed -i'' 's/x64/arm64/g' ""$SOURCE"-$architecture" # for node + sed -i'' 's/amd64/arm64/g' ""$SOURCE"-$architecture" # for tini + fi + + buildah build-using-dockerfile -f ""$SOURCE"-$architecture" -t "$TAG-$architecture" .. + + rm ""$SOURCE"-$architecture" +done diff --git a/container/pull-containers.sh b/container/pull-containers.sh new file mode 100755 index 000000000..5d7314626 --- /dev/null +++ b/container/pull-containers.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -xe + +architectures=('arm64v8' 'arm32v7' 'amd64') + +for architecture in "${architectures[@]}"; do + podman pull "$architecture/ubuntu:18.04" +done From 7dc4cad7a47f1d7ae0b43eb25c90acd7a97dbdc9 Mon Sep 17 00:00:00 2001 From: asonix Date: Tue, 15 Oct 2019 14:12:43 -0500 Subject: [PATCH 2/5] Update container name --- container/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container/Dockerfile b/container/Dockerfile index 6887e3314..6b61a3c91 100644 --- a/container/Dockerfile +++ b/container/Dockerfile @@ -1,4 +1,4 @@ -FROM deps-amd64/latest as build-dep +FROM localhost/ubuntu-dependencies-amd64:latest as build-dep # Use bash for the shell SHELL ["bash", "-c"] From 4c7836424d82a08e999989df523ec5cb3eb6bba2 Mon Sep 17 00:00:00 2001 From: asonix Date: Tue, 15 Oct 2019 19:52:04 -0500 Subject: [PATCH 3/5] Mostly working, hopefully all working --- Makefile | 26 ++++++++++++++++++++------ container/Dockerfile | 2 +- container/Dockerfile-deps | 5 +---- container/build-images.sh | 22 +++++++++++++--------- container/pull-containers.sh | 6 +++++- container/push-containers.sh | 23 +++++++++++++++++++++++ 6 files changed, 63 insertions(+), 21 deletions(-) create mode 100755 container/push-containers.sh diff --git a/Makefile b/Makefile index 3bb0fd453..5a484d8fb 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,24 @@ -all: pull-containers build-deps build-images +GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD) +IMAGE_NAME = "asonix/masto-deps" -pull-containers: +masto: pull-deps images push-images + +all: pull-base deps push-deps pull-deps images push-images + +pull-base: ./container/pull-containers.sh -build-deps: - ./container/build-images.sh Dockerfile-deps ubuntu-dependencies +deps: + ./container/build-images.sh container/Dockerfile-deps $(IMAGE_NAME) -build-images: - ./container/build-images.sh Dockerfile $(git rev-parse --abbrev-ref HEAD) +push-deps: + ./container/push-containers.sh $(IMAGE_NAME) + +pull-deps: + ./container/pull-containers.sh $(IMAGE_NAME) + +images: + ./container/build-images.sh container/Dockerfile "$(GIT_BRANCH)" + +push-images: + ./container/push-containers.sh $(GIT_BRANCH) diff --git a/container/Dockerfile b/container/Dockerfile index 6b61a3c91..c1149c8d9 100644 --- a/container/Dockerfile +++ b/container/Dockerfile @@ -1,4 +1,4 @@ -FROM localhost/ubuntu-dependencies-amd64:latest as build-dep +FROM asonix/masto-deps:amd64 as build-dep # Use bash for the shell SHELL ["bash", "-c"] diff --git a/container/Dockerfile-deps b/container/Dockerfile-deps index 102521922..5090792f8 100644 --- a/container/Dockerfile-deps +++ b/container/Dockerfile-deps @@ -1,7 +1,4 @@ -FROM ubuntu:18.04 - -# Use bash for the shell -SHELL ["bash", "-c"] +FROM amd64/ubuntu:18.04 # Install Node ENV NODE_VER="12.11.1" diff --git a/container/build-images.sh b/container/build-images.sh index 1c76078b5..0501c2e6f 100755 --- a/container/build-images.sh +++ b/container/build-images.sh @@ -18,21 +18,25 @@ require_input() { require_input "$SOURCE" "Dockerfile" require_input "$TAG" "container tag" -architectures=('arm64v8' 'arm32v7' 'amd64') +ARCHITECTURES=('arm64v8' 'arm32v7' 'amd64') -for architecture in "${architectures[@]}"; do - sed "s/FROM ubuntu/FROM $architecture\/ubuntu/g" "$SOURCE" > ""$SOURCE"-$architecture" +for architecture in "${ARCHITECTURES[@]}"; do + cp "$SOURCE" "$SOURCE-$architecture" if [ "$architecture" == arm32v7 ]; then - sed -i'' 's/x64/armv7l/g' ""$SOURCE"-$architecture" # for node - sed -i'' 's/amd64/armhf/g' ""$SOURCE"-$architecture" # for tini + sed -i'' 's/:amd64/:arm32v7/g' "$SOURCE-$architecture" # source image + sed -i'' 's/FROM amd64/FROM arm32v7/g' "$SOURCE-$architecture" # source image + sed -i'' 's/x64/armv7l/g' "$SOURCE-$architecture" # for node + sed -i'' 's/amd64/armhf/g' "$SOURCE-$architecture" # for tini fi if [ "$architecture" == arm64v8 ]; then - sed -i'' 's/x64/arm64/g' ""$SOURCE"-$architecture" # for node - sed -i'' 's/amd64/arm64/g' ""$SOURCE"-$architecture" # for tini + sed -i'' 's/:amd64/:arm64v8/g' "$SOURCE-$architecture" # source image + sed -i'' 's/FROM amd64/FROM arm64v8/g' "$SOURCE-$architecture" # source image + sed -i'' 's/x64/arm64/g' "$SOURCE-$architecture" # for node + sed -i'' 's/amd64/arm64/g' "$SOURCE-$architecture" # for tini fi - buildah build-using-dockerfile -f ""$SOURCE"-$architecture" -t "$TAG-$architecture" .. + buildah build-using-dockerfile -f "$SOURCE-$architecture" -t "$TAG:$architecture" . - rm ""$SOURCE"-$architecture" + rm "$SOURCE-$architecture" done diff --git a/container/pull-containers.sh b/container/pull-containers.sh index 5d7314626..ab16e3efc 100755 --- a/container/pull-containers.sh +++ b/container/pull-containers.sh @@ -5,5 +5,9 @@ set -xe architectures=('arm64v8' 'arm32v7' 'amd64') for architecture in "${architectures[@]}"; do - podman pull "$architecture/ubuntu:18.04" + if [ "$1" == "" ]; then + podman pull "$architecture/ubuntu:18.04" + else + podman pull "$1:$architecture" + fi done diff --git a/container/push-containers.sh b/container/push-containers.sh new file mode 100755 index 000000000..37f449a7f --- /dev/null +++ b/container/push-containers.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +set -xe + +TAG=$1 + +require_input() { + input=$1 + name=$2 + + if [ "$input" == "" ]; then + echo "Input, $name, required but not present" + exit 1; + fi +} + +require_input "$TAG" "container tag" + +ARCHITECTURES=('arm64v8' 'arm32v7' 'amd64') + +for arch in "${ARCHITECTURES[@]}"; do + podman push "$TAG:$arch" "docker://docker.io/asonix/masto-deps:$arch" +done From 8d2e9b71328874be3c7b50c9d4a7a202b7c3397f Mon Sep 17 00:00:00 2001 From: asonix Date: Wed, 16 Oct 2019 02:11:31 -0500 Subject: [PATCH 4/5] Don't try deleting immutable folder --- container/Dockerfile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/container/Dockerfile b/container/Dockerfile index c1149c8d9..596517a79 100644 --- a/container/Dockerfile +++ b/container/Dockerfile @@ -62,10 +62,6 @@ RUN cd ~ && \ OTP_SECRET=precompile_placeholder SECRET_KEY_BASE=precompile_placeholder rails assets:precompile && \ yarn cache clean -# Remove cached packages, break apt -RUN rm -rf /var/cache && \ - rm -rf /var/lib/apt/lists/* - # Set the work dir and the container entry point WORKDIR /opt/mastodon ENTRYPOINT ["/tini", "--"] From 3bb8557ae7e580875ccf874edf36f57b690062d8 Mon Sep 17 00:00:00 2001 From: asonix Date: Wed, 16 Oct 2019 09:28:21 -0500 Subject: [PATCH 5/5] Push to proper repo --- container/push-containers.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container/push-containers.sh b/container/push-containers.sh index 37f449a7f..fe951d56c 100755 --- a/container/push-containers.sh +++ b/container/push-containers.sh @@ -19,5 +19,5 @@ require_input "$TAG" "container tag" ARCHITECTURES=('arm64v8' 'arm32v7' 'amd64') for arch in "${ARCHITECTURES[@]}"; do - podman push "$TAG:$arch" "docker://docker.io/asonix/masto-deps:$arch" + podman push "$TAG:$arch" "docker://docker.io/$TAG:$arch" done