feat: build and publish weekly alpha/development container image

- make use of docker buildx caching when possible (helpful with local docker builds) - introduce a reusable container workflow which is triggered by docker-release and docker-weekly workflows - added an alpine-dev Dockerfile - split release.sh contents into different Makefile targets - make use of job matrix to build alpine + ubuntu in parallel - make alpine build optional by checking for Dockerfile presence -- as the pre-built binaries don't work with alpine, because of glibc <-> musl incompatibilities Signed-off-by: Philipp Born <git@pborn.eu>
2024-11-21 23:19:53 +00:00 · 2022-12-23 18:48:15 +01:00 · 2022-12-23 18:48:15 +01:00 · 98b92a0073
commit 98b92a0073
parent 8802fb4a3c
9 changed files with 355 additions and 94 deletions
--- a/.github/workflows/docker-release.yml
+++ b/.github/workflows/docker-release.yml
@ -24,10 +24,39 @@ env:
  IS_PRERELEASE: ${{ github.event.release.prerelease || github.event.inputs.PRERELEASE }}

 jobs:
-# ===============================================================
-#   Building Dev Images
-# ===============================================================
+  # This workaround is needed, as it's not possible to reference env.FOOBAR directly at this stage
+  # - for ex. https://github.com/actions/runner/issues/1189
+  release-container-prereq:
+    runs-on: ubuntu-latest
+    outputs:
+      tag: ${{ steps.release-param.outputs.tag }}
+      tag_latest: ${{ steps.release-param.outputs.tag_latest }}
+    steps:
+    - id: release-param
+      run: |
+        # If env.IS_PRERELEASE is true, set tag to alpha and do not enable tag_latest
+        # If env.IS_PRERELEASE is not true (aka false), don't set an extra tag and enable tag_latest
+        echo "::set-output name=tag::${{ env.IS_PRERELEASE == 'true' && 'alpha' }}"
+        echo "::set-output name=tag_latest::${{ env.IS_PRERELEASE == 'true' && 'false' || 'true' }}"
+
+  release-container:
+    needs: release-container-prereq
+    uses: ./.github/workflows/reusable-container-workflow.yaml
+    with:
+      build_type: prod
+      tag: ${{ needs.release-container-prereq.outputs.tag }}
+      tag_latest: ${{ needs.release-container-prereq.outputs.tag_latest == 'true' }}
+      image: ghcr.io/${{ github.repository }}
+      registry: ghcr.io
+      registry_username: ${{ github.repository_owner }}
+      fetch_release: true
+      release_version: ${{ github.event.inputs.TAG_NAME || github.event.release.tag_name }}
+    secrets:
+      registry_password: ${{ secrets.GITHUB_TOKEN }}
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
  release:
+    needs: [release-container-prereq, release-container]
    runs-on: ubuntu-latest
    steps:
    - name: print_env
@ -41,76 +70,6 @@ jobs:
    - name: Install helm
      uses: azure/setup-helm@v3

-    - uses: dsaltares/fetch-gh-release-asset@master
-      with:
-        version: 'tags/${{ env.TAG_NAME }}'
-        regex: true
-        file: "dragonfly-.*\\.tar\\.gz"
-        target: 'releases/'
-        token: ${{ secrets.GITHUB_TOKEN }}
-
-    - name: Set up QEMU
-      id: qemu
-      uses: docker/setup-qemu-action@v1
-      with:
-          platforms: arm64,amd64
-
-    - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v1
-
-    - name: Login to GitHub Container Registry
-      uses: docker/login-action@v2
-      with:
-        registry: ghcr.io
-        username: ${{ github.actor }}
-        password: ${{ secrets.GITHUB_TOKEN }}
-
-    - name: Extract artifacts
-      run: |
-        echo "Event prerelease ${{ github.event.release.prerelease }}"
-        echo "Input prerelease ${{ github.event.inputs.PRERELEASE }}"
-        ls -l
-        ls -l releases
-        for f in releases/*.tar.gz; do tar xvfz $f -C releases; done
-        rm releases/*.tar.gz
-
-    - name: Build release image
-      if: env.IS_PRERELEASE != 'true'
-      uses: docker/build-push-action@v3
-      with:
-        context: .
-        platforms: linux/amd64,linux/arm64
-
-        # Define QEMU settings inside the builder
-        build-args: |
-          QEMU_CPU=max,pauth-impdef=on
-
-        push: ${{ github.event_name != 'pull_request' }}
-        tags: |
-          ghcr.io/${{ github.repository }}:ubuntu
-          ghcr.io/${{ github.repository }}:latest
-          ghcr.io/${{ github.repository }}:${{ env.TAG_NAME }}
-          ghcr.io/${{ github.repository }}:${{ env.TAG_NAME }}-ubuntu
-
-        file: tools/docker/Dockerfile.ubuntu-prod
-        cache-from: type=registry,ref=${{ github.repository }}:latest
-        cache-to: type=inline
-
-    - name: Build pre-release image
-      if: env.IS_PRERELEASE == 'true'
-      uses: docker/build-push-action@v3
-      with:
-        context: .
-        platforms: linux/amd64,linux/arm64
-
-        push: ${{ github.event_name != 'pull_request' }}
-        tags: |
-          ghcr.io/${{ github.repository }}:alpha
-          ghcr.io/${{ github.repository }}:${{ env.TAG_NAME }}
-        file: tools/docker/Dockerfile.ubuntu-prod
-        cache-from: type=registry,ref=${{ github.repository }}:latest
-        cache-to: type=inline
-
    - name: Configure Git
      if: env.IS_PRERELEASE != 'true'
      run: |
--- a/.github/workflows/docker-weekly.yml
+++ b/.github/workflows/docker-weekly.yml
@ -0,0 +1,24 @@
+name: weekly docker build
+
+on:
+  schedule:
+    # Monday midnight
+    - cron: '0 0 * * 1'
+  workflow_dispatch:
+
+permissions:
+  packages: write
+  contents: write
+  id-token: write
+
+jobs:
+  weekly-container-build:
+    uses: ./.github/workflows/reusable-container-workflow.yaml
+    with:
+      build_type: dev
+      tag: alpha
+      image: ghcr.io/${{ github.repository }}
+      registry: ghcr.io
+      registry_username: ${{ github.repository_owner }}
+    secrets:
+      registry_password: ${{ secrets.GITHUB_TOKEN }}
--- a/.github/workflows/reusable-container-workflow.yaml
+++ b/.github/workflows/reusable-container-workflow.yaml
@ -0,0 +1,146 @@
+name: Reusable Container Build Workflow
+
+on:
+  workflow_call:
+    inputs:
+      # Which suffix to look for with the Dockerfile. Can be dev or prod
+      build_type:
+        required: true
+        type: string
+      # For example 'alpha', for pre-release or weekly builds
+      tag:
+        required: false
+        type: string
+      # Is this a final release? Then we set this to true, so the 'latest' tag is updated
+      tag_latest:
+        required: false
+        type: boolean
+      # The container image dragonflydb/dragonfly
+      image:
+        required: true
+        type: string
+      # ghcr.io / hub.docker.com / quay.io / you name it
+      registry:
+        required: true
+        type: string
+      # Username used to login to the registry
+      registry_username:
+        required: true
+        type: string
+      # Do we have to fetch release assets? Then set this to true.
+      # Not required for build_type == dev, as they entirely build from source
+      # But for build_type == prod, as they're based on the release assets
+      fetch_release:
+        required: false
+        type: boolean
+      # Which version are we fetching? Should be identical to the release version.
+      # For example v0.12.0
+      release_version:
+        required: false
+        type: string
+    secrets:
+      # Password used to login to the registry
+      registry_password:
+        required: true
+      # Github Personal Access Token used to fetch assets from a release
+      GH_TOKEN:
+        required: false
+
+jobs:
+  container-build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - flavor: alpine
+            dockerfile: tools/docker/Dockerfile.alpine
+            tag_main: false
+          - flavor: ubuntu
+            dockerfile: tools/docker/Dockerfile.ubuntu
+            tag_main: true
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
+          submodules: true
+
+      - if: inputs.fetch_release
+        name: Fetch release asset
+        uses: dsaltares/fetch-gh-release-asset@1.1.0
+        with:
+          version: "tags/${{ inputs.release_version }}"
+          regex: true
+          file: "dragonfly-.*\\.tar\\.gz"
+          target: 'releases/'
+          token: ${{ secrets.GH_TOKEN }}
+
+      - if: inputs.fetch_release
+        name: Extract artifacts
+        run: |
+          echo "Event prerelease ${{ github.event.release.prerelease }}"
+          echo "Input prerelease ${{ github.event.inputs.PRERELEASE }}"
+          ls -l
+          ls -l releases
+          for f in releases/*.tar.gz; do tar xvfz $f -C releases; done
+          rm releases/*.tar.gz
+
+      - name: Set up QEMU
+        id: qemu
+        uses: docker/setup-qemu-action@v1
+        with:
+          platforms: arm64,amd64
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ inputs.registry }}
+          username: ${{ inputs.registry_username }}
+          password: ${{ secrets.registry_password }}
+
+      - name: Docker meta
+        id: metadata
+        uses: docker/metadata-action@v3
+        with:
+          images: |
+            ${{ inputs.image }}
+          tags: |
+            # will set tag 'latest' for ubuntu build on production push
+            # inputs.tag_latest will be true on when triggered by docker-release.yml
+            # matrix.tag_main will only be true for ubuntu flavor
+            type=raw,value=latest,enable=${{ matrix.tag_main && inputs.tag_latest }}
+
+            # set a tag like 'alpine' or 'ubuntu', if we're setting 'latest' during this build as well
+            type=raw,value=${{ matrix.flavor }},enable=${{ inputs.tag_latest }}
+
+            # will set tag like 'alpha' for ubuntu build, if inputs.tag is not empty
+            # but will set the non-flavored tag only, if matrix.tag_main is true
+            type=raw,value=${{ inputs.tag }},enable=${{ inputs.tag != 'false' && matrix.tag_main }}
+
+            # will set tag like 'alpha-(ubuntu|alpine)', if inputs.tag is not empty
+            type=raw,value=${{ inputs.tag }}-${{ matrix.flavor }},enable=${{ inputs.tag != 'false' }}
+
+            # will set tag like 'v0.12.0' for ubuntu build, if inputs.release_version is not empty
+            # but will set the non-flavored tag only, if matrix.tag_main is true
+            type=raw,value=${{ inputs.release_version }},enable=${{ matrix.tag_main && inputs.release_version != '' }}
+
+            # will set tag like 'v0.12.0-(ubuntu|alpine)', if inputs.release_version is not empty
+            type=raw,value=${{ inputs.release_version }}-${{ matrix.flavor }},enable=${{ inputs.release_version != '' }}
+
+      - if: ${{ hashFiles(format('{0}-{1}', matrix.dockerfile, inputs.build_type)) }}
+        name: Build release image
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          build-args: |
+            QEMU_CPU=max,pauth-impdef=on
+          push: true
+          tags: |
+            ${{ steps.metadata.outputs.tags }}
+          labels: ${{ steps.metadata.outputs.labels }}
+          file: ${{ matrix.dockerfile }}-${{ inputs.build_type }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
--- a/43
+++ b/43
@ -0,0 +1,43 @@
+BUILD_ARCH := $(shell uname -m)
+RELEASE_NAME := "dragonfly-${BUILD_ARCH}"
+
+HELIO_RELEASE := $(if $(HELIO_RELEASE),y,)
+HELIO_RELEASE_FLAGS = -DHELIO_RELEASE_FLAGS="-flto"
+HELIO_USE_STATIC_LIBS = ON
+HELIO_OPENSSL_USE_STATIC_LIBS = ON
+HELIO_ENABLE_GIT_VERSION = ON
+HELIO_WITH_UNWIND = OFF
+
+HELIO_FLAGS = $(if $(HELIO_RELEASE),-release $(HELIO_RELEASE_FLAGS),) \
+              -DBoost_USE_STATIC_LIBS=$(HELIO_USE_STATIC_LIBS) \
+              -DOPENSSL_USE_STATIC_LIBS=$(HELIO_OPENSSL_USE_STATIC_LIBS) \
+              -DENABLE_GIT_VERSION=$(HELIO_ENABLE_GIT_VERSION) \
+              -DWITH_UNWIND=$(HELIO_WITH_UNWIND) \
+
+.PHONY: default
+
+configure:
+	./helio/blaze.sh $(HELIO_FLAGS)
+
+build:
+	cd build-opt; \
+	ninja dragonfly; \
+	ldd dragonfly
+
+build-debug:
+	cd build-dbg; \
+	ninja dragonfly; \
+	ldd dragonfly
+
+package:
+	cd build-opt; \
+	mv dragonfly $(RELEASE_NAME); \
+	tar cvfz $(RELEASE_NAME).unstripped.tar.gz $(RELEASE_NAME) ../LICENSE.md; \
+	strip $(RELEASE_NAME); \
+	tar cvfz $(RELEASE_NAME).tar.gz $(RELEASE_NAME) ../LICENSE.md
+
+release: configure build
+
+release-debug: configure build-debug
+
+default: release
--- a/tools/docker/Dockerfile.alpine-dev
+++ b/tools/docker/Dockerfile.alpine-dev
@ -0,0 +1,44 @@
+# syntax=docker/dockerfile:1
+FROM gcr.io/cadvisor/cadvisor:v0.46.0 as libpfm_donor
+
+FROM alpine:3.17.0 as builder
+
+# "openssl-libs-static" fixes "Could NOT find OpenSSL, try to set the path to OpenSSL root folder in the"
+RUN apk add autoconf-archive automake bash bison boost-dev cmake coreutils \
+        curl ccache git gcc gdb g++ libunwind-dev libtool libxml2-dev make ninja \
+        openssl-dev openssl-libs-static patch zip zstd-dev
+
+# This is required to make static linking work
+RUN ls -1 /usr/lib/libboost_*.so | while read -r _file; do ln -sfv ${_file} ${_file//.so/.a}; done
+
+# Borrow libpfm from cadvisor, so we don't have to build it ourselves
+# https://github.com/google/cadvisor/blob/master/deploy/Dockerfile
+COPY --from=libpfm_donor /usr/local/lib/libpfm.so* /usr/local/lib/
+
+WORKDIR /build
+
+COPY . ./
+
+RUN make HELIO_RELEASE=y release
+
+RUN build-opt/dragonfly --version
+
+FROM alpine:3.17.0
+
+RUN apk --no-cache add libgcc libstdc++ libunwind boost1.80-fiber zstd-dev su-exec netcat-openbsd
+
+RUN addgroup -S -g 1000 dfly && adduser -S -G dfly -u 999 dfly
+RUN mkdir /data && chown dfly:dfly /data
+
+VOLUME /data
+WORKDIR /data
+COPY tools/docker/entrypoint.sh /usr/local/bin/entrypoint.sh
+COPY tools/docker/healthcheck.sh /usr/local/bin/healthcheck.sh
+COPY --from=builder /build/build-opt/dragonfly /usr/local/bin/
+
+HEALTHCHECK CMD /usr/local/bin/healthcheck.sh
+ENTRYPOINT ["entrypoint.sh"]
+
+EXPOSE 6379
+
+CMD ["dragonfly", "--logtostderr"]
--- a/tools/docker/Dockerfile.alpine-prod.wip
+++ b/tools/docker/Dockerfile.alpine-prod.wip
@ -12,9 +12,6 @@ WORKDIR build-opt
 RUN ninja dragonfly

 FROM alpine:latest
-ARG ORG_NAME=dragonflydb
-LABEL org.opencontainers.image.title Dragonfly
-LABEL org.opencontainers.image.source https://github.com/${ORG_NAME}/dragonfly

 RUN addgroup -S -g 1000 dfly && adduser -S -G dfly -u 999 dfly
 RUN apk --no-cache add libgcc libstdc++ libunwind boost1.77-fiber \
--- a/tools/docker/Dockerfile.ubuntu-dev
+++ b/tools/docker/Dockerfile.ubuntu-dev
@ -0,0 +1,56 @@
+# syntax=docker/dockerfile:1
+FROM ubuntu:20.04 as builder
+
+RUN \
+  rm -f /etc/apt/apt.conf.d/docker-clean; \
+  echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache
+
+RUN \
+  --mount=type=cache,target=/var/cache/apt,sharing=locked \
+  --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    export DEBIAN_FRONTEND=noninteractive && \
+      apt update && \
+      apt install -q -y autoconf-archive cmake curl git libssl-dev \
+        libunwind-dev ninja-build libtool gcc-9 g++-9 libboost-fiber-dev \
+        libxml2-dev zip libzstd-dev
+
+RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 40  \
+                        --slave   /usr/bin/g++ g++ /usr/bin/g++-9
+
+WORKDIR /build
+
+COPY . ./
+
+RUN make HELIO_RELEASE=y release
+
+RUN build-opt/dragonfly --version
+
+RUN curl -O https://raw.githubusercontent.com/ncopa/su-exec/212b75144bbc06722fbd7661f651390dc47a43d1/su-exec.c && \
+     gcc -Wall -O2 su-exec.c -o su-exec
+
+FROM ubuntu:20.04
+
+RUN \
+  --mount=type=tmpfs,target=/var/cache/apt \
+  --mount=type=tmpfs,target=/var/lib/apt \
+    export DEBIAN_FRONTEND=noninteractive && \
+      apt update && \
+      apt install -q -y netcat-openbsd
+
+RUN groupadd -r -g 999 dfly && useradd -r -g dfly -u 999 dfly
+RUN mkdir /data && chown dfly:dfly /data
+
+VOLUME /data
+WORKDIR /data
+COPY tools/docker/entrypoint.sh /usr/local/bin/entrypoint.sh
+COPY tools/docker/healthcheck.sh /usr/local/bin/healthcheck.sh
+COPY --from=builder /build/su-exec /usr/local/bin/
+COPY --from=builder /build/build-opt/dragonfly /usr/local/bin/
+
+HEALTHCHECK CMD /usr/local/bin/healthcheck.sh
+ENTRYPOINT ["entrypoint.sh"]
+
+# For inter-container communication.
+EXPOSE 6379
+
+CMD ["dragonfly", "--logtostderr"]
--- a/tools/docker/Dockerfile.ubuntu-prod
+++ b/tools/docker/Dockerfile.ubuntu-prod
@ -19,12 +19,8 @@ FROM ubuntu:20.04
 # ARG in fact change the env vars during the build process
 # ENV persist the env vars for the built image as well.
 ARG QEMU_CPU
-ARG ORG_NAME=dragonflydb
 ARG DEBIAN_FRONTEND=noninteractive

-LABEL org.opencontainers.image.title Dragonfly
-LABEL org.opencontainers.image.source https://github.com/${ORG_NAME}/dragonfly
-
 RUN apt clean && apt update && apt -y install netcat-openbsd


--- a/tools/release.sh
+++ b/tools/release.sh
@ -2,22 +2,18 @@

 set -e

-if ! [ -d helio ]; then
-   echo "could not find helio"
+if ! [ -f "helio/blaze.sh" ]; then
+   echo "ERROR"
+   echo "Could not find helio. Please only run this script from repo root."
+   echo "If you are already on the repo root, you might've cloned without submodules."
+   echo "Try running 'git submodule update --init --recursive'"
   exit 1
 fi

-ARCH=`uname -m`
-NAME="dragonfly-${ARCH}"
-
 pwd
-./helio/blaze.sh -release -DBoost_USE_STATIC_LIBS=ON -DOPENSSL_USE_STATIC_LIBS=ON \
-          -DENABLE_GIT_VERSION=ON -DWITH_UNWIND=OFF -DHELIO_RELEASE_FLAGS="-flto"

-cd build-opt 
-ninja dragonfly && ldd dragonfly
-./dragonfly --version
-mv dragonfly $NAME
-tar cvfz $NAME.unstripped.tar.gz $NAME ../LICENSE.md
-strip $NAME
-tar cvfz $NAME.tar.gz $NAME ../LICENSE.md
+make HELIO_RELEASE=y release
+
+build-opt/dragonfly --version
+
+make package