From c45624d6b86e7f964ad23a093cfdb5fc80986eb7 Mon Sep 17 00:00:00 2001 From: Florian Forster Date: Fri, 23 Apr 2021 09:40:13 +0200 Subject: [PATCH] chore(pipeline): change caching keys (#1651) * Update zitadel.yml * use different files for hashing the keys * use correct path for docker and hash the dockerfile * use codecov within steps * proper indenting * rename steps * fix duplicate name * move dockerignore * rename things * codecov as pre step before the build * ignore cache path * debug * Update .dockerignore * debug * debug * debug * debug * debug Co-authored-by: Livio Amstutz --- build/.dockerignore => .dockerignore | 8 ++- .github/workflows/codecov.yml | 61 ------------------- .github/workflows/zitadel.yml | 91 ++++++++++++++++++++-------- build/dockerfile | 3 - 4 files changed, 72 insertions(+), 91 deletions(-) rename build/.dockerignore => .dockerignore (77%) delete mode 100644 .github/workflows/codecov.yml diff --git a/build/.dockerignore b/.dockerignore similarity index 77% rename from build/.dockerignore rename to .dockerignore index 94f47b8baf..2f83c6aae7 100644 --- a/build/.dockerignore +++ b/.dockerignore @@ -1,13 +1,15 @@ .git .codecov .github -build/dockerfile -site +.gitignore +.dockerignore +k8s +docs console/node_modules console/src/app/proto/generated console/tmp .releaserc.js -.typo-ci.yml +changelog.config.js CONTRIBUTING.md LICENSE README.md diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml deleted file mode 100644 index 2d61527b68..0000000000 --- a/.github/workflows/codecov.yml +++ /dev/null @@ -1,61 +0,0 @@ -name: Code Coverage -on: - push: - branches: - - '**' - tags-ignore: - - '**' - -env: - REGISTRY: ghcr.io - NODE_VERSION: '12' - GO_VERSION: '1.15' - -jobs: - container: - runs-on: ubuntu-18.04 - steps: - - name: Source checkout - uses: actions/checkout@v2 - - name: Cache Docker layers - uses: actions/cache@v2 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - - name: Cache Docker layers - uses: actions/cache@v2 - with: - path: /tmp/.buildx-cache-op - key: ${{ runner.os }}-buildx-op-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx-op- - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - uses: docker/build-push-action@v2 - with: - context: . - file: ./build/dockerfile - platforms: linux/amd64 - tags: ${{ env.REGISTRY }}/${{ github.repository }}:coverage - push: false - cache-from: type=local,src=/tmp/.buildx-cache - target: go-codecov - outputs: type=local,dest=/tmp/zitadel - - uses: docker/build-push-action@v2 - with: - context: . - file: ./build/operator/Dockerfile - platforms: linux/amd64 - tags: ${{ env.REGISTRY }}/${{ github.repository }}:coverage - push: false - cache-from: type=local,src=/tmp/.buildx-cache-op - target: go-codecov - outputs: type=local,dest=/tmp/operator - - uses: codecov/codecov-action@v1 - with: - files: /tmp/zitadel/profile.cov,/tmp/operator/profile.cov - name: codecov-go \ No newline at end of file diff --git a/.github/workflows/zitadel.yml b/.github/workflows/zitadel.yml index 5b6b3d834f..6fb53f3b43 100644 --- a/.github/workflows/zitadel.yml +++ b/.github/workflows/zitadel.yml @@ -1,4 +1,4 @@ -name: Zitadel Release +name: ZITADEL Release on: push: branches: @@ -17,6 +17,7 @@ env: jobs: refs: + name: Prepare CI Vars runs-on: ubuntu-18.04 outputs: sha_short: ${{ steps.refs.outputs.sha_short }} @@ -54,6 +55,7 @@ jobs: echo "::set-output name=version::${VERSION}" zitadel-image: + name: Build ZITADEL needs: refs runs-on: ubuntu-18.04 steps: @@ -63,11 +65,9 @@ jobs: uses: actions/cache@v2 with: path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} + key: ${{ runner.os }}-zitadel-${{ hashFiles('**/go.sum', 'console/package-lock.json', 'build/dockerfile') }} restore-keys: | - ${{ runner.os }}-buildx- - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + ${{ runner.os }}-zitadel- - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 - name: Login to DockerHub @@ -76,6 +76,7 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.CR_PAT }} registry: ${{ env.REGISTRY }} + - run: ls -la - uses: docker/build-push-action@v2 with: context: . @@ -84,12 +85,34 @@ jobs: tags: ${{ env.REGISTRY }}/${{ github.repository }}:${{ needs.refs.outputs.sha_short }},${{ env.REGISTRY }}/${{ github.repository }}:${{ needs.refs.outputs.short_ref }} push: true cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,mode=max,dest=/tmp/.buildx-cache + cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-new + - uses: docker/build-push-action@v2 + with: + context: . + file: ./build/dockerfile + platforms: linux/amd64 + tags: ${{ env.REGISTRY }}/${{ github.repository }}:coverage + push: false + cache-from: type=local,src=/tmp/.buildx-cache-new + target: go-codecov + outputs: type=local,dest=/tmp/zitadel + - uses: codecov/codecov-action@v1 + with: + files: /tmp/zitadel/profile.cov + name: codecov-go + - + # Temp fix + # https://github.com/docker/build-push-action/issues/252 + # https://github.com/moby/buildkit/issues/1896 + name: Move cache + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache operator-image: needs: refs runs-on: ubuntu-18.04 - name: Build ${{ matrix.goos }}-${{ matrix.goarch }} + name: Build ZITADEL Operator ${{ matrix.goos }}-${{ matrix.goarch }} strategy: matrix: goos: [ 'linux', 'darwin', 'windows' ] @@ -101,9 +124,9 @@ jobs: uses: actions/cache@v2 with: path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-op-${{ github.sha }} + key: ${{ matrix.goos }}-${{ matrix.goarch }}-operator-image-${{ hashFiles('**/go.sum', 'build/operator/Dockerfile') }} restore-keys: | - ${{ runner.os }}-buildx-op- + ${{ matrix.goos }}-${{ matrix.goarch }}-operator-image- - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx @@ -123,7 +146,7 @@ jobs: tags: ${{ env.REGISTRY }}/${{ env.OPERATOR_IMAGE_NAME }}:${{ needs.refs.outputs.sha_short }},${{ env.REGISTRY }}/${{ env.OPERATOR_IMAGE_NAME }}:${{ needs.refs.outputs.short_ref }} push: false cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,mode=max,dest=/tmp/.buildx-cache + cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-new outputs: type=local,dest=/tmp/operator build-args: | OS=${{ matrix.goos }} @@ -147,17 +170,36 @@ jobs: platforms: linux/amd64 tags: ${{ env.REGISTRY }}/${{ env.OPERATOR_IMAGE_NAME }}:${{ needs.refs.outputs.sha_short }},${{ env.REGISTRY }}/${{ env.OPERATOR_IMAGE_NAME }}:${{ needs.refs.outputs.short_ref }} push: true - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,mode=max,dest=/tmp/.buildx-cache + cache-from: type=local,src=/tmp/.buildx-cache-new build-args: | OS=${{ matrix.goos }} ARCH=${{ matrix.goarch }} VERSION=${{ needs.refs.outputs.version }} - GITHUBOAUTHCLIENTID=${{ secrets.GITHUBOAUTHCLIENTID }} - GITHUBOAUTHCLIENTSECRET=${{ secrets.GITHUBOAUTHCLIENTSECRET }} - + - uses: docker/build-push-action@v2 + with: + context: . + file: ./build/operator/Dockerfile + platforms: linux/amd64 + tags: ${{ env.REGISTRY }}/${{ github.repository }}:coverage + push: false + cache-from: type=local,src=/tmp/.buildx-cache-new + target: go-codecov + outputs: type=local,dest=/tmp/operator + - uses: codecov/codecov-action@v1 + with: + files: /tmp/operator/profile.cov + name: codecov-go + - + # Temp fix + # https://github.com/docker/build-push-action/issues/252 + # https://github.com/moby/buildkit/issues/1896 + name: Move cache + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache crdb-image: + name: Build CockroachDB Image needs: refs runs-on: ubuntu-18.04 steps: @@ -167,9 +209,9 @@ jobs: uses: actions/cache@v2 with: path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} + key: ${{ runner.os }}-crdb-image-${{ hashFiles('build/cr-backup/Dockerfile') }} restore-keys: | - ${{ runner.os }}-buildx- + ${{ runner.os }}-crdb-image- - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx @@ -192,6 +234,7 @@ jobs: cache-to: type=local,mode=max,dest=/tmp/.buildx-cache release: + name: Semantic Release Images and Artifacts runs-on: ubuntu-18.04 needs: [ refs, zitadel-image, operator-image, crdb-image ] env: @@ -202,13 +245,13 @@ jobs: uses: actions/checkout@v2 - name: Docker Login run: docker login $REGISTRY -u $GITHUB_ACTOR -p $GITHUB_TOKEN - - name: Docker Pull short-sha + - name: Docker Pull ZITADEL Image run: docker pull $REGISTRY/$GITHUB_REPOSITORY:${{ needs.refs.outputs.sha_short }} - - name: Docker Pull short-sha + - name: Docker Pull ZITADEL Operator Image run: docker pull $REGISTRY/$OPERATOR_IMAGE_NAME:${{ needs.refs.outputs.sha_short }} - - name: Docker Pull short-sha + - name: Docker Pull CockroachDB Image run: docker pull $REGISTRY/$CRDB_IMAGE_NAME:${{ needs.refs.outputs.sha_short }} - - name: Download all zitadelctl artifact + - name: Download zitadelctl Artifacts uses: actions/download-artifact@v2 with: path: ${{ env.ARTIFACTS_FOLDER }}/ @@ -217,13 +260,13 @@ jobs: mv ${ARTIFACTS_FOLDER}/zitadelctl-windows-amd64/zitadelctl-windows-amd64 ${ARTIFACTS_FOLDER}/zitadelctl-windows-amd64/zitadelctl-windows-amd64.exe find ${ARTIFACTS_FOLDER} - - name: Semantic Release + - name: Run Semantic Release id: semantic uses: cycjimmy/semantic-release-action@v2 with: dry_run: false semantic_version: 17.0.4 - - name: Do something when a new release published + - name: Echo Semantic Release Versions if: steps.semantic.outputs.new_release_published == 'true' run: | echo ${{ steps.semantic.outputs.new_release_version }} @@ -267,7 +310,7 @@ jobs: docker push $REGISTRY/$OPERATOR_IMAGE_NAME:${{ needs.refs.outputs.short_ref }} docker push $REGISTRY/$CRDB_IMAGE_NAME:${{ needs.refs.outputs.short_ref }} if: steps.semantic.outputs.new_release_published != 'true' && needs.refs.outputs.short_ref != 'main' && needs.refs.outputs.short_ref != '' - - name: Dev-Release + - name: Development Release id: create_release uses: ncipollo/release-action@v1.8.1 if: steps.semantic.outputs.new_release_published != 'true' && needs.refs.outputs.short_ref != 'main' && needs.refs.outputs.short_ref != '' diff --git a/build/dockerfile b/build/dockerfile index 65828df458..100fd99aa9 100644 --- a/build/dockerfile +++ b/build/dockerfile @@ -117,7 +117,6 @@ FROM go-gen as go-base # copy all zitadel files COPY . . - ####################### ## copy for local dev ####################### @@ -133,7 +132,6 @@ COPY --from=go-gen /go/src/github.com/caos/zitadel/docs/apis/proto ./docs/docs/a ## Go test ####################### FROM go-base as go-test -COPY . . # Migrations for cockroach-secure RUN go install github.com/rakyll/statik @@ -141,7 +139,6 @@ RUN ./build/operator/prebuild.sh ./migrations RUN go test -race -v -coverprofile=profile.cov $(go list ./... | grep -v /operator/) - ####################### ## Go test results #######################