mirrors/zitadel
1
0
Fork 0
mirror of https://github.com/zitadel/zitadel synced 2024-11-23 03:03:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore(ci): release pipeline for v2 (#3197)

Browse Source 
* not working

* test ci

* correct path

* test

* test

* test

* remove k8s & test

* test

* test image

* typo

* test

* fix

* test

* fix

* test

* fix

* fix windows

* fix

* fix

* fix codecov

* fix with arch

* test

* test

* test

* test

* trial

* trial

* trial

* trial

* trial

* trial

* trial

* try

* try

* first improvement

* trial

* improve

* improve

* improve

* use bash

* trial

* random test

* wip

* test with goreleaser and semrel

* fix typo

* trial

* add buildx

* define default

* force buildkit

* use buildx

* debug GH docker

* use images

* debug docker

* work around image name issues

* trial

* test

* test with prebuilt image

* fix

* use load

* test

* use only docker

* trial

* test

* use load

* test

* test with docker driver

* trial

* trial

* trial

* use arm

* upload artifacts

* try to cache grpc base

* try to use a cache

* fall back

* test

* fix

* improve

* upload artifacts

* fix

* name things

* add tag if poc

* args for protos

* remove v2 build

Co-authored-by: Livio Amstutz <livio.a@gmail.com>
This commit is contained in:
Florian Forster 2022-02-14 15:35:16 +01:00 committed by GitHub
parent 2d208dd8ba
commit 2f3a482ade
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
27 changed files with 217 additions and 17490 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.dockerignore
View File

@ -16,4 +16,6 @@ LICENSE
README.md
SECURITY.md
pkg/grpc/*/*.pb.*
pkg/grpc/*/*.swagger.json
pkg/grpc/*/*.swagger.json
.goreleaser.yaml
.artifacts/

281
.github/workflows/zitadel.yml vendored
View File

@ -1,27 +1,15 @@
name: ZITADEL Release
on:
push:
branches:
- main
tags-ignore:
- '**'
pull_request:
branches:
- '**'
workflow_dispatch:
env:
REGISTRY: ghcr.io
ARTIFACTS_FOLDER: './artifacts'
on:
#pull_request:
push:
permissions:
contents: write
jobs:
refs:
name: Prepare CI Vars
runs-on: ubuntu-18.04
outputs:
sha_short: ${{ steps.refs.outputs.sha_short }}
short_ref: ${{ steps.refs.outputs.short_ref }}
version: ${{ steps.refs.outputs.version }}
Build:
runs-on: ubuntu-20.04
steps:
- name: Source checkout
uses: actions/checkout@v2
@ -30,224 +18,55 @@ jobs:
uses: cycjimmy/semantic-release-action@v2
with:
dry_run: true
semantic_version: 17.0.4
semantic_version: 19.0.2
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Set outputs
id: refs
run: |
export BRANCH=${GITHUB_REF#refs/*/}
echo "branch: ${BRANCH}"
export BRANCHTRANSLATED=$(echo ${BRANCH} | tr '/' '_')
echo "short_ref: ${BRANCHTRANSLATED}"
export VERSION=""
if ${{ steps.semantic.outputs.new_release_published == 'true' }}; then
export VERSION=${{ steps.semantic.outputs.new_release_version }}
else
export VERSION=${BRANCHTRANSLATED}
fi
echo "New semantic release: ${{ steps.semantic.outputs.new_release_published }}"
echo "version: ${VERSION}"
echo "::set-output name=short_ref::${BRANCHTRANSLATED}"
echo "::set-output name=sha_short::SHA-$(git rev-parse --short=12 HEAD)"
echo "::set-output name=version::${VERSION}"
zitadel:
name: Build ZITADEL
needs: refs
runs-on: ubuntu-18.04
steps:
- name: Source checkout
uses: actions/checkout@v2
- name: Set up Go
uses: actions/setup-go@v2
with:
go-version: 1.17
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
- uses: docker/build-push-action@v2
with:
context: .
file: ./build/zitadel/Dockerfile
platforms: linux/amd64
cache-from: type=gha,scope=${{ github.workflow }}
cache-to: type=gha,scope=${{ github.workflow }},mode=max
outputs: type=local,dest=/tmp/zitadel
build-args: |
VERSION=${{ needs.refs.outputs.version }}
driver: docker
install: true
- name: Run GoReleaser Snapshot
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
uses: goreleaser/goreleaser-action@v2
with:
distribution: goreleaser
version: v1.5.0
args: release --snapshot
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Run GoReleaser Release on tag
if: ${{ startsWith(github.ref, 'refs/tags/') }}
uses: goreleaser/goreleaser-action@v2
with:
distribution: goreleaser
version: v1.5.0
args: release
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- uses: actions/upload-artifact@v2
name: Upload goreleaser dist
with:
name: zitadel
path: /tmp/zitadel/
zitadel-codecov:
name: Upload ZITADEL Codecov
needs: [refs, zitadel]
runs-on: ubuntu-18.04
steps:
- name: Source checkout
uses: actions/checkout@v2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
- uses: docker/build-push-action@v2
name: goreleaser-dist
path: .artifacts/goreleaser
- uses: actions/upload-artifact@v2
name: Upload gRPC clients
with:
context: .
file: ./build/zitadel/Dockerfile
platforms: linux/amd64
target: go-codecov
cache-from: type=gha,scope=${{ github.workflow }}
outputs: type=local,dest=/tmp/zitadel
name: grpc-clients
path: .artifacts/grpc
- uses: actions/upload-artifact@v2
name: Upload compiled console
with:
name: console
path: .artifacts/console
- uses: codecov/codecov-action@v1
name: Upload go coverage
with:
name: zitadel-codecov
files: /tmp/zitadel/profile.cov
zitadel-console:
name: Build ZITADEL Console
needs: refs
runs-on: ubuntu-18.04
steps:
- name: Source checkout
uses: actions/checkout@v2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
- uses: docker/build-push-action@v2
with:
context: .
file: ./build/console/Dockerfile
platforms: linux/amd64
cache-from: type=gha,scope=${{ github.workflow }}
cache-to: type=gha,scope=${{ github.workflow }},mode=max
outputs: type=local,dest=/tmp/zitadel-console
- uses: actions/upload-artifact@v2
with:
name: zitadel-console
path: /tmp/zitadel-console/
zitadel-image:
name: Package ZITADEL Image
needs: [refs, zitadel, zitadel-console]
runs-on: ubuntu-18.04
if: ${{ github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/main' }}
steps:
- name: Source checkout
uses: actions/checkout@v2
- uses: actions/download-artifact@v2
with:
name: zitadel
path: .download/zitadel
- uses: actions/download-artifact@v2
with:
name: zitadel-console
path: .download/zitadel/console
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
- name: Login to DockerHub
uses: docker/login-action@v1
with:
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
registry: ${{ env.REGISTRY }}
- uses: docker/build-push-action@v2
with:
context: .
file: ./build/Dockerfile
platforms: linux/amd64
tags: ${{ env.REGISTRY }}/${{ github.repository }}:${{ needs.refs.outputs.sha_short }},${{ env.REGISTRY }}/${{ github.repository }}:${{ needs.refs.outputs.short_ref }}
push: true
cache-from: type=gha,scope=${{ github.workflow }}
cache-to: type=gha,scope=${{ github.workflow }},mode=max
release:
name: Semantic Release Images and Artifacts
runs-on: ubuntu-18.04
needs: [ refs, zitadel-image ]
if: ${{ github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/main' }}
env:
DOCKER_USERNAME: ${{ github.repository_owner }}
DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
steps:
- name: Source checkout
uses: actions/checkout@v2
- name: Login to DockerHub
uses: docker/login-action@v1
with:
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
registry: ${{ env.REGISTRY }}
- name: Docker Pull ZITADEL Image
run: docker pull $REGISTRY/$GITHUB_REPOSITORY:${{ needs.refs.outputs.sha_short }}
- name: Download zitadelctl Artifacts
uses: actions/download-artifact@v2
with:
path: ${{ env.ARTIFACTS_FOLDER }}/
- name: Workaround for exe
run: |
mv ${ARTIFACTS_FOLDER}/zitadelctl-windows-amd64/zitadelctl-windows-amd64 ${ARTIFACTS_FOLDER}/zitadelctl-windows-amd64/zitadelctl-windows-amd64.exe
find ${ARTIFACTS_FOLDER}
- name: Run Semantic Release
id: semantic
uses: cycjimmy/semantic-release-action@v2
with:
dry_run: false
semantic_version: 17.0.4
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Echo Semantic Release Versions
if: steps.semantic.outputs.new_release_published == 'true'
run: |
echo ${{ steps.semantic.outputs.new_release_version }}
echo ${{ steps.semantic.outputs.new_release_major_version }}
echo ${{ steps.semantic.outputs.new_release_minor_version }}
echo ${{ steps.semantic.outputs.new_release_patch_version }}
- name: Docker Tag Version
run: |
docker tag $REGISTRY/$GITHUB_REPOSITORY:${{ needs.refs.outputs.sha_short }} $REGISTRY/$GITHUB_REPOSITORY:${{ steps.semantic.outputs.new_release_version }}
if: steps.semantic.outputs.new_release_published == 'true'
- name: Docker Tag Latest
run: |
docker tag $REGISTRY/$GITHUB_REPOSITORY:${{ needs.refs.outputs.sha_short }} $REGISTRY/$GITHUB_REPOSITORY:latest
if: steps.semantic.outputs.new_release_published == 'true'
- name: Docker Push Version
run: |
docker push $REGISTRY/$GITHUB_REPOSITORY:${{ steps.semantic.outputs.new_release_version }}
if: steps.semantic.outputs.new_release_published == 'true'
- name: Docker Push Latest
run: |
docker push $REGISTRY/$GITHUB_REPOSITORY:latest
if: steps.semantic.outputs.new_release_published == 'true'
- name: Docker Tag Version
run: |
docker tag $REGISTRY/$GITHUB_REPOSITORY:${{ needs.refs.outputs.sha_short }} $REGISTRY/$GITHUB_REPOSITORY:${{ needs.refs.outputs.short_ref }}
if: steps.semantic.outputs.new_release_published != 'true' && needs.refs.outputs.short_ref != 'main' && needs.refs.outputs.short_ref != ''
- name: Docker Push Version
run: |
docker push $REGISTRY/$GITHUB_REPOSITORY:${{ needs.refs.outputs.short_ref }}
if: steps.semantic.outputs.new_release_published != 'true' && needs.refs.outputs.short_ref != 'main' && needs.refs.outputs.short_ref != ''
- name: Development Release
id: create_release
uses: ncipollo/release-action@v1.8.4
if: steps.semantic.outputs.new_release_published != 'true' && needs.refs.outputs.short_ref != 'main' && needs.refs.outputs.short_ref != ''
with:
artifacts: "${{ env.ARTIFACTS_FOLDER }}/zitadelctl-darwin-amd64/zitadelctl-darwin-amd64,${{ env.ARTIFACTS_FOLDER }}/zitadelctl-linux-amd64/zitadelctl-linux-amd64,${{ env.ARTIFACTS_FOLDER }}/zitadelctl-windows-amd64/zitadelctl-windows-amd64.exe"
body: |
This is a release from a development branch.
Do not use these artifacts in production.
tag: ${{ needs.refs.outputs.short_ref }}-dev
commit: ${{ needs.refs.outputs.short_ref }}
name: Branch ${{ needs.refs.outputs.short_ref }}
token: ${{ secrets.GITHUB_TOKEN }}
replacesArtifacts: true
prerelease: true
draft: false
allowUpdates: true
- name: Create Sentry release
if: ${{ github.ref == 'refs/heads/main' }}
uses: getsentry/action-release@v1
env:
SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
SENTRY_ORG: ${{ secrets.SENTRY_ORG }}
SENTRY_PROJECT: ${{ secrets.SENTRY_PROJECT }}
with:
version: zitadel-${{ needs.refs.outputs.version }}
projects: "console zitadel zitadelctl"
files: .artifacts/codecov/profile.cov
name: go-codecov

2
.gitignore vendored
View File

@ -59,3 +59,5 @@ openapi/**/*.json
build/local/cloud.env
migrations/cockroach/migrate_cloud.go
.notifications
.artifacts

55
.goreleaser.yaml Normal file
View File

@ -0,0 +1,55 @@
project_name: zitadel
release:
github:
owner: caos
name: zitadel
draft: true
prerelease: auto
before:
hooks:
- docker build -f build/grpc/Dockerfile -t zitadel-base:local .
- docker build -f build/zitadel/Dockerfile . -t zitadel-go-test --target go-codecov -o .artifacts/codecov
- docker build -f build/zitadel/Dockerfile . -t zitadel-go-base --target go-copy -o .artifacts/grpc/go-client
- docker build -f build/console/Dockerfile . -t zitadel-npm-base --target npm-copy -o .artifacts/grpc/js-client
- docker build -f build/console/Dockerfile . -t zitadel-npm-base --target angular-export -o .artifacts/console
builds:
- env:
- CGO_ENABLED=0
goos:
- linux
- windows
- darwin
goarch:
- amd64
- arm64
dist: .artifacts/goreleaser
dockers:
- image_templates:
- "caos/zitadel:{{ .Version }}-amd64"
use: buildx
dockerfile: build/Dockerfile
build_flag_templates:
- "--platform=linux/amd64"
- image_templates:
- "caos/zitadel:{{ .Version }}-arm64"
use: buildx
dockerfile: build/Dockerfile
build_flag_templates:
- "--platform=linux/arm64"
archives:
- format: binary
gomod:
proxy: true
env:
- GOPROXY=https://proxy.golang.org,direct
- GOSUMDB=sum.golang.org
checksum:
name_template: 'checksums.txt'
snapshot:
name_template: "{{ incpatch .Version }}-next"
changelog:
sort: asc
filters:
exclude:
- '^docs:'
- '^test:'

24
.releaserc.js
View File

@ -1,23 +1,9 @@
module.exports = {
branches: ["main"],
branches: [
{name: 'main'},
{name: '1.x.x', range: '1.x.x', channel: '1.x.x'},
],
plugins: [
"@semantic-release/commit-analyzer",
"@semantic-release/release-notes-generator",
["@semantic-release/github", {
"assets": [
{
"path": "./artifacts/zitadelctl-darwin-amd64/zitadelctl-darwin-amd64",
"label": "Zitadelctl Darwin x86_64"
},
{
"path": "./artifacts/zitadelctl-linux-amd64/zitadelctl-linux-amd64",
"label": "Zitadelctl Linux x86_64"
},
{
"path": "./artifacts/zitadelctl-windows-amd64/zitadelctl-windows-amd64.exe",
"label": "Zitadelctl Windows x86_64"
}
]
}],
"@semantic-release/commit-analyzer"
]
};

6
build/dockerfile → build/Dockerfile
View File

@ -2,12 +2,8 @@
## Final Production Image
#######################
FROM alpine:3 as artifact
ARG BUILDARCH
COPY zitadel /app/zitadel
RUN adduser -D zitadel
COPY cmd/zitadel/*.yaml /app/
COPY .download/zitadel/zitadel-linux-${BUILDARCH} /app/zitadel
COPY .download/zitadel/console /app/console/
RUN chmod a+x /app/zitadel
#######################
## Scratch Image

65
build/console/Dockerfile
View File

@ -1,49 +1,5 @@
ARG NODE_VERSION=14
#######################
## These steps set platform / arch type specific variables
#######################
FROM alpine:3 AS arm64-base
ENV PROTOC_ARCH aarch_64
FROM alpine:3 AS amd64-base
ENV PROTOC_ARCH x86_64
#######################
## This step sets up the folder structure,
## initalices go mods,
## downloads the protofiles,
## protoc and protoc-gen-grpc-web for later use
#######################
FROM ${BUILDARCH}-base AS base
ARG PROTOC_VERSION=3.13.0
ARG PROTOC_ZIP=protoc-${PROTOC_VERSION}-linux-${PROTOC_ARCH}.zip
ARG GRPC_WEB_VERSION=1.3.0
# no arm specific version available and x86 works fine at the moment:
ARG GRPC_WEB=protoc-gen-grpc-web-${GRPC_WEB_VERSION}-linux-x86_64
RUN apk add tar curl
WORKDIR /proto
#protoc
RUN apk add tar curl \
&& curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/$PROTOC_ZIP \
&& unzip -o $PROTOC_ZIP -d /usr/local bin/protoc \
&& unzip -o $PROTOC_ZIP -d /proto include/* \
&& rm -f $PROTOC_ZIP \
&& curl -OL https://github.com/grpc/grpc-web/releases/download/${GRPC_WEB_VERSION}/${GRPC_WEB} \
&& mv ${GRPC_WEB} /usr/local/bin/protoc-gen-grpc-web \
&& chmod +x /usr/local/bin/protoc-gen-grpc-web \
&& curl https://raw.githubusercontent.com/envoyproxy/protoc-gen-validate/v0.6.2/validate/validate.proto --create-dirs -o include/validate/validate.proto \
&& curl https://raw.githubusercontent.com/grpc-ecosystem/grpc-gateway/v2.2.0/protoc-gen-openapiv2/options/annotations.proto --create-dirs -o include/protoc-gen-openapiv2/options/annotations.proto \
&& curl https://raw.githubusercontent.com/grpc-ecosystem/grpc-gateway/v2.2.0/protoc-gen-openapiv2/options/openapiv2.proto --create-dirs -o include/protoc-gen-openapiv2/options/openapiv2.proto \
&& curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/annotations.proto --create-dirs -o include/google/api/annotations.proto \
&& curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/http.proto --create-dirs -o include/google/api/http.proto \
&& curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/field_behavior.proto --create-dirs -o include/google/api/field_behavior.proto
#zitadel protos
COPY proto/ include/.
#######################
## With this step we prepare all node_modules, this helps caching the build
## Speed up this step by mounting your local node_modules directory
@ -52,11 +8,11 @@ FROM node:${NODE_VERSION} as npm-base
WORKDIR /console
COPY console/package.json console/package-lock.json ./
RUN npm install
RUN npm ci
COPY console .
COPY --from=base /proto /proto
COPY --from=base /usr/local/bin /usr/local/bin/.
COPY --from=zitadel-base:local /proto /proto
COPY --from=zitadel-base:local /usr/local/bin /usr/local/bin/.
COPY build/console build/console/
RUN build/console/generate-grpc.sh
@ -64,23 +20,18 @@ RUN build/console/generate-grpc.sh
## copy for local dev
#######################
FROM scratch as npm-copy
COPY --from=npm-base /console/src/app/proto/generated ./console/src/app/proto/generated
#######################
## angular dev build
#######################
FROM npm-base as dev-angular-build
RUN npm install -g @angular/cli
COPY --from=npm-base /console/src/app/proto/generated .
#######################
## angular lint workspace and prod build
#######################
FROM npm-base as prod-angular-build
FROM npm-base as angular-build
RUN npm run lint
RUN npm run prodbuild
RUN ls -la /console/dist/console
#######################
## Only Copy Assets
#######################
FROM scratch as prod-angular-export
COPY --from=prod-angular-build /console/dist/console .
FROM scratch as angular-export
COPY --from=angular-build /console/dist/console .

2
build/console/generate-grpc.sh
View File

@ -1,4 +1,4 @@
#! /bin/sh
#!/bin/sh
set -eux

47
build/grpc/Dockerfile Normal file
View File

@ -0,0 +1,47 @@
#ARG BUILDARCH=x86_64
#######################
## These steps set platform / arch type specific variables
#######################
FROM alpine:3 AS arm64-base
ENV PROTOC_ARCH aarch_64
FROM alpine:3 AS amd64-base
ENV PROTOC_ARCH x86_64
#######################
## This step sets up the folder structure,
## initalices go mods,
## downloads the protofiles,
## protoc and protoc-gen-grpc-web for later use
#######################
FROM ${BUILDARCH}-base
ARG PROTOC_VERSION=3.18.0
ARG PROTOC_ZIP=protoc-${PROTOC_VERSION}-linux-${PROTOC_ARCH}.zip
ARG GRPC_WEB_VERSION=1.3.0
ARG GATEWAY_VERSION=2.6.0
ARG VALIDATOR_VERSION=0.6.2
# no arm specific version available and x86 works fine at the moment:
ARG GRPC_WEB=protoc-gen-grpc-web-${GRPC_WEB_VERSION}-linux-x86_64
RUN apk add tar curl
WORKDIR /proto
#protoc
RUN apk add tar curl \
&& curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/$PROTOC_ZIP \
&& unzip -o $PROTOC_ZIP -d /usr/local bin/protoc \
&& unzip -o $PROTOC_ZIP -d /proto include/* \
&& rm -f $PROTOC_ZIP \
&& curl -OL https://github.com/grpc/grpc-web/releases/download/${GRPC_WEB_VERSION}/${GRPC_WEB} \
&& mv ${GRPC_WEB} /usr/local/bin/protoc-gen-grpc-web \
&& chmod +x /usr/local/bin/protoc-gen-grpc-web \
&& curl https://raw.githubusercontent.com/envoyproxy/protoc-gen-validate/v${VALIDATOR_VERSION}/validate/validate.proto --create-dirs -o include/validate/validate.proto \
&& curl https://raw.githubusercontent.com/grpc-ecosystem/grpc-gateway/v${GATEWAY_VERSION}/protoc-gen-openapiv2/options/annotations.proto --create-dirs -o include/protoc-gen-openapiv2/options/annotations.proto \
&& curl https://raw.githubusercontent.com/grpc-ecosystem/grpc-gateway/v${GATEWAY_VERSION}/protoc-gen-openapiv2/options/openapiv2.proto --create-dirs -o include/protoc-gen-openapiv2/options/openapiv2.proto \
&& curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/annotations.proto --create-dirs -o include/google/api/annotations.proto \
&& curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/http.proto --create-dirs -o include/google/api/http.proto \
&& curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/field_behavior.proto --create-dirs -o include/google/api/field_behavior.proto
#zitadel protos
COPY proto/ include/.

11
build/local/Dockerfile.clientid
View File

@ -1,11 +0,0 @@
FROM ubuntu:latest AS client-id
#install dependencies
RUN apt-get update \
&& apt-get install curl -y \
&& apt-get install jq -y
#prepare script
COPY build/local/clientid.sh clientid.sh
RUN chmod +x /clientid.sh
ENTRYPOINT [ "/clientid.sh" ]

33
build/local/Dockerfile.gateway
View File

@ -1,33 +0,0 @@
# copy from https://raw.githubusercontent.com/grpc/grpc-web/master/net/grpc/gateway/docker/grpcwebproxy/Dockerfile
FROM golang:1.17-alpine3.13
RUN apk add --no-cache curl git ca-certificates && \
rm -rf /var/lib/apt/lists/*
ARG VERSION=0.14.0
WORKDIR /tmp
RUN curl -sS https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
RUN wget https://github.com/improbable-eng/grpc-web/archive/v$VERSION.tar.gz
WORKDIR /go/src/github.com/improbable-eng/
RUN tar -zxf /tmp/v$VERSION.tar.gz -C .
RUN mv grpc-web-$VERSION grpc-web
WORKDIR /go/src/github.com/improbable-eng/grpc-web
RUN dep ensure && \
go env -w GO111MODULE=auto && \
go install ./go/grpcwebproxy
ENV BKD_HOST=backend-run
ENV BKD_PORT=50001
ENTRYPOINT [ "/bin/sh", "-c", "exec /go/bin/grpcwebproxy \
--backend_addr=${BKD_HOST}:${BKD_PORT} \
--run_tls_server=false \
--use_websockets \
--allow_all_origins " ]

7
build/local/Dockerfile.keys
View File

@ -1,7 +0,0 @@
FROM alpine:latest AS gen-keys
COPY build/local/keys.sh keys.sh
RUN chmod +x /keys.sh
ENTRYPOINT [ "/keys.sh" ]
FROM scratch AS copy-keys
COPY --from=gen-keys /.keys /.keys

18
build/local/clientid.sh
View File

@ -1,18 +0,0 @@
#!/bin/bash
# ------------------------------
# sets the client id in environment.json
# ------------------------------
clientid=""
while [ -z $clientid ]; do
echo "no from zitadel ==> retrying in 5 seconds"
sleep 5
clientid=$(curl -s http://${HOST}:${PORT}/clientID)
if [[ "$clientid" != *@zitadel* ]]; then
echo "invalid response from zitadel ==> retrying in 5 seconds"
clientid=""
fi
done
echo "$(jq ".clientid = $clientid" /environment.json)" > environment.json

180
build/local/docker-compose-local.yml
View File

@ -2,7 +2,6 @@ version: '3.8'
services:
db:
profiles: ['backend', 'storage']
restart: always
networks:
- zitadel
@ -18,106 +17,11 @@ services:
- 8080:8080
- 26257:26257
# schema changes on the database
db-migrations:
profiles: ['backend', 'storage']
restart: on-failure
networks:
- zitadel
depends_on:
db:
condition: service_healthy
image: flyway/flyway:latest
volumes:
- ../../migrations/cockroach:/flyway/sql
environment:
- FLYWAY_PLACEHOLDERS_eventstorepassword=NULL
- FLYWAY_PLACEHOLDERS_managementpassword=NULL
- FLYWAY_PLACEHOLDERS_adminapipassword=NULL
- FLYWAY_PLACEHOLDERS_authpassword=NULL
- FLYWAY_PLACEHOLDERS_notificationpassword=NULL
- FLYWAY_PLACEHOLDERS_authzpassword=NULL
- FLYWAY_PLACEHOLDERS_queriespassword=NULL
command: -url=jdbc:postgresql://db:26257/defaultdb -user=root -password= -connectRetries=5 migrate
# minio is used to store assets
minio:
profiles: ['backend', 'storage']
image: minio/minio:RELEASE.2021-06-14T01-29-23Z
restart: on-failure
networks:
- zitadel
environment:
- MINIO_ACCESS_KEY=access_key
- MINIO_SECRET_KEY=secret_key
healthcheck:
test: ['CMD', 'curl', '-f', 'http://localhost:9000/minio/health/live']
interval: 30s
timeout: 5s
retries: 3
start_period: 5s
command:
- gateway
- nas
- /export
# ZITADEL needs several keys to encrypt data
# this container generates the required keys
# and stores them into zitadel/.keys
keys:
profiles: ['backend', 'backend-stub']
restart: on-failure
networks:
- zitadel
build:
context: ../..
dockerfile: build/local/Dockerfile.keys
target: gen-keys
volumes:
- ../../.:/zitadel
env_file:
- ./local.env
# To interact with ZITADEL requires some data setted up.
# Due to the evolution of ZITADEL it's required to add additional
# setup steps, because of this fact it's recommended to rerun the setup
# on each restart, at least after a new version got released
backend-setup:
profiles: ['backend']
restart: on-failure
networks:
- zitadel
depends_on:
db-migrations:
condition: service_completed_successfully
keys:
condition: service_completed_successfully
build:
context: ../..
dockerfile: build/zitadel/Dockerfile
target: dev-go-build
args:
ENV: dev
volumes:
- ../../.keys:/go/src/github.com/caos/zitadel/.keys
env_file:
- ./local.env
environment:
- ZITADEL_EVENTSTORE_HOST=db
command:
[
'-setup-files=cmd/zitadel/setup.yaml',
'-setup-files=cmd/zitadel/system-defaults.yaml',
'-setup-files=cmd/zitadel/authz.yaml',
'setup',
]
# starts the backend (API's) of ZITADEL
# Port 50001 serves the GRPC API
# Port 50002 serves the REST API
# Port 50003 serves the login GUI
backend-run:
profiles: ['backend']
restart: on-failure
networks:
- zitadel
@ -130,99 +34,15 @@ services:
depends_on:
db:
condition: service_healthy
minio:
condition: service_healthy
backend-setup:
condition: service_completed_successfully
build:
context: ../..
dockerfile: build/zitadel/Dockerfile
target: dev-go-build
args:
ENV: dev
volumes:
- ../../.keys:/go/src/github.com/caos/zitadel/.keys
- ../../.notifications:/go/src/github.com/caos/zitadel/.notifications
env_file:
- ./local.env
environment:
- ZITADEL_EVENTSTORE_HOST=db
ports:
- 50001:50001
- 50002:50002
- 50003:50003
command:
[
'-console=false',
'-localDevMode=true',
'-config-files=cmd/zitadel/startup.yaml',
'-config-files=cmd/zitadel/system-defaults.yaml',
'-config-files=cmd/zitadel/authz.yaml',
'start',
]
# the GRPC web gateway proxies the
# GRPC web calls to GRPC
# it's used in console (frontend) for example
grpc-web-gateway:
profiles: ['backend']
restart: on-failure
logging:
driver: none
depends_on:
backend-run:
condition: service_healthy
networks:
- zitadel
build:
context: ../..
dockerfile: build/local/Dockerfile.gateway
image: grpcweb/grpcwebproxy
ports:
- '50000:8080'
# this service generates the environemnt.json
# needed in console.
# It curl's the client id of console and writes it to
# the environment.json
client-id:
profiles: ['frontend', 'console-stub']
depends_on:
backend-run:
condition: service_healthy
networks:
- zitadel
build:
context: ../..
dockerfile: build/local/Dockerfile.clientid
target: client-id
volumes:
- ./environment.json:/environment.json
environment:
- HOST=backend-run
- PORT=50002
# starts console in development mode
frontend-run:
profiles: ['frontend']
networks:
- zitadel
depends_on:
grpc-web-gateway:
condition: service_started
client-id:
condition: service_completed_successfully
build:
context: ../..
dockerfile: build/console/Dockerfile
target: dev-angular-build
args:
ENV: dev
volumes:
- ./environment.json:/console/src/assets/environment.json
command: sh -c "ng serve --host 0.0.0.0 --disable-host-check"
ports:
- 4200:4200
networks:
zitadel: {}

23
build/local/keys.sh
View File

@ -1,23 +0,0 @@
#!/bin/sh
# ----------------------------------------------------------------
# generates necessary ZITADEL keys
# ----------------------------------------------------------------
set -e
KEY_PATH=$(echo "/zitadel/$(dirname ${ZITADEL_KEY_PATH})")
KEY_FILE=${KEY_PATH}/local_keys.yaml
mkdir -p ${KEY_PATH}
if [ ! -f ${KEY_FILE} ]; then
touch ${KEY_FILE}
fi
for key in $(env | grep "ZITADEL_.*_KEY" | cut -d'=' -f2); do
if [ $(grep -L ${key} ${KEY_FILE}) ]; then
echo "create key for ${key} in ${KEY_FILE}"
echo -e "${key}: $(head -c22 /dev/urandom | base64)" >> ${KEY_FILE}
fi
done

67
build/zitadel/Dockerfile
View File

@ -1,44 +1,5 @@
ARG GO_VERSION=1.17
#######################
## These steps set platform / arch type specific variables
#######################
FROM alpine:3 AS arm64-base
ENV PROTOC_ARCH aarch_64
FROM alpine:3 AS amd64-base
ENV PROTOC_ARCH x86_64
#######################
## This step sets up the folder structure,
## initalices go mods,
## downloads the protofiles and protoc for later use
#######################
FROM ${BUILDARCH}-base AS base
ARG PROTOC_VERSION=3.18.0
ARG PROTOC_ZIP=protoc-${PROTOC_VERSION}-linux-${PROTOC_ARCH}.zip
ARG GATEWAY_VERSION=2.6.0
ARG VALIDATOR_VERSION=0.6.2
RUN apk add tar curl
WORKDIR /proto
#protoc
RUN apk add tar curl \
&& curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/$PROTOC_ZIP \
&& unzip -o $PROTOC_ZIP -d /usr/local bin/protoc \
&& unzip -o $PROTOC_ZIP -d /proto include/* \
&& rm -f $PROTOC_ZIP \
&& curl https://raw.githubusercontent.com/envoyproxy/protoc-gen-validate/v${VALIDATOR_VERSION}/validate/validate.proto --create-dirs -o include/validate/validate.proto \
&& curl https://raw.githubusercontent.com/grpc-ecosystem/grpc-gateway/v${GATEWAY_VERSION}/protoc-gen-openapiv2/options/annotations.proto --create-dirs -o include/protoc-gen-openapiv2/options/annotations.proto \
&& curl https://raw.githubusercontent.com/grpc-ecosystem/grpc-gateway/v${GATEWAY_VERSION}/protoc-gen-openapiv2/options/openapiv2.proto --create-dirs -o include/protoc-gen-openapiv2/options/openapiv2.proto \
&& curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/annotations.proto --create-dirs -o include/google/api/annotations.proto \
&& curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/http.proto --create-dirs -o include/google/api/http.proto \
&& curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/field_behavior.proto --create-dirs -o include/google/api/field_behavior.proto
#zitadel protos
COPY proto/ include/.
#######################
## Go dependencies
## Speed up this step by mounting your local go mod pkg directory
@ -48,7 +9,7 @@ RUN mkdir -p src/github.com/caos/zitadel
WORKDIR /go/src/github.com/caos/zitadel
#download modules
COPY ./go.* .
COPY . .
RUN go mod download
# install tools
@ -84,8 +45,8 @@ RUN go generate internal/ui/login/statik/generate.go \
## generates grpc stub
#######################
FROM go-static AS go-stub
COPY --from=base /proto /proto
COPY --from=base /usr/local/bin /usr/local/bin/.
COPY --from=zitadel-base:local /proto /proto
COPY --from=zitadel-base:local /usr/local/bin /usr/local/bin/.
COPY build/zitadel/generate-grpc.sh build/zitadel/generate-grpc.sh
COPY internal/protoc internal/protoc
@ -140,25 +101,3 @@ RUN go install github.com/rakyll/statik \
#######################
FROM scratch as go-codecov
COPY --from=go-test /go/src/github.com/caos/zitadel/profile.cov profile.cov
#######################
## Go prod build
#######################
FROM go-test as prod-go-build
ARG BUILDARCH
ARG VERSION=""
RUN CGO_ENABLED=0 GOOS=linux GOARCH=${BUILDARCH} go build -a -installsuffix cgo -ldflags "-X main.version=${VERSION:-'dev'} -extldflags \"-static\"" -o zitadel-linux-${BUILDARCH} main.go
#######################
## Go dev build
#######################
FROM go-base as dev-go-build
ENTRYPOINT [ "go", "run", "main.go" ]
#######################
## Only Copy Assets
#######################
FROM scratch as prod-go-export
ARG BUILDARCH
COPY --from=prod-go-build /go/src/github.com/caos/zitadel/zitadel-linux-${BUILDARCH} .

2
build/zitadel/generate-grpc.sh
View File

@ -1,4 +1,4 @@
#! /bin/sh
#!/bin/sh
set -eux

16708
console/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

2
console/package.json
View File

@ -5,7 +5,7 @@
"ng": "ng",
"start": "ng serve",
"build": "ng build",
"prodbuild": "ng build --configuration production",
"prodbuild": "ng build --aot=true --buildOptimizer=true",
"lint": "ng lint && stylelint './src/**/*.scss' --syntax scss"
},
"private": true,

4
guides/development.md
View File

@ -24,7 +24,7 @@ This part is relevant if you start the backend or console without docker compose
This command generates the grpc stub for console into the folder console/src/app/proto/generated for local development.
```bash
DOCKER_BUILDKIT=1 docker build -f build/console/Dockerfile . -t zitadel:gen-fe --target npm-copy -o .
DOCKER_BUILDKIT=1 docker build -f build/zitadel/Dockerfile . -t zitadel:gen-fe --target js-client -o .
```
### Start the Backend
@ -33,7 +33,7 @@ With these commands you can generate the stub for the backend.
```bash
# generates grpc stub
DOCKER_BUILDKIT=1 docker build -f build/zitadel/Dockerfile . -t zitadel:gen-be --target go-copy -o .
DOCKER_BUILDKIT=1 docker build -f build/zitadel/Dockerfile . -t zitadel:gen-be --target go-client -o .
# generates keys for cryptography
COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 \
&& docker compose -f ./build/local/docker-compose-local.yml --profile backend-stub up --exit-code-from keys

85
k8s/base/deployment.yaml
View File

@ -1,85 +0,0 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: zitadel
labels:
app: zitadel
spec:
replicas: 3
selector:
matchLabels:
app: zitadel
version: v1
template:
metadata:
labels:
app: zitadel
version: v1
spec:
containers:
- name: zitadel
image: docker.pkg.github.com/caos/zitadel/zitadel:latest
imagePullPolicy: IfNotPresent
args: ["start"]
ports:
- name: grpc
containerPort: 50001
- name: http
containerPort: 50002
- name: ui
containerPort: 50003
readinessProbe:
httpGet:
path: /ready
port: http
periodSeconds: 5
failureThreshold: 2
livenessProbe:
httpGet:
path: /healthz
port: http
periodSeconds: 5
failureThreshold: 2
env:
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: CHAT_URL
valueFrom:
secretKeyRef:
name: zitadel-secrets-vars
key: ZITADEL_GOOGLE_CHAT_URL
- name: TWILIO_TOKEN
valueFrom:
secretKeyRef:
name: zitadel-secrets-vars
key: ZITADEL_TWILIO_AUTH_TOKEN
- name: TWILIO_SERVICE_SID
valueFrom:
secretKeyRef:
name: zitadel-secrets-vars
key: ZITADEL_TWILIO_SID
- name: SMTP_PASSWORD
valueFrom:
secretKeyRef:
name: zitadel-secrets-vars
key: ZITADEL_EMAILAPPKEY
envFrom:
- configMapRef:
name: zitadel-vars
volumeMounts:
- name: zitadel-secret
mountPath: /secret
- name: console-config
mountPath: /console/environment.json
subPath: environment.json
imagePullSecrets:
- name: githubsecret
volumes:
- name: zitadel-secret
secret:
secretName: zitadel-secret
- name: console-config
configMap:
name: console-config

18
k8s/base/flow.yaml
View File

@ -1,18 +0,0 @@
apiVersion: logging.banzaicloud.io/v1beta1
kind: Flow
metadata:
name: flow-zitadel
spec:
filters:
- parser:
remove_key_name_field: true
reserve_data: true
parse:
type: logfmt
- tag_normaliser:
metadata: ${namespace}.${container}.${pod}
metadata_name: ${namespace_name}.${container_name}.${pod_name}
selectors:
app: zitadel
outputRefs:
- output-loki

11
k8s/base/kustomization.yaml
View File

@ -1,11 +0,0 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: zitadel
resources:
- deployment.yaml
- service-grpc.yaml
- service-http.yaml
- service-ui.yaml
- flow.yaml

13
k8s/base/output.yaml
View File

@ -1,13 +0,0 @@
apiVersion: logging.banzaicloud.io/v1beta1
kind: Output
metadata:
name: output-loki
spec:
loki:
buffer:
timekey: 1m
timekey_use_utc: true
timekey_wait: 30s
configure_kubernetes_labels: true
url: http://loki.caos-system:3100

13
k8s/base/service-grpc.yaml
View File

@ -1,13 +0,0 @@
---
apiVersion: v1
kind: Service
metadata:
name: grpc-v1
spec:
ports:
- name: grpc
port: 80
targetPort: grpc
selector:
app: zitadel
version: v1

13
k8s/base/service-http.yaml
View File

@ -1,13 +0,0 @@
---
apiVersion: v1
kind: Service
metadata:
name: http-v1
spec:
ports:
- name: http
port: 80
targetPort: http
selector:
app: zitadel
version: v1

13
k8s/base/service-ui.yaml
View File

@ -1,13 +0,0 @@
---
apiVersion: v1
kind: Service
metadata:
name: ui-v1
spec:
ports:
- name: ui
port: 80
targetPort: ui
selector:
app: zitadel
version: v1