dragonfly/.github/workflows/ci.yml

name: ci-tests

on:
  # push:
  # branches: [ main ]
  pull_request:
    branches: [main]
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  pre-commit:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 2
      - uses: actions/setup-python@v5
      - name: Install dependencies
        run: |
          python -m venv venv
          source venv/bin/activate
          python -m pip install pre-commit
          lsblk -l
          echo "sda rotational = $(cat /sys/block/sda/queue/rotational)"
          echo "sdb rotational = $(cat /sys/block/sdb/queue/rotational)"          
      - uses: actions/cache@v4
        with:
          path: ~/.cache/pre-commit
          key: pre-commit|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }}
      - name: Run pre-commit checks
        run:
          source venv/bin/activate
          pre-commit run --show-diff-on-failure --color=always --from-ref HEAD^ --to-ref HEAD
        shell: bash
  build:
    strategy:
      matrix:
        # Test of these containers
        container: ["ubuntu-dev:20", "alpine-dev:latest"]
        build-type: [Debug, Release]
        compiler: [{ cxx: g++, c: gcc }]
        cxx_flags: ["-Werror"]
        include:
          - container: "alpine-dev:latest"
            build-type: Debug
            compiler: { cxx: clang++, c: clang }
            cxx_flags: ""

    runs-on: ubuntu-latest
    env:
      SCCACHE_GHA_ENABLED: "true"
      SCCACHE_CACHE_SIZE: 6G
      SCCACHE_ERROR_LOG: /tmp/sccache_log.txt
      # SCCACHE_LOG: debug

    container:
      image: ghcr.io/romange/${{ matrix.container }}
      # Seems that docker by default prohibits running iouring syscalls
      options: --security-opt seccomp=unconfined --sysctl "net.ipv6.conf.all.disable_ipv6=0"
      volumes:
        - /:/hostroot
        - /mnt:/mnt
      credentials:
        username: ${{ github.repository_owner }}
        password: ${{ secrets.GITHUB_TOKEN }}

    steps:
      - uses: actions/checkout@v4
        with:
          submodules: true
      - name: Prepare Environment
        run: |
          uname -a
          cmake --version
          mkdir -p ${GITHUB_WORKSPACE}/build
          mount

          echo "===================Before freeing up space ============================================"
          df -h
          rm -rf /hostroot/usr/share/dotnet
          rm -rf /hostroot/usr/local/share/boost
          rm -rf /hostroot/usr/local/lib/android
          rm -rf /hostroot/opt/ghc
          echo "===================After freeing up space ============================================"
          df -h
          touch /mnt/foo
          ls -la /mnt/foo          
      - name: Run sccache-cache
        uses: mozilla-actions/sccache-action@v0.0.6

      - name: Configure Cache Env
        uses: actions/github-script@v7
        with:
          script: |
            core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || '');
            core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '')            

      - name: Configure CMake
        # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
        # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
        run: |
          # -no-pie to disable address randomization so we could symbolize stacktraces
          cmake -B ${GITHUB_WORKSPACE}/build \
            -DCMAKE_BUILD_TYPE=${{matrix.build-type}} \
            -GNinja \
            -DCMAKE_C_COMPILER="${{matrix.compiler.c}}" \
            -DCMAKE_CXX_COMPILER="${{matrix.compiler.cxx}}" \
            -DCMAKE_CXX_COMPILER_LAUNCHER=sccache -DCMAKE_C_COMPILER_LAUNCHER=sccache \
            -DCMAKE_CXX_FLAGS="${{matrix.cxx_flags}} -no-pie" -DWITH_AWS:BOOL=OFF \
            -L
          cd ${GITHUB_WORKSPACE}/build && pwd
          du -hcs _deps/          

      - name: Build
        run: |
          cd ${GITHUB_WORKSPACE}/build
          ninja search_family_test
          df -h
          echo "-----------------------------"
          ninja src/all          

      - name: PostFail
        if: failure()
        run: |
          echo "disk space is:"
          df -h          

      - name: C++ Unit Tests
        run: |
          cd ${GITHUB_WORKSPACE}/build
          echo Run ctest -V -L DFLY
          GLOG_alsologtostderr=1 GLOG_vmodule=rdb_load=1,rdb_save=1,snapshot=1 timeout 20m ctest -V -L DFLY

          echo "Running tests with --force_epoll"

          # Create a rule that automatically prints stacktrace upon segfault
          cat > ./init.gdb <<EOF
          catch signal SIGSEGV
          command
          bt
          end
          EOF

          gdb -ix ./init.gdb --batch -ex r --args ./dragonfly_test --force_epoll
          FLAGS_force_epoll=true GLOG_vmodule=rdb_load=1,rdb_save=1,snapshot=1 timeout 20m ctest -V -L DFLY

          echo "Finished running tests with --force_epoll"

          echo "Running tests with --cluster_mode=emulated"
          FLAGS_cluster_mode=emulated timeout 20m ctest -V -L DFLY

          echo "Running tests with both --cluster_mode=emulated & --lock_on_hashtags"
          FLAGS_cluster_mode=emulated FLAGS_lock_on_hashtags=true timeout 20m ctest -V -L DFLY

          timeout 5m ./dragonfly_test
          timeout 5m ./multi_test --multi_exec_mode=1
          timeout 5m ./multi_test --multi_exec_mode=3
          timeout 5m ./json_family_test --jsonpathv2=false
          timeout 5m ./tiered_storage_test --vmodule=db_slice=2 --logtostderr          
      - name: Upload unit logs on failure
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: unit_logs
          path: /tmp/*INFO*

      - name: Run regression tests
        if: matrix.container == 'ubuntu-dev:20'
        uses: ./.github/actions/regression-tests
        with:
          dfly-executable: dragonfly
          run-only-on-ubuntu-latest: true
          build-folder-name: build
          filter: ${{ matrix.build-type == 'Release' && '(not slow) and (not dbg_only)' || '(not slow) and (not opt_only)' }}

      - name: Upload regression logs on failure
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: regression_logs
          path: /tmp/failed/*

  lint-test-chart:
    runs-on: ubuntu-latest
    needs: [build]
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/lint-test-chart