diff --git a/.github/workflows/ci_builds.yml b/.github/workflows/ci_builds.yml
index ef792dcbafc..b5e5bcb4fd5 100644
--- a/.github/workflows/ci_builds.yml
+++ b/.github/workflows/ci_builds.yml
@@ -39,11 +39,13 @@ jobs:
     - name: Run `qmk mass-compile` (keymap ${{ matrix.keymap }})
       run: |
         export NCPUS=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || getconf _NPROCESSORS_ONLN 2>/dev/null)
-        qmk mass-compile -t -j $NCPUS -km ${{ matrix.keymap }} -e DUMP_CI_METADATA=yes
+        qmk mass-compile -t -j $NCPUS -km ${{ matrix.keymap }} -e DUMP_CI_METADATA=yes || touch .failed
         # Generate the step summary markdown
-        ./util/ci/generate_failure_markdown.sh > $GITHUB_STEP_SUMMARY
+        ./util/ci/generate_failure_markdown.sh > $GITHUB_STEP_SUMMARY || true
         # Truncate to a maximum of 1MB to deal with GitHub workflow limit
-        truncate --size='<960K' $GITHUB_STEP_SUMMARY
+        truncate --size='<960K' $GITHUB_STEP_SUMMARY || true
+        # Exit with failure if the compilation stage failed
+        [ ! -f .failed ] || exit 1
 
     - name: 'Upload artifacts'
       uses: actions/upload-artifact@v3
diff --git a/util/ci/generate_failure_markdown.sh b/util/ci/generate_failure_markdown.sh
index 5f21f06856f..ccb3eacb350 100755
--- a/util/ci/generate_failure_markdown.sh
+++ b/util/ci/generate_failure_markdown.sh
@@ -19,3 +19,5 @@ dump_failure_info() {
 for failure_file in $(find "$qmk_firmware_dir/.build" -name 'failed.log.*' | sort); do
     dump_failure_info "$failure_file"
 done
+
+exit 0