[MISC] Consolidate test reports into single PR comment and document libmagic1 dependency (#1598)
* Update tox.ini to document libmagic1 system dependency requirement * [CI] Consolidate test reports into single PR comment This change reduces noise in PR comments by combining multiple test reports (runner and sdk1) into a single consolidated comment. Changes: - Add combine-test-reports.sh script to merge test reports - Update CI workflow to combine reports before posting - Replace separate runner/sdk1 comment steps with single combined step - Summary section shows test counts (passed/failed/total) collapsed by default - Full detailed reports available in collapsible sections - Simplify job summary to use combined report Benefits: - Single PR comment instead of multiple separate comments - Cleaner PR comment section with less clutter - Easy-to-read summary with detailed inspection on demand - Maintains all existing test information 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com> * Fix test count extraction in combine-test-reports script The previous version was using text pattern matching which incorrectly extracted test counts. This fix properly parses the pytest-md-report markdown table format by: - Finding the table header row to determine column positions - Locating the TOTAL row (handles both TOTAL and **TOTAL** formatting) - Extracting values from the passed, failed, and SUBTOTAL columns - Using proper table parsing instead of pattern matching This resolves the issue where the summary showed incorrect counts (23 for both runner and sdk1 instead of the actual 11 and 66). Fixes: Test count summary in PR comments now shows correct values 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com> * Fix LaTeX formatting in pytest-md-report test count extraction pytest-md-report wraps all table values in LaTeX formatting: $$\textcolor{...}{\tt{VALUE}}$$ The previous fix attempted to parse the table but didn't handle the LaTeX formatting, causing it to extract 0 for all counts. Changes: - Add strip_latex() function to extract values from LaTeX wrappers - Update grep pattern to match TOTAL row specifically (not SUBTOTAL in header) - Apply LaTeX stripping to all extracted values before parsing This fix was tested locally with tox-generated reports and correctly shows: Runner=11 passed, SDK1=66 passed 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com> --------- Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Chandrasekharan M
committed by
GitHub
GitHub
parent
2bef7d30f2
commit
8913c08197
171
.github/scripts/combine-test-reports.sh
vendored
Executable file
171
.github/scripts/combine-test-reports.sh
vendored
Executable file
@@ -0,0 +1,171 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
# Script to combine multiple test reports into a single markdown file
|
||||
# Usage: ./combine-test-reports.sh
|
||||
|
||||
OUTPUT_FILE="combined-test-report.md"
|
||||
REPORTS=()
|
||||
|
||||
# Find all test report files
|
||||
for report in runner-report.md sdk1-report.md; do
|
||||
if [ -f "$report" ]; then
|
||||
REPORTS+=("$report")
|
||||
fi
|
||||
done
|
||||
|
||||
# Exit if no reports found
|
||||
if [ ${#REPORTS[@]} -eq 0 ]; then
|
||||
echo "No test reports found. Skipping report generation."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Function to strip LaTeX formatting from pytest-md-report output
|
||||
# Converts $$\textcolor{...}{\tt{VALUE}}$$ to just VALUE
|
||||
strip_latex() {
|
||||
local text="$1"
|
||||
# Extract content between \tt{ and }}
|
||||
if [[ "$text" =~ \\tt\{([^}]+)\} ]]; then
|
||||
echo "${BASH_REMATCH[1]}"
|
||||
else
|
||||
echo "$text"
|
||||
fi
|
||||
}
|
||||
|
||||
# Function to extract test counts from pytest-md-report markdown table
|
||||
extract_test_counts() {
|
||||
local report_file=$1
|
||||
local passed=0
|
||||
local failed=0
|
||||
local total=0
|
||||
|
||||
# Find the header row to determine column positions
|
||||
local header_line=$(grep -E '^\|.*filepath' "$report_file" | head -1)
|
||||
|
||||
if [ -z "$header_line" ]; then
|
||||
echo "0:0:0"
|
||||
return
|
||||
fi
|
||||
|
||||
# Extract column names and find positions (strip LaTeX from headers)
|
||||
IFS='|' read -ra headers <<< "$header_line"
|
||||
local passed_col=-1
|
||||
local failed_col=-1
|
||||
local subtotal_col=-1
|
||||
|
||||
for i in "${!headers[@]}"; do
|
||||
local col=$(strip_latex "${headers[$i]}" | tr -d ' ' | tr '[:upper:]' '[:lower:]')
|
||||
case "$col" in
|
||||
passed) passed_col=$i ;;
|
||||
failed) failed_col=$i ;;
|
||||
subtotal|sub) subtotal_col=$i ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Find the TOTAL row (TOTAL appears in first column, not as SUBTOTAL in header)
|
||||
local total_line=$(grep -E '^\|.*\\tt\{TOTAL\}' "$report_file" | head -1)
|
||||
|
||||
if [ -z "$total_line" ]; then
|
||||
echo "0:0:0"
|
||||
return
|
||||
fi
|
||||
|
||||
# Parse the TOTAL row values
|
||||
IFS='|' read -ra values <<< "$total_line"
|
||||
|
||||
# Extract passed count (strip LaTeX and get number)
|
||||
if [ "$passed_col" -ge 0 ] && [ "$passed_col" -lt "${#values[@]}" ]; then
|
||||
local clean_value=$(strip_latex "${values[$passed_col]}")
|
||||
passed=$(echo "$clean_value" | tr -d ' ' | grep -oE '[0-9]+' | head -1 || echo "0")
|
||||
fi
|
||||
|
||||
# Extract failed count (strip LaTeX and get number)
|
||||
if [ "$failed_col" -ge 0 ] && [ "$failed_col" -lt "${#values[@]}" ]; then
|
||||
local clean_value=$(strip_latex "${values[$failed_col]}")
|
||||
failed=$(echo "$clean_value" | tr -d ' ' | grep -oE '[0-9]+' | head -1 || echo "0")
|
||||
fi
|
||||
|
||||
# Extract total from SUBTOTAL column (strip LaTeX and get number)
|
||||
if [ "$subtotal_col" -ge 0 ] && [ "$subtotal_col" -lt "${#values[@]}" ]; then
|
||||
local clean_value=$(strip_latex "${values[$subtotal_col]}")
|
||||
total=$(echo "$clean_value" | tr -d ' ' | grep -oE '[0-9]+' | head -1 || echo "0")
|
||||
fi
|
||||
|
||||
# If total is still 0, calculate from passed + failed
|
||||
if [ "$total" -eq 0 ]; then
|
||||
total=$((passed + failed))
|
||||
fi
|
||||
|
||||
echo "${total}:${passed}:${failed}"
|
||||
}
|
||||
|
||||
# Initialize the combined report with collapsed summary
|
||||
cat > "$OUTPUT_FILE" << 'EOF'
|
||||
# Test Results
|
||||
|
||||
<details open>
|
||||
<summary><b>Summary</b></summary>
|
||||
|
||||
EOF
|
||||
|
||||
# Extract and display summary for each report
|
||||
for report in "${REPORTS[@]}"; do
|
||||
report_name=$(basename "$report" .md)
|
||||
|
||||
# Convert report name to title case
|
||||
if [ "$report_name" = "runner-report" ]; then
|
||||
title="Runner Tests"
|
||||
elif [ "$report_name" = "sdk1-report" ]; then
|
||||
title="SDK1 Tests"
|
||||
else
|
||||
title="${report_name}"
|
||||
fi
|
||||
|
||||
# Extract counts
|
||||
counts=$(extract_test_counts "$report")
|
||||
IFS=':' read -r total passed failed <<< "$counts"
|
||||
|
||||
# Determine status icon
|
||||
if [ "$failed" -gt 0 ]; then
|
||||
status="❌"
|
||||
elif [ "$passed" -gt 0 ]; then
|
||||
status="✅"
|
||||
else
|
||||
status="⚠️"
|
||||
fi
|
||||
|
||||
echo "- ${status} **${title}**: ${passed} passed, ${failed} failed (${total} total)" >> "$OUTPUT_FILE"
|
||||
done
|
||||
|
||||
cat >> "$OUTPUT_FILE" << 'EOF'
|
||||
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
EOF
|
||||
|
||||
# Combine all reports with collapsible sections
|
||||
for report in "${REPORTS[@]}"; do
|
||||
report_name=$(basename "$report" .md)
|
||||
|
||||
# Convert report name to title case
|
||||
if [ "$report_name" = "runner-report" ]; then
|
||||
title="Runner Tests"
|
||||
elif [ "$report_name" = "sdk1-report" ]; then
|
||||
title="SDK1 Tests"
|
||||
else
|
||||
title="${report_name}"
|
||||
fi
|
||||
|
||||
echo "<details>" >> "$OUTPUT_FILE"
|
||||
echo "<summary><b>${title} - Full Report</b></summary>" >> "$OUTPUT_FILE"
|
||||
echo "" >> "$OUTPUT_FILE"
|
||||
cat "$report" >> "$OUTPUT_FILE"
|
||||
echo "" >> "$OUTPUT_FILE"
|
||||
echo "</details>" >> "$OUTPUT_FILE"
|
||||
echo "" >> "$OUTPUT_FILE"
|
||||
done
|
||||
|
||||
echo "Combined test report created: $OUTPUT_FILE"
|
||||
echo "Included reports: ${REPORTS[*]}"
|
||||
37
.github/workflows/ci-test.yaml
vendored
37
.github/workflows/ci-test.yaml
vendored
@@ -48,36 +48,21 @@ jobs:
|
||||
run: |
|
||||
tox
|
||||
|
||||
- name: Render the Runner report to the PR
|
||||
uses: marocchino/sticky-pull-request-comment@773744901bac0e8cbb5a0dc842800d45e9b2b405 # v2.9.4
|
||||
if: always() && hashFiles('runner-report.md') != ''
|
||||
with:
|
||||
header: runner-test-report
|
||||
recreate: true
|
||||
path: runner-report.md
|
||||
- name: Combine test reports
|
||||
if: always() && (hashFiles('runner-report.md') != '' || hashFiles('sdk1-report.md') != '')
|
||||
run: |
|
||||
bash .github/scripts/combine-test-reports.sh
|
||||
|
||||
- name: Render the SDK1 report to the PR
|
||||
- name: Render combined test report to PR
|
||||
uses: marocchino/sticky-pull-request-comment@773744901bac0e8cbb5a0dc842800d45e9b2b405 # v2.9.4
|
||||
if: always() && hashFiles('sdk1-report.md') != ''
|
||||
if: always() && hashFiles('combined-test-report.md') != ''
|
||||
with:
|
||||
header: sdk1-test-report
|
||||
header: test-results
|
||||
recreate: true
|
||||
path: sdk1-report.md
|
||||
path: combined-test-report.md
|
||||
|
||||
- name: Output reports to the job summary when tests fail
|
||||
- name: Output combined report to job summary
|
||||
if: always() && hashFiles('combined-test-report.md') != ''
|
||||
shell: bash
|
||||
run: |
|
||||
if [ -f "runner-report.md" ]; then
|
||||
echo "<details><summary>Runner Test Report</summary>" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
cat "runner-report.md" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "</details>" >> $GITHUB_STEP_SUMMARY
|
||||
fi
|
||||
if [ -f "sdk1-report.md" ]; then
|
||||
echo "<details><summary>SDK1 Test Report</summary>" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
cat "sdk1-report.md" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "</details>" >> $GITHUB_STEP_SUMMARY
|
||||
fi
|
||||
cat combined-test-report.md >> $GITHUB_STEP_SUMMARY
|
||||
|
||||
Reference in New Issue
Block a user