mirror of
https://github.com/alexhopeoconnor/firmware.git
synced 2026-10-04 03:18:10 +10:00
Replace native-suite-count file with dynamic test discovery (#11413)
* Derive the native suite count on the fly instead of registering it in a file test/native-suite-count was a manually-maintained register of the test_* directory count, reconciled against the actual directories by bin/run-tests.sh (as an AMBER verdict) and by a dedicated suite-count-check CI job. The reconciliation only ever guarded the file itself: the check that matters - suites that actually ran vs. the test_* directories on disk - already derives its expected count from a directory walk, so the file added a bookkeeping step to every suite addition/removal without adding signal. Remove the file and everything that existed to keep it honest: - bin/run-tests.sh: drop the canonical-count file read, the count-mismatch AMBER verdict, and the [canonical: x/y] suffix; the verdict lines already carry ran/expected from the directory walk. The shuffle seed suffix stays. - test_native.yml: delete the suite-count-check job and its needs: edges. - Docs (copilot-instructions.md, AGENTS.md, test/README.md) and the test-script comments now describe the count as derived from test/test_* at run time. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01APCEfNjd1X7ErDHEzT6Dqd * Add suite-shrinkage-check: fail a PR that silently loses a test_* suite With test/native-suite-count gone, nothing in CI noticed the suite set shrinking: platformio test discovers and runs whatever test_* directories exist, and bin/run-tests.sh derives its expected count from the same walk, so a suite directory lost in a bad rebase or an overzealous cleanup just means fewer suites run - every remaining check stays green. Restore that tripwire git-aware instead of file-based: on pull_request runs, compare the test_* directory list at the PR's merge base against the PR result. A vanished suite fails the job unless its name appears in the PR title, PR body, or a commit message in the PR's range - a deliberate removal satisfies that by stating what it removes; an accidental loss cannot. Other events skip: they have no natural base, and PRs are where accidents arrive. No job depends on this one (a skipped job would skip its dependents). Incidentally: test/ currently holds 47 test_* directories while the deleted count file said 46 - the manual register had already drifted, which is exactly the bookkeeping failure mode this replaces. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01APCEfNjd1X7ErDHEzT6Dqd * Re-pad the verdict table after shortening the AMBER row Shrinking the AMBER cell left the table's column padding inconsistent, which trunk (prettier + markdownlint MD060) rejects. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01APCEfNjd1X7ErDHEzT6Dqd --------- Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Jonathan Bennett
co-authored by
Claude Fable 5
Claude Fable 5
parent
d765bd99ca
commit
af56a11f00
+13
-38
@@ -163,20 +163,11 @@ export MESHTASTIC_TEST_STATE_SUMMARY="$STATE_SUMMARY"
|
||||
$KEEP_STATE && export MESHTASTIC_TEST_KEEP_STATE=1
|
||||
$WRITE_MANIFEST && export MESHTASTIC_TEST_KEEP_STATE=1
|
||||
|
||||
# Canonical suite set = the directories in test/. This is the source of truth for
|
||||
# "what should run"; a filtered run only expects its filtered suite.
|
||||
# Canonical suite set = the directories in test/, detected on the fly. This is the sole source
|
||||
# of truth for "what should run"; a filtered run only expects its filtered suite.
|
||||
mapfile -t ALL_SUITES < <(find test -maxdepth 1 -type d -name 'test_*' -printf '%f\n' | sort)
|
||||
EXPECTED_COUNT=${#ALL_SUITES[@]}
|
||||
|
||||
# Canonical suite count - the registered total, maintained in test/native-suite-count.
|
||||
# Update that file whenever a test suite is added or removed.
|
||||
CANONICAL_COUNT_FILE="test/native-suite-count"
|
||||
if [[ -f $CANONICAL_COUNT_FILE ]]; then
|
||||
CANONICAL_COUNT=$(tr -d '[:space:]' <"$CANONICAL_COUNT_FILE")
|
||||
else
|
||||
CANONICAL_COUNT=""
|
||||
fi
|
||||
|
||||
# Cached object-count for this env, written after each completed build (in the gitignored build
|
||||
# dir). Used as the progress denominator: accurate for a full rebuild (every object recompiles),
|
||||
# only a rough upper bound for an incremental run.
|
||||
@@ -462,31 +453,15 @@ if ! grep -qE "$PASS_RE" "$LOG"; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Canonical-count rating suffix - appended to every verdict line so the result is always
|
||||
# rated against the registered total, not just the directory count.
|
||||
# If the two counts diverge (suite added/removed without updating native-suite-count), that
|
||||
# is itself surfaced as AMBER before we reach any verdict.
|
||||
canonical_rating() {
|
||||
# Verdict-line suffix. The suite count itself is derived from the test_* directories on the fly
|
||||
# (EXPECTED_COUNT above), so the only extra context a verdict needs is the shuffle seed - carried
|
||||
# into the machine-readable line so a verdict is always replayable from it alone.
|
||||
verdict_suffix() {
|
||||
local rating=""
|
||||
if [[ -n $CANONICAL_COUNT ]]; then
|
||||
rating="[canonical: ${RAN_COUNT}/${CANONICAL_COUNT}]"
|
||||
fi
|
||||
# Carry the seed into the machine-readable line so a verdict is always replayable from it alone.
|
||||
$SHUFFLE && rating="$rating [seed: $SEED]"
|
||||
$SHUFFLE && rating="[seed: $SEED]"
|
||||
echo "$rating"
|
||||
}
|
||||
|
||||
# AMBER: directory count disagrees with native-suite-count - file needs updating.
|
||||
if [[ -n $CANONICAL_COUNT && $EXPECTED_COUNT -ne $CANONICAL_COUNT ]]; then
|
||||
echo ""
|
||||
if [[ $EXPECTED_COUNT -gt $CANONICAL_COUNT ]]; then
|
||||
echo "RESULT: AMBER test/ has $EXPECTED_COUNT suite directories but native-suite-count says $CANONICAL_COUNT - update test/native-suite-count after registering new suites"
|
||||
else
|
||||
echo "RESULT: AMBER test/ has $EXPECTED_COUNT suite directories but native-suite-count says $CANONICAL_COUNT - update test/native-suite-count after removing suites"
|
||||
fi
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# --- Shared-state axis --------------------------------------------------------
|
||||
# Read what the per-suite wrapper recorded. Reported after the count checks so a structural problem
|
||||
# still wins, and before the pass/fail verdict lines so the state summary always prints.
|
||||
@@ -546,7 +521,7 @@ if [[ $IGNORED_COUNT -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "$IGNORE_DETAIL"
|
||||
echo ""
|
||||
echo "RESULT: AMBER ${IGNORED_COUNT} test case(s) ignored $(canonical_rating)"
|
||||
echo "RESULT: AMBER ${IGNORED_COUNT} test case(s) ignored $(verdict_suffix)"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
@@ -558,7 +533,7 @@ if [[ -z $FILTER && $ACCOUNTED_COUNT -lt $EXPECTED_COUNT ]]; then
|
||||
printf '%s\n' "${RAN_SUITES[@]}" "${SKIPPED_SUITES[@]}" | grep -qx "$s" || missing+=("$s")
|
||||
done
|
||||
echo ""
|
||||
echo "RESULT: AMBER ${RAN_COUNT}/${EXPECTED_COUNT} suites ran (missing: ${missing[*]}) - all that ran passed $(canonical_rating)"
|
||||
echo "RESULT: AMBER ${RAN_COUNT}/${EXPECTED_COUNT} suites ran (missing: ${missing[*]}) - all that ran passed $(verdict_suffix)"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
@@ -572,7 +547,7 @@ if ((${#DIRTY_SUITES[@]} > 0)); then
|
||||
echo ""
|
||||
echo " -> declare these in test/state-manifest.tsv with a reason, or stop the write."
|
||||
echo " -> ./bin/run-tests.sh --write-manifest prints the entries to paste."
|
||||
echo "RESULT: AMBER ${#DIRTY_SUITES[@]} suite(s) left undeclared shared state $(canonical_rating)"
|
||||
echo "RESULT: AMBER ${#DIRTY_SUITES[@]} suite(s) left undeclared shared state $(verdict_suffix)"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
@@ -588,7 +563,7 @@ if ((${#SURVIVOR_SUITES[@]} > 0)); then
|
||||
echo ""
|
||||
echo " -> end every setup() branch with exit(UNITY_END()), not a bare UNITY_END()."
|
||||
echo " -> ./bin/lint-unity-exit.sh test/**/*.cpp finds the sites; see test/README.md."
|
||||
echo "RESULT: AMBER ${#SURVIVOR_SUITES[@]} suite(s) still running after the suite finished $(canonical_rating)"
|
||||
echo "RESULT: AMBER ${#SURVIVOR_SUITES[@]} suite(s) still running after the suite finished $(verdict_suffix)"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
@@ -599,10 +574,10 @@ if [[ -n $FILTER ]]; then
|
||||
for s in "${ALL_SUITES[@]}"; do
|
||||
printf '%s\n' "${RAN_SUITES[@]}" "${SKIPPED_SUITES[@]}" | grep -qx "$s" || not_run+=("$s")
|
||||
done
|
||||
echo "RESULT: FILTERED ${RAN_COUNT}/${EXPECTED_COUNT} suites ran (not run: ${not_run[*]}) - filtered: $FILTER $(canonical_rating)"
|
||||
echo "RESULT: FILTERED ${RAN_COUNT}/${EXPECTED_COUNT} suites ran (not run: ${not_run[*]}) - filtered: $FILTER $(verdict_suffix)"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
# GREEN: all canonical suites ran, all passed, no ignored test cases, nothing undeclared left behind.
|
||||
echo "RESULT: GREEN ${RAN_COUNT}/${EXPECTED_COUNT} suites passed, all CLEAN $(canonical_rating)"
|
||||
echo "RESULT: GREEN ${RAN_COUNT}/${EXPECTED_COUNT} suites passed, all CLEAN $(verdict_suffix)"
|
||||
exit 0
|
||||
|
||||
Reference in New Issue
Block a user