diff options
Diffstat (limited to 'org')
| -rw-r--r-- | org/tests_for_document_abstraction_shell_scripts.org | 78 |
1 files changed, 55 insertions, 23 deletions
diff --git a/org/tests_for_document_abstraction_shell_scripts.org b/org/tests_for_document_abstraction_shell_scripts.org index d62bf17..793d69f 100644 --- a/org/tests_for_document_abstraction_shell_scripts.org +++ b/org/tests_for_document_abstraction_shell_scripts.org @@ -30,14 +30,18 @@ # SpinePOD=../../markup/sisudoc-spine-samples/markup/pod-samples/pod \ # ./test/run-tests.sh ./bin/spine-ldc # -# sqlite3 is needed by one of the five. If it is not on PATH the script says -# so and re-runs itself inside a nix shell that has it, so this works: +# Two outside tools are needed: sqlite3 by one script and epubcheck by +# another. If either is not on PATH the harness re-runs itself inside a nix +# shell that has both, so this works as it stands: # SpinePOD=... ./test/run-tests.sh ./bin/spine-ldc # # Exit codes: -# 0 every test passed -# n the number of tests that failed -# 2 set-up problem (no $SpinePOD, no binary) +# 0 every test ran and passed +# n the number of tests that ran and failed +# 2 the suite did not run in full: a set-up problem here (no $SpinePOD, +# no binary, a missing tool) or a script exiting 2 for one of its own. +# Reported as INCOMPLETE rather than PASS, because nothing should read +# "it passed" out of a check that never ran # # Order, and what depends on what: # @@ -85,12 +89,27 @@ if [ -z "$SpinePOD" ]; then exit 2 fi -if ! command -v sqlite3 > /dev/null 2>&1; then +# the tools the scripts need, and the shell that provides them. +# +# Both are checked, not just sqlite3. Checking one and providing two meant +# that a shell with sqlite3 already on PATH - a dev shell, or any machine +# with sqlite installed - did not re-exec, and test-epub-validity.sh then +# could not find epubcheck and reported a set-up problem that read as a +# failing test. +_missing="" +command -v sqlite3 > /dev/null 2>&1 || _missing="$_missing sqlite3" +command -v epubcheck > /dev/null 2>&1 || _missing="$_missing epubcheck" +if [ -n "$_missing" ]; then + if [ -n "$SpineTestsReExeced" ]; then + echo "ERROR: still missing$_missing inside the nix shell." >&2 + exit 2 + fi if command -v nix > /dev/null 2>&1; then - echo "sqlite3 not on PATH, re-running inside nix shell (sqlite, epubcheck)" - exec nix shell "nixpkgs#sqlite" "nixpkgs#epubcheck" -c "$0" "$@" + echo "not on PATH:$_missing - re-running inside nix shell (sqlite, epubcheck)" + SpineTestsReExeced=1 exec nix shell "nixpkgs#sqlite" "nixpkgs#epubcheck" \ + -c "$0" "$@" fi - echo "ERROR: sqlite3 not on PATH and nix not available." >&2 + echo "ERROR: missing$_missing and nix is not available." >&2 exit 2 fi @@ -104,6 +123,7 @@ test-epub-validity.sh" PASSED=0 FAILED=0 +SETUP=0 RESULTS="" LOG="${TMPDIR:-/tmp}/spine-run-tests.$$" @@ -112,28 +132,40 @@ for t in $TESTS; do # the status has to be the test's, not the last command of a pipeline: # "cmd | sed" reports sed's status, which is always 0, and a harness that # cannot fail is worse than no harness - if sh "$SCRIPT_DIR/$t" $SPINE_BIN > "$LOG" 2>&1; then - sed 's/^/ /' "$LOG" - PASSED=$((PASSED + 1)) - RESULTS="$RESULTS - ok $t" - else - sed 's/^/ /' "$LOG" - FAILED=$((FAILED + 1)) - RESULTS="$RESULTS - FAIL $t" - fi + # "|| _rc=$?" and not a bare call: set -e is on, and a bare non-zero + # would end the harness at the first failing test instead of running the + # rest and summarising + _rc=0 + sh "$SCRIPT_DIR/$t" $SPINE_BIN > "$LOG" 2>&1 || _rc=$? + sed 's/^/ /' "$LOG" + # every script uses 2 for "could not run" and 1 for "ran and failed", and + # the two must not read alike: "FAIL test-epub-validity.sh" says the epubs + # are invalid, where exit 2 says nobody checked them + case "$_rc" in + 0) PASSED=$((PASSED + 1)); RESULTS="$RESULTS + ok $t" ;; + 2) SETUP=$((SETUP + 1)); RESULTS="$RESULTS + SET-UP $t (could not run; nothing was checked)" ;; + *) FAILED=$((FAILED + 1)); RESULTS="$RESULTS + FAIL $t" ;; + esac done rm -f "$LOG" echo "" echo "summary:$RESULTS" echo "" -if [ "$FAILED" -eq 0 ]; then - echo "PASS: $PASSED of $PASSED" +_total=$((PASSED + FAILED + SETUP)) +if [ "$FAILED" -eq 0 ] && [ "$SETUP" -eq 0 ]; then + echo "PASS: $PASSED of $_total" exit 0 +elif [ "$FAILED" -eq 0 ]; then + # nothing failed, but the suite did not run in full, and saying PASS here + # would claim more than was checked + echo "INCOMPLETE: $PASSED of $_total ran, $SETUP could not" + exit 2 else - echo "FAIL: $FAILED of $((PASSED + FAILED))" + echo "FAIL: $FAILED of $_total" exit "$FAILED" fi #+END_SRC |