diff options
| -rw-r--r-- | org/tests_for_document_abstraction_shell_scripts.org | 435 | ||||
| -rwxr-xr-x | test/run-tests.sh | 9 | ||||
| -rwxr-xr-x | test/test-abstraction-db.sh | 8 | ||||
| -rwxr-xr-x | test/test-ocn-alignment.sh | 155 |
4 files changed, 472 insertions, 135 deletions
diff --git a/org/tests_for_document_abstraction_shell_scripts.org b/org/tests_for_document_abstraction_shell_scripts.org index 0f41ca6..b4d71f8 100644 --- a/org/tests_for_document_abstraction_shell_scripts.org +++ b/org/tests_for_document_abstraction_shell_scripts.org @@ -45,7 +45,8 @@ # 2 test-abstraction-ssp-roundtrip.sh the .ssp can be read back whole # 3 test-abstraction-db.sh the two serialisations agree # 4 test-abstraction-db-roundtrip.sh the .db can be read back whole -# 5 test-epub-validity.sh the epubs a reader can actually open +# 5 test-ocn-alignment.sh the languages agree on their ocns +# 6 test-epub-validity.sh the epubs a reader can actually open # # None of them calls another, and each can be run on its own. The order # matters only for reading the result: @@ -59,6 +60,11 @@ # - 3 and 4 generate both artefacts themselves and depend on nothing # committed. Since the database is built from the .ssp, they are checks # on the reader rather than on two independent writers. +# - 5 is about the sample documents rather than about a serialisation: it +# asserts that spine reports the two known divergences in live-manual's +# translations and reports nothing about the other documents. It will +# fail if those translations are mended, which is the right moment to +# hear about it. set -e @@ -84,6 +90,7 @@ TESTS="test-abstraction-ssp.sh test-abstraction-ssp-roundtrip.sh test-abstraction-db.sh test-abstraction-db-roundtrip.sh +test-ocn-alignment.sh test-epub-validity.sh" PASSED=0 @@ -122,8 +129,8 @@ else fi #+END_SRC -** ssp ocda abstraction -*** document.ssp abstraction test/test-abstraction-ssp.sh +** 1&2. ssp ocda abstraction +*** 1. document.ssp abstraction test/test-abstraction-ssp.sh #+HEADER: :tangle ../test/test-abstraction-ssp.sh #+HEADER: :tangle-mode (identity #o755) @@ -314,7 +321,7 @@ else fi #+END_SRC -*** document.ssp abstraction test/test-abstraction-ssp-roundtrip.sh +*** 2. document.ssp abstraction test/test-abstraction-ssp-roundtrip.sh #+HEADER: :tangle ../test/test-abstraction-ssp-roundtrip.sh #+HEADER: :tangle-mode (identity #o755) @@ -404,8 +411,8 @@ else fi #+END_SRC -** db ocda abstraction -*** document db ocda test/test-abstraction-db.sh +** 3&4. db ocda abstraction +*** 3. document db ocda test/test-abstraction-db.sh #+HEADER: :tangle ../test/test-abstraction-db.sh #+HEADER: :tangle-mode (identity #o755) @@ -646,7 +653,11 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do done # the header blocks: every @source/@meta/@make/@doc_has property in the .ssp - # must be in the database under its own key, with the same value. this is + # must be in the database under its own key, with the same value. the three + # excluded keys are the database's own notes about the artefact rather than + # properties of the document: what it was built from (source.filename, + # source.ssp_digest) and how its languages compared once they had all been + # read (translation.ocn_aligned), which no single .ssp can state. this is # what catches a header property that reaches one artefact and not the # other, which a count of objects cannot see. awk ' @@ -661,7 +672,7 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do ' "$ssp" | sort > "$OUT_DIR/.sspmeta.$$" sqlite3 -separator "$(printf '\t')" "$db" \ "select key, value from metadata where doc_id=$did and key not in - ('source.filename','source.ssp_digest');" \ + ('source.filename','source.ssp_digest','translation.ocn_aligned');" \ 2>/dev/null | sort > "$OUT_DIR/.dbmeta.$$" if ! diff -q "$OUT_DIR/.sspmeta.$$" "$OUT_DIR/.dbmeta.$$" > /dev/null 2>&1; then [ $doc_fail -eq 0 ] && echo "$base" @@ -726,7 +737,7 @@ else fi #+END_SRC * -*** document db ocda test-abstraction-db-roundtrip.sh +*** 4. document db ocda test-abstraction-db-roundtrip.sh #+HEADER: :tangle ../test/test-abstraction-db-roundtrip.sh #+HEADER: :tangle-mode (identity #o755) @@ -868,6 +879,289 @@ else fi #+END_SRC +** 5. test ocn alignment + +#+HEADER: :tangle ../test/test-ocn-alignment.sh +#+HEADER: :tangle-mode (identity #o755) +#+HEADER: :shebang "#!/usr/bin/env sh" +#+BEGIN_SRC shell +# test-ocn-alignment.sh +# +# The languages of a document share their object numbering, and spine warns +# when they stop doing so. This checks that the warning appears where it +# should and, just as importantly, nowhere else. +# +# The sample set is the fixture, and it is a good one because it contains +# both answers: +# +# live-manual ten languages, two of which diverge. +# ja has two fewer numbered objects than en. A missing +# translation of a short paragraph; everything below it +# is renumbered. +# pl has the same count, but the object at ocn 313 is a +# paragraph in en and a heading in pl. A heading given as +# running text, or running text given as a heading. +# everything else single language, or aligned. Silent. +# +# A check that only ever fires is as useless as one that never does, so the +# silence over the other documents is asserted too. +# +# It also asserts that --strict turns the warning into a non-zero exit, and +# that without --strict the run still succeeds: a document with a diverging +# translation is still a document. +# +# Usage: +# SpinePOD=../../markup/sisudoc-spine-samples/markup/pod-samples/pod \ +# ./test/test-ocn-alignment.sh ./bin/spine-ldc +# +# Exit codes: +# 0 the check reports what it should, and nothing else +# 1 it does not (printed) +# 2 set-up problem (no $SpinePOD, no binary, no samples) + +set -e + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +SPINE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" + +if [ -z "$SpinePOD" ]; then + echo "ERROR: \$SpinePOD is not set." >&2 + echo " e.g. SpinePOD=../../markup/sisudoc-spine-samples/markup/pod-samples/pod" >&2 + exit 2 +fi +SAMPLES_RAW="$SPINE_DIR/$SpinePOD" +if [ -d "$SAMPLES_RAW" ]; then + SAMPLES_DIR="$(cd "$SAMPLES_RAW" && pwd)" +else + SAMPLES_DIR="$SAMPLES_RAW" +fi +if [ ! -d "$SAMPLES_DIR" ]; then + echo "ERROR: sample documents not found at $SAMPLES_DIR" >&2 + exit 2 +fi + +SPINE_BIN="${1:-}" +if [ -z "$SPINE_BIN" ]; then + if [ -x "$SPINE_DIR/result/bin/spine" ]; then SPINE_BIN="$SPINE_DIR/result/bin/spine" + elif [ -x "$SPINE_DIR/bin/spine-ldc" ]; then SPINE_BIN="$SPINE_DIR/bin/spine-ldc" + elif [ -x "$SPINE_DIR/bin/spine" ]; then SPINE_BIN="$SPINE_DIR/bin/spine" + else + echo "ERROR: spine binary not found. Specify path as argument." >&2 + exit 2 + fi +fi + +OUT_DIR="$SCRIPT_DIR/current-ocn-alignment" +rm -rf "$OUT_DIR" +mkdir -p "$OUT_DIR" + +echo "spine binary: $SPINE_BIN" +echo "samples: $SAMPLES_DIR" + +FAILURES=0 +fail() { echo " FAIL: $*"; FAILURES=$((FAILURES + 1)); } + +# the whole sample set, abstraction only: the check does not need outputs +echo "Abstracting the sample set ..." +set +e +$SPINE_BIN --show-abstraction --skip-output \ + --output="$OUT_DIR" "$SAMPLES_DIR"/* \ + > "$OUT_DIR/run.out" 2> "$OUT_DIR/run.err" +RUN_RC=$? +set -e + +# without --strict a diverging document is still built +if [ "$RUN_RC" -ne 0 ]; then + fail "the run exited $RUN_RC without --strict; a warning must not fail a run" + head -5 "$OUT_DIR/run.err" +fi + +# what it said +if ! grep -q "^WARNING ocn alignment: live-manual$" "$OUT_DIR/run.err"; then + fail "no ocn alignment warning for live-manual" +fi +if ! grep -q "^ \[ja\] differs from \[en\]$" "$OUT_DIR/run.err"; then + fail "ja not reported as differing from en" +fi +if ! grep -q "numbered objects: en 1019, ja 1017 (-2)" "$OUT_DIR/run.err"; then + fail "ja's object count difference not reported as -2" +fi +if ! grep -q "^ \[pl\] differs from \[en\]$" "$OUT_DIR/run.err"; then + fail "pl not reported as differing from en" +fi +if ! grep -q "ocn 313: para in en, heading:2 in pl" "$OUT_DIR/run.err"; then + fail "pl's substituted heading at ocn 313 not reported" +fi + +# and what it did not say: exactly one document warned about, exactly two +# languages of it +n_docs=$(grep -c "^WARNING ocn alignment: " "$OUT_DIR/run.err" || true) +if [ "$n_docs" != "1" ]; then + fail "$n_docs documents warned about, expected 1" + grep "^WARNING ocn alignment: " "$OUT_DIR/run.err" | sed 's/^/ /' +fi +n_langs=$(grep -c "^ \[.*\] differs from " "$OUT_DIR/run.err" || true) +if [ "$n_langs" != "2" ]; then + fail "$n_langs languages reported, expected 2 (ja and pl)" + grep "^ \[.*\] differs from " "$OUT_DIR/run.err" | sed 's/^/ /' +fi + +# --strict makes the same run fail +set +e +$SPINE_BIN --show-abstraction --skip-output --strict \ + --output="$OUT_DIR/strict" "$SAMPLES_DIR"/live-manual \ + > "$OUT_DIR/strict.out" 2> "$OUT_DIR/strict.err" +STRICT_RC=$? +set -e +if [ "$STRICT_RC" -eq 0 ]; then + fail "--strict exited 0 on a document whose languages diverge" +fi + +# and does not fail a document whose languages agree, or one that has only +# the one language +set +e +$SPINE_BIN --show-abstraction --skip-output --strict \ + --output="$OUT_DIR/strict-ok" "$SAMPLES_DIR"/the_wealth_of_networks.yochai_benkler \ + > "$OUT_DIR/strict-ok.out" 2> "$OUT_DIR/strict-ok.err" +STRICT_OK_RC=$? +set -e +if [ "$STRICT_OK_RC" -ne 0 ]; then + fail "--strict exited $STRICT_OK_RC on a single language document" + head -5 "$OUT_DIR/strict-ok.err" +fi + +rm -rf "$OUT_DIR" + +if [ "$FAILURES" -eq 0 ]; then + echo "PASS: ja and pl reported on live-manual, silent elsewhere, --strict fails" + exit 0 +else + echo "FAIL: $FAILURES check(s)" + exit 1 +fi +#+END_SRC +** 6. test epub validity + +#+HEADER: :tangle ../test/test-epub-validity.sh +#+HEADER: :tangle-mode (identity #o755) +#+HEADER: :shebang "#!/usr/bin/env sh" +#+BEGIN_SRC shell +# epubcheck over the sample set, as a regression guard. +# +# Every epub spine writes is checked with epubcheck. A *fatal* is never +# acceptable: it means a reader cannot parse the file at all. Errors are +# compared against a recorded allowance, so that the ones known to be +# outstanding do not fail the run while any new one does. +# +# The allowance is deliberately a single number rather than a list of +# messages: a list invites adding to it. When the number goes down, lower it +# here in the same commit that lowers it in the output. +# +# Usage: +# SpinePOD=../../markup/sisudoc-spine-samples/markup/pod-samples/pod \ +# ./test/test-epub-validity.sh ./bin/spine-ldc +# +# Exit codes: +# 0 no fatals, and errors within the allowance +# 1 a fatal, or more errors than allowed +# 2 set-up problem (no $SpinePOD, no binary, no epubcheck, no samples) + +set -e + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +SPINE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" + +# What is known to be outstanding over the whole sample set. Both numbers are +# a ceiling, so a regression fails and an improvement does not; lower them in +# the commit that lowers the output, and say in that commit what went. +# +# There is no allowance for a fatal, and there must never be one. The last +# one - the_public_domain, chapter_2: a bare url ending a footnote body +# swallowed the "}~" close delimiter, so the note closed inside the link +# markup and left an unbalanced </a> behind - was a url boundary defect in +# the abstraction, and is fixed. +# +# No error is allowed either, and none is outstanding. Both numbers being +# zero is the point: anything at all that epubcheck calls an error is a +# regression, and there is no list to add to. +# +# No warning is outstanding either, though warnings are not counted here. +FATALS_ALLOWED=${SpineEpubFatalsAllowed:-0} +ERRORS_ALLOWED=${SpineEpubErrorsAllowed:-0} + +if [ -z "$SpinePOD" ]; then + echo "ERROR: \$SpinePOD is not set." >&2 + echo " e.g. SpinePOD=../../markup/sisudoc-spine-samples/markup/pod-samples/pod" >&2 + exit 2 +fi +SAMPLES_RAW="$SPINE_DIR/$SpinePOD" +if [ -d "$SAMPLES_RAW" ]; then + SAMPLES_DIR="$(cd "$SAMPLES_RAW" && pwd)" +else + SAMPLES_DIR="$SAMPLES_RAW" +fi +SPINE_BIN="${1:-}" +# find spine binary +if [ -z "$SPINE_BIN" ]; then + if [ -x "$SPINE_DIR/result/bin/spine" ]; then SPINE_BIN="$SPINE_DIR/result/bin/spine" + elif [ -x "$SPINE_DIR/bin/spine-ldc" ]; then SPINE_BIN="$SPINE_DIR/bin/spine-ldc" + elif [ -x "$SPINE_DIR/bin/spine" ]; then SPINE_BIN="$SPINE_DIR/bin/spine" + else + echo "ERROR: spine binary not found. Specify path as argument." >&2 + exit 2 + fi +fi +if [ ! -d "$SAMPLES_DIR" ]; then + echo "ERROR: samples not found: $SAMPLES_DIR" >&2 + exit 2 +fi +if ! command -v epubcheck >/dev/null 2>&1; then + echo "ERROR: epubcheck not on PATH." >&2 + echo " e.g. nix shell \"nixpkgs#epubcheck\" -c $0 $SPINE_BIN" >&2 + exit 2 +fi + +echo "spine binary: $SPINE_BIN" +echo "samples: $SAMPLES_DIR" + +OUT_DIR="$(mktemp -d)" +trap 'rm -rf "$OUT_DIR"' EXIT + +echo "Generating epubs ..." +"$SPINE_BIN" --epub --output="$OUT_DIR" "$SAMPLES_DIR"/* 2>&1 | tail -1 + +FATALS=0 +ERRORS=0 +DOCS=0 +for f in $(find "$OUT_DIR" -name '*.epub' | sort); do + DOCS=$((DOCS + 1)) + msg=$(epubcheck "$f" 2>&1 | grep -E '^Messages:' || true) + ff=$(echo "$msg" | sed -n 's/.*: \([0-9]*\) fatal.*/\1/p') + ee=$(echo "$msg" | sed -n 's/.*\/ \([0-9]*\) error.*/\1/p') + ff=${ff:-0}; ee=${ee:-0} + FATALS=$((FATALS + ff)) + ERRORS=$((ERRORS + ee)) + if [ "$ff" -gt 0 ]; then + echo " FATAL in $(basename "$f"): $ff" + epubcheck "$f" 2>&1 | grep -i 'FATAL' | head -3 | sed 's/^/ /' + fi +done + +echo "checked $DOCS epubs: $FATALS fatal (allowed $FATALS_ALLOWED), $ERRORS errors (allowed $ERRORS_ALLOWED)" + +if [ "$FATALS" -gt "$FATALS_ALLOWED" ]; then + echo "FAIL: $FATALS fatal(s), more than the $FATALS_ALLOWED allowed;" + echo " a fatal means a reader cannot parse that file at all" + exit 1 +fi +if [ "$ERRORS" -gt "$ERRORS_ALLOWED" ]; then + echo "FAIL: $ERRORS errors, more than the $ERRORS_ALLOWED allowed" + echo " if this is a deliberate change, say why and adjust ERRORS_ALLOWED" + exit 1 +fi +echo "PASS: $FATALS fatal and $ERRORS errors, both within what is recorded" +exit 0 +#+END_SRC * collection search db + cgi form ** test search db schema @@ -1317,126 +1611,3 @@ else exit 1 fi #+END_SRC - -** test epub validity - -#+HEADER: :tangle ../test/test-epub-validity.sh -#+HEADER: :tangle-mode (identity #o755) -#+HEADER: :shebang "#!/usr/bin/env sh" -#+BEGIN_SRC shell -# epubcheck over the sample set, as a regression guard. -# -# Every epub spine writes is checked with epubcheck. A *fatal* is never -# acceptable: it means a reader cannot parse the file at all. Errors are -# compared against a recorded allowance, so that the ones known to be -# outstanding do not fail the run while any new one does. -# -# The allowance is deliberately a single number rather than a list of -# messages: a list invites adding to it. When the number goes down, lower it -# here in the same commit that lowers it in the output. -# -# Usage: -# SpinePOD=../../markup/sisudoc-spine-samples/markup/pod-samples/pod \ -# ./test/test-epub-validity.sh ./bin/spine-ldc -# -# Exit codes: -# 0 no fatals, and errors within the allowance -# 1 a fatal, or more errors than allowed -# 2 set-up problem (no $SpinePOD, no binary, no epubcheck, no samples) - -set -e - -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -SPINE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" - -# What is known to be outstanding over the whole sample set. Both numbers are -# a ceiling, so a regression fails and an improvement does not; lower them in -# the commit that lowers the output, and say in that commit what went. -# -# There is no allowance for a fatal, and there must never be one. The last -# one - the_public_domain, chapter_2: a bare url ending a footnote body -# swallowed the "}~" close delimiter, so the note closed inside the link -# markup and left an unbalanced </a> behind - was a url boundary defect in -# the abstraction, and is fixed. -# -# No error is allowed either, and none is outstanding. Both numbers being -# zero is the point: anything at all that epubcheck calls an error is a -# regression, and there is no list to add to. -# -# No warning is outstanding either, though warnings are not counted here. -FATALS_ALLOWED=${SpineEpubFatalsAllowed:-0} -ERRORS_ALLOWED=${SpineEpubErrorsAllowed:-0} - -if [ -z "$SpinePOD" ]; then - echo "ERROR: \$SpinePOD is not set." >&2 - echo " e.g. SpinePOD=../../markup/sisudoc-spine-samples/markup/pod-samples/pod" >&2 - exit 2 -fi -SAMPLES_RAW="$SPINE_DIR/$SpinePOD" -if [ -d "$SAMPLES_RAW" ]; then - SAMPLES_DIR="$(cd "$SAMPLES_RAW" && pwd)" -else - SAMPLES_DIR="$SAMPLES_RAW" -fi -SPINE_BIN="${1:-}" -# find spine binary -if [ -z "$SPINE_BIN" ]; then - if [ -x "$SPINE_DIR/result/bin/spine" ]; then SPINE_BIN="$SPINE_DIR/result/bin/spine" - elif [ -x "$SPINE_DIR/bin/spine-ldc" ]; then SPINE_BIN="$SPINE_DIR/bin/spine-ldc" - elif [ -x "$SPINE_DIR/bin/spine" ]; then SPINE_BIN="$SPINE_DIR/bin/spine" - else - echo "ERROR: spine binary not found. Specify path as argument." >&2 - exit 2 - fi -fi -if [ ! -d "$SAMPLES_DIR" ]; then - echo "ERROR: samples not found: $SAMPLES_DIR" >&2 - exit 2 -fi -if ! command -v epubcheck >/dev/null 2>&1; then - echo "ERROR: epubcheck not on PATH." >&2 - echo " e.g. nix shell \"nixpkgs#epubcheck\" -c $0 $SPINE_BIN" >&2 - exit 2 -fi - -echo "spine binary: $SPINE_BIN" -echo "samples: $SAMPLES_DIR" - -OUT_DIR="$(mktemp -d)" -trap 'rm -rf "$OUT_DIR"' EXIT - -echo "Generating epubs ..." -"$SPINE_BIN" --epub --output="$OUT_DIR" "$SAMPLES_DIR"/* 2>&1 | tail -1 - -FATALS=0 -ERRORS=0 -DOCS=0 -for f in $(find "$OUT_DIR" -name '*.epub' | sort); do - DOCS=$((DOCS + 1)) - msg=$(epubcheck "$f" 2>&1 | grep -E '^Messages:' || true) - ff=$(echo "$msg" | sed -n 's/.*: \([0-9]*\) fatal.*/\1/p') - ee=$(echo "$msg" | sed -n 's/.*\/ \([0-9]*\) error.*/\1/p') - ff=${ff:-0}; ee=${ee:-0} - FATALS=$((FATALS + ff)) - ERRORS=$((ERRORS + ee)) - if [ "$ff" -gt 0 ]; then - echo " FATAL in $(basename "$f"): $ff" - epubcheck "$f" 2>&1 | grep -i 'FATAL' | head -3 | sed 's/^/ /' - fi -done - -echo "checked $DOCS epubs: $FATALS fatal (allowed $FATALS_ALLOWED), $ERRORS errors (allowed $ERRORS_ALLOWED)" - -if [ "$FATALS" -gt "$FATALS_ALLOWED" ]; then - echo "FAIL: $FATALS fatal(s), more than the $FATALS_ALLOWED allowed;" - echo " a fatal means a reader cannot parse that file at all" - exit 1 -fi -if [ "$ERRORS" -gt "$ERRORS_ALLOWED" ]; then - echo "FAIL: $ERRORS errors, more than the $ERRORS_ALLOWED allowed" - echo " if this is a deliberate change, say why and adjust ERRORS_ALLOWED" - exit 1 -fi -echo "PASS: $FATALS fatal and $ERRORS errors, both within what is recorded" -exit 0 -#+END_SRC diff --git a/test/run-tests.sh b/test/run-tests.sh index 2f27892..4d950dc 100755 --- a/test/run-tests.sh +++ b/test/run-tests.sh @@ -22,7 +22,8 @@ # 2 test-abstraction-ssp-roundtrip.sh the .ssp can be read back whole # 3 test-abstraction-db.sh the two serialisations agree # 4 test-abstraction-db-roundtrip.sh the .db can be read back whole -# 5 test-epub-validity.sh the epubs a reader can actually open +# 5 test-ocn-alignment.sh the languages agree on their ocns +# 6 test-epub-validity.sh the epubs a reader can actually open # # None of them calls another, and each can be run on its own. The order # matters only for reading the result: @@ -36,6 +37,11 @@ # - 3 and 4 generate both artefacts themselves and depend on nothing # committed. Since the database is built from the .ssp, they are checks # on the reader rather than on two independent writers. +# - 5 is about the sample documents rather than about a serialisation: it +# asserts that spine reports the two known divergences in live-manual's +# translations and reports nothing about the other documents. It will +# fail if those translations are mended, which is the right moment to +# hear about it. set -e @@ -61,6 +67,7 @@ TESTS="test-abstraction-ssp.sh test-abstraction-ssp-roundtrip.sh test-abstraction-db.sh test-abstraction-db-roundtrip.sh +test-ocn-alignment.sh test-epub-validity.sh" PASSED=0 diff --git a/test/test-abstraction-db.sh b/test/test-abstraction-db.sh index 2fac414..78c27b9 100755 --- a/test/test-abstraction-db.sh +++ b/test/test-abstraction-db.sh @@ -234,7 +234,11 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do done # the header blocks: every @source/@meta/@make/@doc_has property in the .ssp - # must be in the database under its own key, with the same value. this is + # must be in the database under its own key, with the same value. the three + # excluded keys are the database's own notes about the artefact rather than + # properties of the document: what it was built from (source.filename, + # source.ssp_digest) and how its languages compared once they had all been + # read (translation.ocn_aligned), which no single .ssp can state. this is # what catches a header property that reaches one artefact and not the # other, which a count of objects cannot see. awk ' @@ -249,7 +253,7 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do ' "$ssp" | sort > "$OUT_DIR/.sspmeta.$$" sqlite3 -separator "$(printf '\t')" "$db" \ "select key, value from metadata where doc_id=$did and key not in - ('source.filename','source.ssp_digest');" \ + ('source.filename','source.ssp_digest','translation.ocn_aligned');" \ 2>/dev/null | sort > "$OUT_DIR/.dbmeta.$$" if ! diff -q "$OUT_DIR/.sspmeta.$$" "$OUT_DIR/.dbmeta.$$" > /dev/null 2>&1; then [ $doc_fail -eq 0 ] && echo "$base" diff --git a/test/test-ocn-alignment.sh b/test/test-ocn-alignment.sh new file mode 100755 index 0000000..f00a8ec --- /dev/null +++ b/test/test-ocn-alignment.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env sh +# test-ocn-alignment.sh +# +# The languages of a document share their object numbering, and spine warns +# when they stop doing so. This checks that the warning appears where it +# should and, just as importantly, nowhere else. +# +# The sample set is the fixture, and it is a good one because it contains +# both answers: +# +# live-manual ten languages, two of which diverge. +# ja has two fewer numbered objects than en. A missing +# translation of a short paragraph; everything below it +# is renumbered. +# pl has the same count, but the object at ocn 313 is a +# paragraph in en and a heading in pl. A heading given as +# running text, or running text given as a heading. +# everything else single language, or aligned. Silent. +# +# A check that only ever fires is as useless as one that never does, so the +# silence over the other documents is asserted too. +# +# It also asserts that --strict turns the warning into a non-zero exit, and +# that without --strict the run still succeeds: a document with a diverging +# translation is still a document. +# +# Usage: +# SpinePOD=../../markup/sisudoc-spine-samples/markup/pod-samples/pod \ +# ./test/test-ocn-alignment.sh ./bin/spine-ldc +# +# Exit codes: +# 0 the check reports what it should, and nothing else +# 1 it does not (printed) +# 2 set-up problem (no $SpinePOD, no binary, no samples) + +set -e + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +SPINE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" + +if [ -z "$SpinePOD" ]; then + echo "ERROR: \$SpinePOD is not set." >&2 + echo " e.g. SpinePOD=../../markup/sisudoc-spine-samples/markup/pod-samples/pod" >&2 + exit 2 +fi +SAMPLES_RAW="$SPINE_DIR/$SpinePOD" +if [ -d "$SAMPLES_RAW" ]; then + SAMPLES_DIR="$(cd "$SAMPLES_RAW" && pwd)" +else + SAMPLES_DIR="$SAMPLES_RAW" +fi +if [ ! -d "$SAMPLES_DIR" ]; then + echo "ERROR: sample documents not found at $SAMPLES_DIR" >&2 + exit 2 +fi + +SPINE_BIN="${1:-}" +if [ -z "$SPINE_BIN" ]; then + if [ -x "$SPINE_DIR/result/bin/spine" ]; then SPINE_BIN="$SPINE_DIR/result/bin/spine" + elif [ -x "$SPINE_DIR/bin/spine-ldc" ]; then SPINE_BIN="$SPINE_DIR/bin/spine-ldc" + elif [ -x "$SPINE_DIR/bin/spine" ]; then SPINE_BIN="$SPINE_DIR/bin/spine" + else + echo "ERROR: spine binary not found. Specify path as argument." >&2 + exit 2 + fi +fi + +OUT_DIR="$SCRIPT_DIR/current-ocn-alignment" +rm -rf "$OUT_DIR" +mkdir -p "$OUT_DIR" + +echo "spine binary: $SPINE_BIN" +echo "samples: $SAMPLES_DIR" + +FAILURES=0 +fail() { echo " FAIL: $*"; FAILURES=$((FAILURES + 1)); } + +# the whole sample set, abstraction only: the check does not need outputs +echo "Abstracting the sample set ..." +set +e +$SPINE_BIN --show-abstraction --skip-output \ + --output="$OUT_DIR" "$SAMPLES_DIR"/* \ + > "$OUT_DIR/run.out" 2> "$OUT_DIR/run.err" +RUN_RC=$? +set -e + +# without --strict a diverging document is still built +if [ "$RUN_RC" -ne 0 ]; then + fail "the run exited $RUN_RC without --strict; a warning must not fail a run" + head -5 "$OUT_DIR/run.err" +fi + +# what it said +if ! grep -q "^WARNING ocn alignment: live-manual$" "$OUT_DIR/run.err"; then + fail "no ocn alignment warning for live-manual" +fi +if ! grep -q "^ \[ja\] differs from \[en\]$" "$OUT_DIR/run.err"; then + fail "ja not reported as differing from en" +fi +if ! grep -q "numbered objects: en 1019, ja 1017 (-2)" "$OUT_DIR/run.err"; then + fail "ja's object count difference not reported as -2" +fi +if ! grep -q "^ \[pl\] differs from \[en\]$" "$OUT_DIR/run.err"; then + fail "pl not reported as differing from en" +fi +if ! grep -q "ocn 313: para in en, heading:2 in pl" "$OUT_DIR/run.err"; then + fail "pl's substituted heading at ocn 313 not reported" +fi + +# and what it did not say: exactly one document warned about, exactly two +# languages of it +n_docs=$(grep -c "^WARNING ocn alignment: " "$OUT_DIR/run.err" || true) +if [ "$n_docs" != "1" ]; then + fail "$n_docs documents warned about, expected 1" + grep "^WARNING ocn alignment: " "$OUT_DIR/run.err" | sed 's/^/ /' +fi +n_langs=$(grep -c "^ \[.*\] differs from " "$OUT_DIR/run.err" || true) +if [ "$n_langs" != "2" ]; then + fail "$n_langs languages reported, expected 2 (ja and pl)" + grep "^ \[.*\] differs from " "$OUT_DIR/run.err" | sed 's/^/ /' +fi + +# --strict makes the same run fail +set +e +$SPINE_BIN --show-abstraction --skip-output --strict \ + --output="$OUT_DIR/strict" "$SAMPLES_DIR"/live-manual \ + > "$OUT_DIR/strict.out" 2> "$OUT_DIR/strict.err" +STRICT_RC=$? +set -e +if [ "$STRICT_RC" -eq 0 ]; then + fail "--strict exited 0 on a document whose languages diverge" +fi + +# and does not fail a document whose languages agree, or one that has only +# the one language +set +e +$SPINE_BIN --show-abstraction --skip-output --strict \ + --output="$OUT_DIR/strict-ok" "$SAMPLES_DIR"/the_wealth_of_networks.yochai_benkler \ + > "$OUT_DIR/strict-ok.out" 2> "$OUT_DIR/strict-ok.err" +STRICT_OK_RC=$? +set -e +if [ "$STRICT_OK_RC" -ne 0 ]; then + fail "--strict exited $STRICT_OK_RC on a single language document" + head -5 "$OUT_DIR/strict-ok.err" +fi + +rm -rf "$OUT_DIR" + +if [ "$FAILURES" -eq 0 ]; then + echo "PASS: ja and pl reported on live-manual, silent elsewhere, --strict fails" + exit 0 +else + echo "FAIL: $FAILURES check(s)" + exit 1 +fi |