diff options
| -rw-r--r-- | org/spine.org | 56 | ||||
| -rw-r--r-- | org/tests_for_document_abstraction_shell_scripts.org | 9 | ||||
| -rw-r--r-- | src/sisudoc/spine.d | 56 | ||||
| -rwxr-xr-x | test/test-abstraction-ssp.sh | 9 |
4 files changed, 60 insertions, 70 deletions
diff --git a/org/spine.org b/org/spine.org index b60fd8d..c26944c 100644 --- a/org/spine.org +++ b/org/spine.org @@ -482,7 +482,7 @@ auto helpInfo = getopt(args, "ssp-round-trip", "=/path/to/file.ssp read a .ssp back and re-emit it on stdout", &settings["ssp-round-trip"], "db-round-trip", "=/path/to/file.ocda.db read it back and emit .ssp on stdout", &settings["db-round-trip"], "abstraction-source", "=/path/to/(.sst|pod|.ssp|.ocda.db) identify it, and load it if it is an abstraction", &settings["abstraction-source"], - "parallel", "parallelisation", &opts["parallel"], + "parallel", "parallelise document processing (opt-in; slower than serial)", &opts["parallel"], "parallel-subprocesses", "nested parallelisation", &opts["parallel-subprocesses"], "pdf", "latex output for pdfs", &opts["pdf"], "pdf-color-links", "mono or color links for pdfs", &opts["pdf-color-links"], @@ -498,7 +498,7 @@ auto helpInfo = getopt(args, "section-endnotes", "document endnotes (default)", &opts["section_endnotes"], "section-glossary", "document glossary (default)", &opts["section_glossary"], "section-toc", "table of contents (default)", &opts["section_toc"], - "serial", "serial processing", &opts["serial"], + "serial", "serial document processing (default)", &opts["serial"], "skip-output", "skip output", &opts["skip-output"], "show-abstraction", "show document abstraction (write .ssp file)", &opts["show-abstraction"], "show-config", "show config", &opts["show-config"], @@ -1026,46 +1026,40 @@ struct OptActions { @trusted string webserver_http() { return settings["www-http"]; } + /+ ↓ serial is the behaviour, --parallel is the option. + . + In-process parallelism costs rather than pays. Measured on 35 sample + documents, 16 cores, --text --html --epub --latex: 14.42s wall and + 154s cpu in parallel against 10.57s wall and 12.9s cpu serially. The + abstraction stage on its own is the same shape, 4.8s against 4.0s for + eleven times the cpu. The cost tracks the number of cores made + available rather than the work done - 3.7s cpu pinned to one core, + 46.8s on sixteen, for identical output - which is threads burning + time without progressing. Separate processes over the same documents + scale as expected (4.2s to 1.3s, 16 processes, output byte + identical), so it is not the work and not the thread count: it is + what the threads share, most likely the GC allocation lock, which is + spun rather than slept on. Not confirmed with a profiler. + . + So --parallel no longer follows from asking for output. It is kept, + not removed: test-abstraction-ssp.sh compares a parallel run against + a serial one, and that comparison is what guards the .ssp write race. + An instrument, until the sharing is understood and fixed. + +/ @trusted bool parallelise() { bool _is; if (opts["serial"] == true) { _is = false; } else if ( + /+ ↓ these cannot run in parallel however asked: --curate aggregates + across documents, and the shared sqlite db has the one writer + +/ sqlite_shared_db_action || source_or_pod ) { _is = false; } else if (opts["parallel"] == true) { _is = true; - if ( - sqlite_shared_db_action - || source_or_pod - ) { - _is = false; - } - } else if ( - /+ ↓ show_abstraction and ocda_db belong here for the same reason as - the rest: each writes one file per document per language and - shares no handle. --pod2 and --source, which also set - show_abstraction, and the shared sqlite db actions are taken out - by the guard above before this list is reached, so listing them - here does not parallelise those - +/ - opts["abstraction"] - || show_abstraction - || ocda_db - || concordance - || curate - || html - || epub - || odt - || latex - || manifest - || sqlite_discrete - || text - || skel - ) { - _is = true; } else { _is = false; } return _is; } diff --git a/org/tests_for_document_abstraction_shell_scripts.org b/org/tests_for_document_abstraction_shell_scripts.org index 199e109..7228170 100644 --- a/org/tests_for_document_abstraction_shell_scripts.org +++ b/org/tests_for_document_abstraction_shell_scripts.org @@ -233,10 +233,11 @@ echo "Generating current .ssp files..." rm -rf "$TMP_DIR" mkdir -p "$TMP_DIR" # this run has to be the parallel one: the serial run below is what it is -# compared against, and two serial runs compare nothing. --show-abstraction -# now parallelises on its own, and --abstraction is kept as well so that the -# test does not go quietly serial if that ever changes. -$SPINE_BIN --abstraction --show-abstraction --skip-output --output="$SCRIPT_DIR/current" "$SAMPLES_DIR"/* 2>&1 | tail -1 +# compared against, and two serial runs compare nothing. Serial is spine's +# behaviour now and --parallel is the option, so it has to be asked for +# here by name. Drop it and this test stops guarding the .ssp write race +# without any test failing to say so. +$SPINE_BIN --parallel --show-abstraction --skip-output --output="$SCRIPT_DIR/current" "$SAMPLES_DIR"/* 2>&1 | tail -1 # flatten find "$SCRIPT_DIR/current" -name "*.ssp" ! -path "$TMP_DIR/*" -exec mv {} "$TMP_DIR/" \; find "$SCRIPT_DIR/current" -mindepth 1 -type d -empty -delete 2>/dev/null || true diff --git a/src/sisudoc/spine.d b/src/sisudoc/spine.d index ed55298..9bbe10f 100644 --- a/src/sisudoc/spine.d +++ b/src/sisudoc/spine.d @@ -282,7 +282,7 @@ string program_name = "spine"; "ssp-round-trip", "=/path/to/file.ssp read a .ssp back and re-emit it on stdout", &settings["ssp-round-trip"], "db-round-trip", "=/path/to/file.ocda.db read it back and emit .ssp on stdout", &settings["db-round-trip"], "abstraction-source", "=/path/to/(.sst|pod|.ssp|.ocda.db) identify it, and load it if it is an abstraction", &settings["abstraction-source"], - "parallel", "parallelisation", &opts["parallel"], + "parallel", "parallelise document processing (opt-in; slower than serial)", &opts["parallel"], "parallel-subprocesses", "nested parallelisation", &opts["parallel-subprocesses"], "pdf", "latex output for pdfs", &opts["pdf"], "pdf-color-links", "mono or color links for pdfs", &opts["pdf-color-links"], @@ -298,7 +298,7 @@ string program_name = "spine"; "section-endnotes", "document endnotes (default)", &opts["section_endnotes"], "section-glossary", "document glossary (default)", &opts["section_glossary"], "section-toc", "table of contents (default)", &opts["section_toc"], - "serial", "serial processing", &opts["serial"], + "serial", "serial document processing (default)", &opts["serial"], "skip-output", "skip output", &opts["skip-output"], "show-abstraction", "show document abstraction (write .ssp file)", &opts["show-abstraction"], "show-config", "show config", &opts["show-config"], @@ -813,46 +813,40 @@ string program_name = "spine"; @trusted string webserver_http() { return settings["www-http"]; } + /+ ↓ serial is the behaviour, --parallel is the option. + . + In-process parallelism costs rather than pays. Measured on 35 sample + documents, 16 cores, --text --html --epub --latex: 14.42s wall and + 154s cpu in parallel against 10.57s wall and 12.9s cpu serially. The + abstraction stage on its own is the same shape, 4.8s against 4.0s for + eleven times the cpu. The cost tracks the number of cores made + available rather than the work done - 3.7s cpu pinned to one core, + 46.8s on sixteen, for identical output - which is threads burning + time without progressing. Separate processes over the same documents + scale as expected (4.2s to 1.3s, 16 processes, output byte + identical), so it is not the work and not the thread count: it is + what the threads share, most likely the GC allocation lock, which is + spun rather than slept on. Not confirmed with a profiler. + . + So --parallel no longer follows from asking for output. It is kept, + not removed: test-abstraction-ssp.sh compares a parallel run against + a serial one, and that comparison is what guards the .ssp write race. + An instrument, until the sharing is understood and fixed. + +/ @trusted bool parallelise() { bool _is; if (opts["serial"] == true) { _is = false; } else if ( + /+ ↓ these cannot run in parallel however asked: --curate aggregates + across documents, and the shared sqlite db has the one writer + +/ sqlite_shared_db_action || source_or_pod ) { _is = false; } else if (opts["parallel"] == true) { _is = true; - if ( - sqlite_shared_db_action - || source_or_pod - ) { - _is = false; - } - } else if ( - /+ ↓ show_abstraction and ocda_db belong here for the same reason as - the rest: each writes one file per document per language and - shares no handle. --pod2 and --source, which also set - show_abstraction, and the shared sqlite db actions are taken out - by the guard above before this list is reached, so listing them - here does not parallelise those - +/ - opts["abstraction"] - || show_abstraction - || ocda_db - || concordance - || curate - || html - || epub - || odt - || latex - || manifest - || sqlite_discrete - || text - || skel - ) { - _is = true; } else { _is = false; } return _is; } diff --git a/test/test-abstraction-ssp.sh b/test/test-abstraction-ssp.sh index 3b2217b..b63c6e4 100755 --- a/test/test-abstraction-ssp.sh +++ b/test/test-abstraction-ssp.sh @@ -105,10 +105,11 @@ echo "Generating current .ssp files..." rm -rf "$TMP_DIR" mkdir -p "$TMP_DIR" # this run has to be the parallel one: the serial run below is what it is -# compared against, and two serial runs compare nothing. --show-abstraction -# now parallelises on its own, and --abstraction is kept as well so that the -# test does not go quietly serial if that ever changes. -$SPINE_BIN --abstraction --show-abstraction --skip-output --output="$SCRIPT_DIR/current" "$SAMPLES_DIR"/* 2>&1 | tail -1 +# compared against, and two serial runs compare nothing. Serial is spine's +# behaviour now and --parallel is the option, so it has to be asked for +# here by name. Drop it and this test stops guarding the .ssp write race +# without any test failing to say so. +$SPINE_BIN --parallel --show-abstraction --skip-output --output="$SCRIPT_DIR/current" "$SAMPLES_DIR"/* 2>&1 | tail -1 # flatten find "$SCRIPT_DIR/current" -name "*.ssp" ! -path "$TMP_DIR/*" -exec mv {} "$TMP_DIR/" \; find "$SCRIPT_DIR/current" -mindepth 1 -type d -empty -delete 2>/dev/null || true |