diff options
Diffstat (limited to 'src/sisudoc')
| -rw-r--r-- | src/sisudoc/spine.d | 56 |
1 files changed, 25 insertions, 31 deletions
diff --git a/src/sisudoc/spine.d b/src/sisudoc/spine.d index ed55298..9bbe10f 100644 --- a/src/sisudoc/spine.d +++ b/src/sisudoc/spine.d @@ -282,7 +282,7 @@ string program_name = "spine"; "ssp-round-trip", "=/path/to/file.ssp read a .ssp back and re-emit it on stdout", &settings["ssp-round-trip"], "db-round-trip", "=/path/to/file.ocda.db read it back and emit .ssp on stdout", &settings["db-round-trip"], "abstraction-source", "=/path/to/(.sst|pod|.ssp|.ocda.db) identify it, and load it if it is an abstraction", &settings["abstraction-source"], - "parallel", "parallelisation", &opts["parallel"], + "parallel", "parallelise document processing (opt-in; slower than serial)", &opts["parallel"], "parallel-subprocesses", "nested parallelisation", &opts["parallel-subprocesses"], "pdf", "latex output for pdfs", &opts["pdf"], "pdf-color-links", "mono or color links for pdfs", &opts["pdf-color-links"], @@ -298,7 +298,7 @@ string program_name = "spine"; "section-endnotes", "document endnotes (default)", &opts["section_endnotes"], "section-glossary", "document glossary (default)", &opts["section_glossary"], "section-toc", "table of contents (default)", &opts["section_toc"], - "serial", "serial processing", &opts["serial"], + "serial", "serial document processing (default)", &opts["serial"], "skip-output", "skip output", &opts["skip-output"], "show-abstraction", "show document abstraction (write .ssp file)", &opts["show-abstraction"], "show-config", "show config", &opts["show-config"], @@ -813,46 +813,40 @@ string program_name = "spine"; @trusted string webserver_http() { return settings["www-http"]; } + /+ ↓ serial is the behaviour, --parallel is the option. + . + In-process parallelism costs rather than pays. Measured on 35 sample + documents, 16 cores, --text --html --epub --latex: 14.42s wall and + 154s cpu in parallel against 10.57s wall and 12.9s cpu serially. The + abstraction stage on its own is the same shape, 4.8s against 4.0s for + eleven times the cpu. The cost tracks the number of cores made + available rather than the work done - 3.7s cpu pinned to one core, + 46.8s on sixteen, for identical output - which is threads burning + time without progressing. Separate processes over the same documents + scale as expected (4.2s to 1.3s, 16 processes, output byte + identical), so it is not the work and not the thread count: it is + what the threads share, most likely the GC allocation lock, which is + spun rather than slept on. Not confirmed with a profiler. + . + So --parallel no longer follows from asking for output. It is kept, + not removed: test-abstraction-ssp.sh compares a parallel run against + a serial one, and that comparison is what guards the .ssp write race. + An instrument, until the sharing is understood and fixed. + +/ @trusted bool parallelise() { bool _is; if (opts["serial"] == true) { _is = false; } else if ( + /+ ↓ these cannot run in parallel however asked: --curate aggregates + across documents, and the shared sqlite db has the one writer + +/ sqlite_shared_db_action || source_or_pod ) { _is = false; } else if (opts["parallel"] == true) { _is = true; - if ( - sqlite_shared_db_action - || source_or_pod - ) { - _is = false; - } - } else if ( - /+ ↓ show_abstraction and ocda_db belong here for the same reason as - the rest: each writes one file per document per language and - shares no handle. --pod2 and --source, which also set - show_abstraction, and the shared sqlite db actions are taken out - by the guard above before this list is reached, so listing them - here does not parallelise those - +/ - opts["abstraction"] - || show_abstraction - || ocda_db - || concordance - || curate - || html - || epub - || odt - || latex - || manifest - || sqlite_discrete - || text - || skel - ) { - _is = true; } else { _is = false; } return _is; } |