From db92f9af3293f8009543448a100b08b7a2f2ffbe Mon Sep 17 00:00:00 2001 From: Ralph Amissah Date: Mon, 23 Apr 2018 12:07:22 -0400 Subject: 0.26.1 single sqlite file, premature? --- org/default_paths.org | 54 +- org/default_regex.org | 2 +- org/output_hub.org | 34 +- org/output_sqlite.org | 280 +++--- org/output_sqlite_discrete.org | 1359 --------------------------- org/sdp.org | 2 +- src/sdp/meta/rgx.d | 2 +- src/sdp/output/hub.d | 35 +- src/sdp/output/paths_source.d | 54 +- src/sdp/output/rgx.d | 2 +- src/sdp/output/sqlite.d | 1930 +++++++++++++++++++------------------- src/sdp/output/sqlite_discrete.d | 1023 -------------------- views/version.txt | 2 +- 13 files changed, 1331 insertions(+), 3448 deletions(-) delete mode 100644 org/output_sqlite_discrete.org delete mode 100644 src/sdp/output/sqlite_discrete.d diff --git a/org/default_paths.org b/org/default_paths.org index af96bea..308475a 100644 --- a/org/default_paths.org +++ b/org/default_paths.org @@ -204,6 +204,7 @@ template PathMatters() { return Pod_(); } auto src() { + string _sep = "␣"; struct SRC_ { auto is_pod() { auto _src_is_pod = (_manifest.pod_manifest_path.length > 0) ? true : false; @@ -212,19 +213,70 @@ template PathMatters() { auto path_and_fn() { return _fns; } + auto pod_name() { /+ work on +/ + auto _pod_name = (is_pod) + ? _manifest.pod_manifest_path + : ""; + return _pod_name; + } auto filename() { auto _fn = (path_and_fn).baseName; return _fn; } + auto filename_base() { + auto _fn = filename.stripExtension; + return _fn; + } + auto filename_extension() { + auto _ext = filename.match(rgx.src_pth_sst_or_ssm).captures["extension"]; + return _ext; + } auto lng() { string _k; - if (auto m = (path_and_fn).match(rgx.language_code_and_filename)) { + if (auto m = path_and_fn.match(rgx.language_code_and_filename)) { _k = m.captures[1]; } else { _k = "en"; } return _k; } + auto docname_composite_unique_per_src_doc() { + /+ + z pod name if any + src filename + lng code + filename ~ "." ~ lng + * unique per src doc + used by + - sqlite discrete index (multilingual, each language of a document) + +/ + string _fn; + if (pod_name.baseName == filename_base) { + _fn = filename_base ~ _sep ~ filename_extension ~ _sep ~ lng; + } else if (!(pod_name.empty)) { + _fn = pod_name.baseName ~ _sep ~ filename_base ~ _sep ~ filename_extension ~ _sep ~ lng; + } else { + _fn = "_" ~ _sep ~ filename_base ~ _sep ~ filename_extension ~ _sep ~ lng; + } + return _fn; + } + auto docname_composite_unique_per_src_pod() { + /+ + z pod name if any + src filename (without lng code) + filename ~ _sep ~ lng + * unique per src pod + used by + - sisupod (multilingual collection) + - sqlite discrete index (multilingual collection) + +/ + string _fn; + if (pod_name.baseName == filename_base) { + _fn = filename_base ~ _sep ~ filename_extension; + } else if (!(pod_name.empty)) { + _fn = pod_name.baseName ~ _sep ~ filename_base ~ _sep ~ filename_extension; + } else { + _fn = "_" ~ _sep ~ filename_base ~ _sep ~ filename_extension; + } + return _fn; + } auto language() { return lng(); } diff --git a/org/default_regex.org b/org/default_regex.org index c848109..e824577 100644 --- a/org/default_regex.org +++ b/org/default_regex.org @@ -420,7 +420,7 @@ static special_markup_chars = ctRegex!(`[【】〖〗 #+name: prgmkup_rgx #+BEGIN_SRC d -static src_pth_sst_or_ssm = ctRegex!(`^(?P[/]?(?:[a-zA-Z0-9._-]+/)*)(?P[a-zA-Z0-9._-]+[.]ss[tm])$`); +static src_pth_sst_or_ssm = ctRegex!(`^(?P[/]?(?:[a-zA-Z0-9._-]+/)*)(?P[a-zA-Z0-9._-]+[.](?Pss[tm]))$`); static src_pth_pod_sst_or_ssm = ctRegex!(`^(?P[/]?(?:[a-zA-Z0-9._-]+/)*)media/text/[a-z]{2}/(?P[a-zA-Z0-9._-]+[.]ss[tm])$`); static src_pth_contents = ctRegex!(`^(?P[/]?(?:[a-zA-Z0-9._-]+/)*)(?P[a-zA-Z0-9._-]+)/sisupod[.]manifest$`); static src_pth_pod_root = ctRegex!(`^(?P(?:[/]?(?:[a-zA-Z0-9._-]+/)*)(sisupod))$`); diff --git a/org/output_hub.org b/org/output_hub.org index 6d62477..3129b72 100644 --- a/org/output_hub.org +++ b/org/output_hub.org @@ -29,6 +29,35 @@ template outputHub() { void outputHub(D,I)(D doc_abstraction, I doc_matters) { mixin SiSUoutputRgxInit; static auto rgx = Rgx(); + debug (substitutions) { + enum Substitute { match, markup, html, } + writeln(__LINE__, ":", __FILE__, ": DEBUG substitutions:"); + writeln("Doc Title: ", doc_matters.conf_make_meta.meta.title_full); + if (doc_matters.conf_make_meta.make.bold) { + writeln("regex to match: ", doc_matters.conf_make_meta.make.bold[Substitute.match]); + writeln("substitution to make: ", doc_matters.conf_make_meta.make.bold[Substitute.html]); + + auto _w = "1. Debian test string. Debian again. (the problem) do not use initialized only not repopulated" + .replaceAll( + regex(doc_matters.conf_make_meta.make.bold[Substitute.match]), + doc_matters.conf_make_meta.make.bold[Substitute.html] + ); + writeln(_w); + + writeln("2a. Debian test string. Debian again. (the problem) do not use initialized only not repopulated" + .replaceAll( + regex(doc_matters.conf_make_meta.make.bold[Substitute.match]), + doc_matters.conf_make_meta.make.bold[Substitute.html] + )); + + auto _v = regex(doc_matters.conf_make_meta.make.bold[Substitute.match]); + writeln("2b. Debian test string. Debian again. (the problem) do not use initialized only not repopulated" + .replaceAll( + _v, + doc_matters.conf_make_meta.make.bold[Substitute.html] + )); + } + } <> } } @@ -50,7 +79,6 @@ import sdp.output, sdp.output.epub3, sdp.output.html, sdp.output.sqlite, - sdp.output.sqlite_discrete, sdp.output.xmls, sdp.output.source_sisupod, sdp.output.create_zip_file, @@ -202,7 +230,7 @@ if (doc_matters.opt.action.sqlite_discrete) { if ((doc_matters.opt.action.verbose)) { writeln("sqlite processing... "); } - SQLiteDiscreteBuildTablesAndPopulate!()(doc_abstraction, doc_matters); + SQLiteHubDiscreteBuildTablesAndPopulate!()(doc_abstraction, doc_matters); } #+END_SRC @@ -214,7 +242,7 @@ if (doc_matters.opt.action.sqlite_update) { if ((doc_matters.opt.action.verbose)) { writeln("sqlite processing... "); } - SQLiteBuildTablesAndPopulate!()(doc_abstraction, doc_matters); + SQLiteHubBuildTablesAndPopulate!()(doc_abstraction, doc_matters); } #+END_SRC diff --git a/org/output_sqlite.org b/org/output_sqlite.org index 7fb843d..566775b 100644 --- a/org/output_sqlite.org +++ b/org/output_sqlite.org @@ -17,10 +17,67 @@ * sql ** 0. module, templates +*** hub +**** collection + #+BEGIN_SRC d :tangle ../src/sdp/output/sqlite.d module sdp.output.sqlite; <> +import d2sqlite3; +import std.typecons : Nullable; +mixin SiSUoutputRgxInit; +static auto rgx = Rgx(); long _metadata_tid_lastrowid; +template SQLiteHubBuildTablesAndPopulate() { + void SQLiteHubBuildTablesAndPopulate(D,I)( + auto ref const D doc_abstraction, + auto ref I doc_matters, + ) { + auto pth_sqlite = SiSUpathsSQLite!()(doc_matters.output_path); + pth_sqlite.base.mkdirRecurse; + auto db = Database(pth_sqlite.sqlite_file(doc_matters.env.pwd.baseName)); + template SQLiteDbStatementComposite() { + void SQLiteDbStatementComposite(Db,D,I)( + auto ref Db db, + auto ref const D doc_abstraction, + auto ref I doc_matters, + ) { + <> + } + } + SQLiteDbStatementComposite!()(db, doc_abstraction, doc_matters); + } +} +#+END_SRC + +**** discrete + +#+BEGIN_SRC d :tangle ../src/sdp/output/sqlite.d +template SQLiteHubDiscreteBuildTablesAndPopulate() { + void SQLiteHubDiscreteBuildTablesAndPopulate(D,I)( + auto ref const D doc_abstraction, + auto ref I doc_matters, + ) { + auto pth_sqlite = SiSUpathsSQLiteDiscrete!()(doc_matters.output_path, doc_matters.src.language); + pth_sqlite.base.mkdirRecurse; + auto db = Database(pth_sqlite.sqlite_file(doc_matters.src.filename)); + template SQLiteDiscreteDbStatementComposite() { + void SQLiteDiscreteDbStatementComposite(Db,D,I)( + auto ref Db db, + auto ref const D doc_abstraction, + auto ref I doc_matters, + ) { + <> + } + } + SQLiteDiscreteDbStatementComposite!()(db, doc_abstraction, doc_matters); + } +} +#+END_SRC + +*** db run + +#+BEGIN_SRC d :tangle ../src/sdp/output/sqlite.d template SQLiteDbRun() { void SQLiteDbRun(Db,St,O)( auto ref Db db, @@ -37,7 +94,9 @@ template SQLiteDbRun() { db_statement ~ "\nCOMMIT;\n" ); - if (!(opt_action.sqlite_create)) { + if (!(opt_action.sqlite_discrete) + && !(opt_action.sqlite_create) + ) { _metadata_tid_lastrowid = db.lastInsertRowid(); writeln("last row id: ", _metadata_tid_lastrowid); } @@ -54,83 +113,70 @@ template SQLiteDbRun() { } } } -template SQLiteBuildTablesAndPopulate() { - void SQLiteBuildTablesAndPopulate(D,I)( - auto ref const D doc_abstraction, +#+END_SRC + +*** munge + +#+BEGIN_SRC d :tangle ../src/sdp/output/sqlite.d +template SQLinsertDelimiter() { + auto SQLinsertDelimiter(string _txt) { + _txt = _txt + .replaceAll(rgx.quotation_mark_sql_insert_delimiter, "$0$0"); + return _txt; + } +} +template SQLiteFormatAndLoadObject() { + auto SQLiteFormatAndLoadObject(I)( auto ref I doc_matters, ) { - import d2sqlite3; - import std.typecons : Nullable; mixin SiSUoutputRgxInit; - static auto rgx = Rgx(); - auto pth_sqlite = SiSUpathsSQLite!()(doc_matters.output_path); - pth_sqlite.base.mkdirRecurse; - auto db = Database(pth_sqlite.sqlite_file(doc_matters.env.pwd.baseName)); - template SQLiteDbStatementComposite() { - void SQLiteDbStatementComposite(Db,D,I)( - auto ref Db db, - auto ref const D doc_abstraction, - auto ref I doc_matters, - ) { - <> - } - } - template SQLinsertDelimiter() { - auto SQLinsertDelimiter(string _txt) { - _txt = _txt - .replaceAll(rgx.quotation_mark_sql_insert_delimiter, "$0$0"); - return _txt; - } + struct sqlite_format_and_load_objects { + <> + <> + <> + <> + <> } - template SQLiteFormatAndLoadObject() { - auto SQLiteFormatAndLoadObject(I)( - auto ref I doc_matters, - ) { - mixin SiSUoutputRgxInit; - struct sqlite_format_and_load_objects { - <> - <> - <> - <> - <> - } - return sqlite_format_and_load_objects(); - } - } - template SQLiteTablesReCreate() { - string SQLiteTablesReCreate()() { - string _sql_instruct; - _sql_instruct = format(q"¶ - <> - <> - <> - <> - <> - ¶",); - return _sql_instruct; - } - } - template SQLiteInsertMetadata() { - string SQLiteInsertMetadata(I)( - auto ref I doc_matters, - ) { - <> - return _insert_metadata; - } - } - template SQLiteInsertDocObjectsLoop() { - string SQLiteInsertDocObjectsLoop(P,I)( - auto ref P doc_parts, - auto ref I _metadata_tid, - ) { - string insertDocObjectsRow(O)(O obj) { - <> - return _insert_doc_objects_row; - } - <> - } + return sqlite_format_and_load_objects(); + } +} +#+END_SRC + +*** sqlite instructions + +#+BEGIN_SRC d :tangle ../src/sdp/output/sqlite.d +template SQLiteTablesReCreate() { + string SQLiteTablesReCreate()() { + string _sql_instruct; + _sql_instruct = format(q"¶ + <> + <> + <> + <> + <> + ¶",); + return _sql_instruct; + } +} +template SQLiteInsertMetadata() { + string SQLiteInsertMetadata(I)( + auto ref I doc_matters, + ) { + <> + return _insert_metadata; + } +} +template SQLiteInsertDocObjectsLoop() { + string SQLiteInsertDocObjectsLoop(D,I,X)( + auto ref const D doc_abstraction, + auto ref I doc_matters, + auto ref X _metadata_tid, + ) { + string insertDocObjectsRow(O)(O obj) { + <> + return _insert_doc_objects_row; } - SQLiteDbStatementComposite!()(db, doc_abstraction, doc_matters); + <> } } template SQLiteTablesCreate() { @@ -169,30 +215,45 @@ template SQLiteTablesDrop() { #+END_SRC ** 1. [#A] sqlite_db_statement +*** collection -#+name: sqlite_db_statement_composite +#+name: sqlite_db_statement_composite_collection #+BEGIN_SRC d { + string _db_statement; { - string _db_statement; - { - if ((doc_matters.opt.action.sqlite_create)) { - auto pth_sqlite = SiSUpathsSQLiteDiscrete!()(doc_matters.output_path, doc_matters.src.language); - pth_sqlite.base.mkdirRecurse; - _db_statement ~= SQLiteTablesReCreate!()(); - SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "TABLE RE-CREATE"); - } - if ((doc_matters.opt.action.sqlite_update)) { // TODO - _db_statement ~= SQLiteInsertMetadata!()(doc_matters); - SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "table INSERT MetaData"); - /+ get tid (lastrowid or max) for use in doc_objects table +/ - _metadata_tid_lastrowid = db.lastInsertRowid(); - _db_statement ~= SQLiteInsertDocObjectsLoop!()(doc_matters.xml.keys_seq.sql, _metadata_tid_lastrowid); // FIX - SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "table INSERT DocObjects"); - } + if ((doc_matters.opt.action.sqlite_create)) { + auto pth_sqlite = SiSUpathsSQLite!()(doc_matters.output_path); + pth_sqlite.base.mkdirRecurse; + _db_statement ~= SQLiteTablesReCreate!()(); + SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "TABLE RE-CREATE"); + } + if ((doc_matters.opt.action.sqlite_update)) { // TODO + _db_statement ~= SQLiteInsertMetadata!()(doc_matters); + SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "table INSERT MetaData"); + /+ get tid (lastrowid or max) for use in doc_objects table +/ + _metadata_tid_lastrowid = db.lastInsertRowid(); + _db_statement ~= SQLiteInsertDocObjectsLoop!()(doc_abstraction, doc_matters, _metadata_tid_lastrowid); // FIX + SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "table INSERT DocObjects"); } - db.close; } + db.close; +} +#+END_SRC + +*** discrete + +#+name: sqlite_db_statement_composite_discrete +#+BEGIN_SRC d +{ + string _db_statement; + { + _db_statement ~= SQLiteTablesReCreate!()(); + _db_statement ~= SQLiteInsertMetadata!()(doc_matters); + _db_statement ~= SQLiteInsertDocObjectsLoop!()(doc_abstraction, doc_matters, 1); // FIX + SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "table CREATE Tables, INSERT DocObjects"); + } + db.close; } #+END_SRC @@ -869,7 +930,7 @@ auto format_and_sqlite_load = SQLiteFormatAndLoadObject!()(doc_matters); string[string] obj_txt; string doc_text; string[] _insert_doc_objects; -foreach (part; doc_parts) { +foreach (part; doc_matters.xml.keys_seq.sql) { foreach (obj; doc_abstraction[part]) { switch (obj.of_part) { case "frontmatter": assert(part == "head", part); @@ -1023,7 +1084,6 @@ return _insert_doc_objects.join.to!(char[]).toUTF8; ** 5. SQL statements *** create tables - **** DROP INDEX IF EXISTS #+name: sqlite_statement_drop_existing_index @@ -1061,7 +1121,8 @@ DROP TABLE IF EXISTS urls; #+BEGIN_SRC sql CREATE TABLE metadata_and_text ( tid INTEGER PRIMARY KEY, -/* src_filename_composite VARCHAR(256) NOT NULL UNIQUE, /* z pod name if any + src filename + language code */ + src_composite_id_per_txt VARCHAR(256) NOT NULL UNIQUE, /* z pod name if any + src filename + language code */ + src_composite_id_per_pod VARCHAR(256) NOT NULL UNIQUE, /* z pod name if any + src filename + language code */ title VARCHAR(800) NOT NULL, title_main VARCHAR(400) NOT NULL, title_sub VARCHAR(400) NULL, @@ -1208,9 +1269,8 @@ CREATE INDEX idx_language ON metadata_and_text(language_document_char); CREATE INDEX idx_topics ON metadata_and_text(classify_topic_register); #+END_SRC -**** TODO local site link & info - -*** sql insert statement formatted doc objects +*** inserts +**** INSERT doc objects lid unique, increment by 1 per object, not ocn @@ -1219,14 +1279,15 @@ either: - increment by adding 1 for each document, - make hash of document filename or url and use? -**** sql statement: dlang format +***** sql statement: dlang format + #+name: sqlite_formatted_insertions_doc_objects #+BEGIN_SRC d string _insert_doc_objects_row; _insert_doc_objects_row = format(q"¶ #+END_SRC -**** INSERT INTO +***** INSERT INTO #+name: sqlite_formatted_insertions_doc_objects #+BEGIN_SRC sql @@ -1241,7 +1302,7 @@ _insert_doc_objects_row = format(q"¶ ) #+END_SRC -**** VALUES +***** VALUES #+name: sqlite_formatted_insertions_doc_objects #+BEGIN_SRC sql @@ -1256,7 +1317,7 @@ _insert_doc_objects_row = format(q"¶ ); #+END_SRC -**** sql statement: dlang values for formatting +***** dlang values for formatting #+name: sqlite_formatted_insertions_doc_objects #+BEGIN_SRC d @@ -1271,20 +1332,23 @@ _insert_doc_objects_row = format(q"¶ ); #+END_SRC -*** sqlite insert statement formatted doc matters metadata -**** sql statement: dlang format +**** INSERT doc matters metadata +***** sql statement: dlang format + #+name: sqlite_formatted_insertions_doc_matters_metadata #+BEGIN_SRC d string _insert_metadata; _insert_metadata = format(q"¶ #+END_SRC -**** INSERT INTO +***** INSERT INTO #+name: sqlite_formatted_insertions_doc_matters_metadata #+BEGIN_SRC sql INSERT INTO metadata_and_text ( src_filename, + src_composite_id_per_txt, + src_composite_id_per_pod, title, title_main, title_sub, @@ -1331,7 +1395,7 @@ _insert_metadata = format(q"¶ ) #+END_SRC -**** VALUES +***** VALUES #+name: sqlite_formatted_insertions_doc_matters_metadata #+BEGIN_SRC sql @@ -1379,16 +1443,20 @@ _insert_metadata = format(q"¶ '%s', '%s', '%s', + '%s', + '%s', '%s' ); #+END_SRC -**** sql statement: values for formatting +***** dlang values for formatting #+name: sqlite_formatted_insertions_doc_matters_metadata #+BEGIN_SRC d ¶", SQLinsertDelimiter!()(doc_matters.src.filename), + SQLinsertDelimiter!()(doc_matters.src.docname_composite_unique_per_src_doc), + SQLinsertDelimiter!()(doc_matters.src.docname_composite_unique_per_src_pod), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_full), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_main), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_subtitle), diff --git a/org/output_sqlite_discrete.org b/org/output_sqlite_discrete.org deleted file mode 100644 index c2f273c..0000000 --- a/org/output_sqlite_discrete.org +++ /dev/null @@ -1,1359 +0,0 @@ -#+TITLE: sdp output sqlite discrete -#+AUTHOR: Ralph Amissah -#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] -#+DESCRIPTION: documents - structuring, publishing in multiple formats & search -#+KEYWORDS -#+LANGUAGE: en -#+STARTUP: indent content -#+OPTIONS: H:3 num:nil toc:t \n:nil @:t ::t |:t ^:nil _:nil -:t f:t *:t <:t -#+OPTIONS: TeX:t LaTeX:t skip:nil d:nil todo:t pri:nil tags:not-in-toc -#+OPTIONS: author:nil email:nil creator:nil timestamp:nil -#+PROPERTY: header-args :padline no :exports code :noweb yes -#+EXPORT_SELECT_TAGS: export -#+EXPORT_EXCLUDE_TAGS: noexport -#+FILETAGS: :sdp:niu:output: -#+TAGS: assert(a) class(c) debug(d) mixin(m) sdp(s) tangle(T) template(t) WEB(W) noexport(n) - -* sql -** 0. module, templates - -#+BEGIN_SRC d :tangle ../src/sdp/output/sqlite_discrete.d -module sdp.output.sqlite_discrete; -<> -template SQLiteDiscreteDbRun() { - void SQLiteDiscreteDbRun(Db,St,O)( - auto ref Db db, - auto ref St db_statement, - auto ref O opt_action, - string note, - ) { - debug(sql_statement) { - writeln(db_statement); - } - try { - db.run( - "\nBEGIN;\n" ~ - db_statement ~ - "\nCOMMIT;\n" - ); - } catch (ErrnoException ex) { - writeln("ERROR SQLite : ", ex); - } catch (Exception ex) { - writeln("ERROR SQLite : ", ex); - } - if (opt_action.debug_do) { - writeln(note); - if (opt_action.verbose) { - writeln(db_statement); - } - } - } -} -template SQLiteDiscreteBuildTablesAndPopulate() { - void SQLiteDiscreteBuildTablesAndPopulate(D,I)( - auto ref const D doc_abstraction, - auto ref I doc_matters, - ) { - import d2sqlite3; - import std.typecons : Nullable; - mixin SiSUoutputRgxInit; - static auto rgx = Rgx(); - auto pth_sqlite = SiSUpathsSQLiteDiscrete!()(doc_matters.output_path, doc_matters.src.language); - pth_sqlite.base.mkdirRecurse; - auto db = Database(pth_sqlite.sqlite_file(doc_matters.src.filename)); - template SQLiteDiscreteDbStatementComposite() { - void SQLiteDiscreteDbStatementComposite(Db,D,I)( - auto ref Db db, - auto ref const D doc_abstraction, - auto ref I doc_matters, - ) { - <> - } - } - template SQLinsertDelimiter() { - auto SQLinsertDelimiter(string _txt) { - _txt = _txt - .replaceAll(rgx.quotation_mark_sql_insert_delimiter, "$0$0"); - return _txt; - } - } - template SQLiteFormatAndLoadObject() { - auto SQLiteFormatAndLoadObject(I)( - auto ref I doc_matters, - ) { - mixin SiSUoutputRgxInit; - struct sqlite_format_and_load_objects { - <> - <> - <> - <> - <> - } - return sqlite_format_and_load_objects(); - } - } - template SQLiteDiscreteTablesReCreate() { - string SQLiteDiscreteTablesReCreate()() { - string _sql_instruct; - _sql_instruct = format(q"¶ - <> - <> - <> - <> - <> - ¶",); - return _sql_instruct; - } - } - template SQLiteDiscreteInsertMetadata() { - string SQLiteDiscreteInsertMetadata(I)( - auto ref I doc_matters, - ) { - <> - return _insert_metadata; - } - } - template SQLiteDiscreteInsertDocObjectsLoop() { - string SQLiteDiscreteInsertDocObjectsLoop(P,I)( - auto ref P doc_parts, - auto ref I tid, - ) { - string insertDocObjectsRow(O)(O obj) { - <> - return _insert_doc_objects_row; - } - <> - } - } - SQLiteDiscreteDbStatementComposite!()(db, doc_abstraction, doc_matters); - } -} -#+END_SRC - -** 1. sqlite_db_statement [#A] - -#+name: sqlite_db_statement_composite -#+BEGIN_SRC d -{ - string _db_statement; - { - _db_statement ~= SQLiteDiscreteTablesReCreate!()(); - _db_statement ~= SQLiteDiscreteInsertMetadata!()(doc_matters); - _db_statement ~= SQLiteDiscreteInsertDocObjectsLoop!()(doc_matters.xml.keys_seq.sql, 1); // FIX - SQLiteDiscreteDbRun!()(db, _db_statement, doc_matters.opt.action, "table CREATE Tables, INSERT DocObjects"); - } - db.close; -} -#+END_SRC - -** 2. imports - -#+name: output_imports -#+BEGIN_SRC d -import sdp.output; -import - std.file, - std.conv : to; -#+END_SRC - -** 3. format and load template -*** 1. prepare objects (munge, sanitize, markup) -**** 1. _text_ generic munge (sanitize text for search) - -- [3/4] (search text, applies to all but code blocks) - - [ ] remove whitespace, paragraph on single line (formatting kept so far) - - [X] remove font face attributes - - [X] move embedded endnotes - - [X] place after text object - - [X] remove embedded endnote numbers (rely on html output to represent) - - [X] urls - - [X] clean url markers, leave plain link text - - [X] place urls after text object and its endnotes - -#+name: sanitize_text_for_search -#+BEGIN_SRC d -auto generic_munge_sanitize_text_for_search( - string _txt, -) { - string _notes; - string _urls; - if (_txt.matchFirst(rgx.inline_notes_al_gen)) { - foreach (m; _txt.matchAll(rgx.inline_notes_al_gen_text)) { - _notes ~= "\n" ~ m["text"]; - } - _txt = _txt.replaceAll(rgx.inline_notes_al_gen, ""); - } - if (_txt.matchFirst(rgx.inline_link)) { - foreach (m; _txt.matchAll(rgx.inline_link)) { - if (m["link"].match(rgx.url)) { - _urls ~= "\n" ~ m["link"]; - } - } - _txt = _txt.replaceAll(rgx.inline_link_clean, ""); - } - if (_notes.length > 0) { - _txt ~= _notes; - } - if (_urls.length > 0) { - _txt ~= _urls; - } - if (doc_matters.opt.action.debug_do) { - writeln(_txt, "\n"); - } - debug(sql_text_clean) { - writeln(_txt); - } - return _txt; -} -#+END_SRC - -**** 2. _html_ - -light html objects -- place endnotes after text object -- light inline html markup - -***** munge -****** general munge (special characters, inline markup, move notes) - -#+name: sanitize_and_munge_inline_html -#+BEGIN_SRC d -auto munge_html(O)( - auto return ref const O obj, -) { - string _html_special_characters(string _txt){ - _txt = _txt - .replaceAll(rgx.xhtml_ampersand, "&") - .replaceAll(rgx.xhtml_quotation, """) - .replaceAll(rgx.xhtml_less_than, "<") - .replaceAll(rgx.xhtml_greater_than, ">") - .replaceAll(rgx.nbsp_char, " ") - .replaceAll(rgx.xhtml_line_break, "
"); - return _txt; - } - string _html_font_face(string _txt){ - _txt = _txt - .replaceAll(rgx.inline_emphasis, "$1") - .replaceAll(rgx.inline_bold, "$1") - .replaceAll(rgx.inline_underscore, "$1") - .replaceAll(rgx.inline_italics, "$1") - .replaceAll(rgx.inline_superscript, "$1") - .replaceAll(rgx.inline_subscript, "$1") - .replaceAll(rgx.inline_strike, "$1") - .replaceAll(rgx.inline_insert, "$1") - .replaceAll(rgx.inline_mono, "$1") - .replaceAll(rgx.inline_cite, "$1"); - return _txt; - } - string _notes; - string _urls; - string _txt = _html_font_face(_html_special_characters(obj.text)); - if (_txt.matchFirst(rgx.inline_notes_al_gen)) { - foreach (m; _txt.matchAll(rgx.inline_notes_al_gen_text)) { - _notes ~= "\n" ~ m["text"]; - } - _txt = _txt.replaceAll(rgx.inline_notes_al_gen_ref, "$1 "); - } - if (_txt.matchFirst(rgx.inline_link)) { - foreach (m; _txt.matchAll(rgx.inline_link)) { - } - _txt = _txt.replaceAll(rgx.inline_link_clean, ""); - } - if (_notes.length > 0) { - _txt ~= _notes; - } - if (doc_matters.opt.action.debug_do) { - writeln(_txt, "\n"); - } - return _txt; -} -#+END_SRC - -****** special characters - -#+name: sanitize_and_munge_inline_html -#+BEGIN_SRC d -string html_special_characters(string _txt){ - _txt = _txt - .replaceAll(rgx.xhtml_ampersand, "&") - .replaceAll(rgx.xhtml_quotation, """) - .replaceAll(rgx.xhtml_less_than, "<") - .replaceAll(rgx.xhtml_greater_than, ">") - .replaceAll(rgx.nbsp_char, " ") - .replaceAll(rgx.xhtml_line_break, "
"); - return _txt; -} -#+END_SRC - -****** special characters for code - -#+name: sanitize_and_munge_inline_html -#+BEGIN_SRC d -string html_special_characters_code(string _txt){ - _txt = _txt - .replaceAll(rgx.xhtml_ampersand, "&") - .replaceAll(rgx.xhtml_quotation, """) - .replaceAll(rgx.xhtml_less_than, "<") - .replaceAll(rgx.xhtml_greater_than, ">") - .replaceAll(rgx.nbsp_char, " "); - return _txt; -} -#+END_SRC - -****** font_face - -#+name: sanitize_and_munge_inline_html -#+BEGIN_SRC d -string html_font_face(string _txt){ - _txt = _txt - .replaceAll(rgx.inline_emphasis, "$1") - .replaceAll(rgx.inline_bold, "$1") - .replaceAll(rgx.inline_underscore, "$1") - .replaceAll(rgx.inline_italics, "$1") - .replaceAll(rgx.inline_superscript, "$1") - .replaceAll(rgx.inline_subscript, "$1") - .replaceAll(rgx.inline_strike, "$1") - .replaceAll(rgx.inline_insert, "$1") - .replaceAll(rgx.inline_mono, "$1") - .replaceAll(rgx.inline_cite, "$1"); - return _txt; -} -#+END_SRC - -***** objects -****** heading - -#+name: html_objects -#+BEGIN_SRC d -auto html_heading(O)( - auto return ref const O obj, -) { - string _txt = munge_html(obj); - string o = format(q"¶

- %s -

¶", - obj.is_a, - _txt, - ); - return o; -} -#+END_SRC - -******* +fancy+ - -##+name: prepare_objects_html -#+BEGIN_SRC d -auto html_heading(O)( - auto return ref const O obj, -) { - string o; - string _txt = munge_html(obj); - o = format(q"¶ - %s - ¶", - obj.heading_lev_markup, - obj.is_a, - _txt, - obj.heading_lev_markup, - ); - return o; -} -#+END_SRC - -****** para - -#+name: html_objects -#+BEGIN_SRC d -auto html_para(O)( - auto return ref const O obj, -) { - string _txt = munge_html(obj); - _txt = (obj.bullet) ? ("●  " ~ _txt) : _txt; - string o = format(q"¶

- %s -

¶", - obj.is_a, - obj.indent_hang, - obj.indent_base, - _txt - ); - return o; -} -#+END_SRC - -****** quote - -#+name: html_objects -#+BEGIN_SRC d -auto html_quote(O)( - auto return ref const O obj, -) { - string _txt = munge_html(obj); - string o = format(q"¶

- %s -

¶", - obj.is_a, - _txt - ); - return o; -} -#+END_SRC - -****** group - -#+name: html_objects -#+BEGIN_SRC d -auto html_group(O)( - auto return ref const O obj, -) { - string _txt = munge_html(obj); - string o = format(q"¶

- %s -

¶", - obj.is_a, - _txt - ); - return o; -} -#+END_SRC - -****** block - -#+name: html_objects -#+BEGIN_SRC d -auto html_block(O)( - auto return ref const O obj, -) { - string _txt = munge_html(obj); - string o = format(q"¶ -

%s

¶", - obj.is_a, - _txt.stripRight - ); - return o; -} -#+END_SRC - -****** verse - -#+name: html_objects -#+BEGIN_SRC d -auto html_verse(O)( - auto return ref const O obj, -) { - string _txt = munge_html(obj); - string o = format(q"¶

%s

¶", - obj.is_a, - _txt - ); - return o; -} -#+END_SRC - -****** code - -#+name: html_objects -#+BEGIN_SRC d -auto html_code(O)( - auto return ref const O obj, -) { - string _txt = html_special_characters_code(obj.text); - string o = format(q"¶

%s

¶", - obj.is_a, - _txt - ); - return o; -} -#+END_SRC - -****** table - -#+name: html_objects -#+BEGIN_SRC d -auto html_table(O)( - auto return ref const O obj, -) { - auto _tablarize(O)( - auto return ref const O obj, - string _txt, - ) { - string[] _table_rows = _txt.split(rgx.table_delimiter_row); - string[] _table_cols; - string _table; - string _tablenote; - foreach(row_idx, row; _table_rows) { - _table_cols = row.split(rgx.table_delimiter_col); - _table ~= ""; - foreach(col_idx, cell; _table_cols) { - if ((_table_cols.length == 1) - && (_table_rows.length <= row_idx+2)) { // check row_idx+2 (rather than == ++row_idx) - _tablenote ~= cell; - } else { - string _col_is = (row_idx == 0 && obj.table_heading) ? "th" : "td"; - string _align = ("style=\"text-align:" - ~ ((obj.table_column_aligns[col_idx] == "l") - ? "left\"" : "right\"")); - _table ~= "<" - ~ _col_is - ~ " width=\"" - ~ obj.table_column_widths[col_idx].to!string - ~ "%\" " - ~ _align - ~ ">"; - _table ~= cell; - _table ~= ""; - } - } - _table ~= ""; - } - auto t = tuple( - _table, - _tablenote, - ); - return t; - } - string _txt = munge_html(obj); - auto t = _tablarize(obj, _txt); - _txt = t[0]; - string _note = t[1]; - string o = format(q"¶

- - %s -
- %s -

¶", - obj.is_a, - _txt, - _note - ); - return o; -} -#+END_SRC - -*** 2. _sqlite_ (loop preparation & pre-loop action) -**** prepare insert statements and do pre-loop inserts -***** 1. _metadata & src_ text -***** 2. _doc objects_ (used with doc_objects in document loop) -****** TODO (within loop not here - insert doc objects - -work out - -*** 3. hub (sqlite_format_and_load_objects) -**** sql related - -#+name: sqlite_load_object -#+BEGIN_SRC d -auto sqlite_load_string(O,Dm)( - auto return ref const O obj, - auto return ref Dm doc_matters, -) { - string o; - return o; -} -#+END_SRC - -#+name: sqlite_load_object -#+BEGIN_SRC d -auto postgresql_load_string(O,Dm)( - auto return ref const O obj, - auto return ref Dm doc_matters, -) { - string o; - return o; -} -#+END_SRC - -#+name: sqlite_load_object -#+BEGIN_SRC d -string sqlite_statement(O)( - auto return ref const O obj, - string _txt, - string _html, -) { - void _sql_exe(O)( - string _sql, - ) { - writeln(_html); - writeln(_sql); - } - string _sql; - return _sql; -} -#+END_SRC - -**** heading - -#+name: hub_format_and_sqlite_load_objects -#+BEGIN_SRC d -auto heading(O)( - auto return ref const O obj, -) { - string[string] obj_txt = [ - "text": generic_munge_sanitize_text_for_search(obj.text), - "html": html_heading(obj) - ]; - if (doc_matters.opt.action.debug_do) { - debug(sql_txt) { - writeln(obj_txt["text"]); - } - debug(sql_html) { - writeln(obj_txt["html"]); - } - } else { - // load sql - } - return obj_txt; -} -#+END_SRC - -**** para - -#+name: hub_format_and_sqlite_load_objects -#+BEGIN_SRC d -auto para(O)( - auto return ref const O obj, -) { - string[string] obj_txt = [ - "text": generic_munge_sanitize_text_for_search(obj.text), - "html": html_para(obj) - ]; - if (doc_matters.opt.action.debug_do) { - debug(sql_txt) { - writeln(obj_txt["text"]); - } - debug(sql_html) { - writeln(obj_txt["html"]); - } - } else { - // load sql - } - return obj_txt; -} -#+END_SRC - -**** quote - -#+name: hub_format_and_sqlite_load_objects -#+BEGIN_SRC d -auto quote(O)( - auto return ref const O obj, -) { - string[string] obj_txt = [ - "text": generic_munge_sanitize_text_for_search(obj.text), - "html": html_quote(obj) - ]; - if (doc_matters.opt.action.debug_do) { - debug(sql_txt) { - writeln(obj_txt["text"]); - } - debug(sql_html) { - writeln(obj_txt["html"]); - } - } else { - // load sql - } - return obj_txt; -} -#+END_SRC -**** group - -#+name: hub_format_and_sqlite_load_objects -#+BEGIN_SRC d -auto group(O)( - auto return ref const O obj, -) { - string[string] obj_txt = [ - "text": generic_munge_sanitize_text_for_search(obj.text), - "html": html_group(obj) - ]; - if (doc_matters.opt.action.debug_do) { - debug(sql_txt) { - writeln(obj_txt["text"]); - } - debug(sql_html) { - writeln(obj_txt["html"]); - } - } else { - // load sql - } - return obj_txt; -} -#+END_SRC - -**** block - -#+name: hub_format_and_sqlite_load_objects -#+BEGIN_SRC d -auto block(O)( - auto return ref const O obj, -) { - string[string] obj_txt = [ - "text": generic_munge_sanitize_text_for_search(obj.text), - "html": html_block(obj) - ]; - if (doc_matters.opt.action.debug_do) { - debug(sql_txt) { - writeln(obj_txt["text"]); - } - debug(sql_html) { - writeln(obj_txt["html"]); - } - } else { - // load sql - } - return obj_txt; -} -#+END_SRC - -**** verse - -#+name: hub_format_and_sqlite_load_objects -#+BEGIN_SRC d -auto verse(O)( - auto return ref const O obj, -) { - string[string] obj_txt = [ - "text": generic_munge_sanitize_text_for_search(obj.text), - "html": html_verse(obj) - ]; - if (doc_matters.opt.action.debug_do) { - debug(sql_txt) { - writeln(obj_txt["text"]); - } - debug(sql_html) { - writeln(obj_txt["html"]); - } - } else { - // load sql - } - return obj_txt; -} -#+END_SRC - -**** code - -#+name: hub_format_and_sqlite_load_objects -#+BEGIN_SRC d -auto code(O)( - auto return ref const O obj, -) { - string[string] obj_txt = [ - "text": generic_munge_sanitize_text_for_search(obj.text), - "html": html_code(obj) - ]; - if (doc_matters.opt.action.debug_do) { - debug(sql_txt) { - writeln(obj_txt["text"]); - } - debug(sql_html) { - writeln(obj_txt["html"]); - } - } else { - // load sql - } - return obj_txt; -} -#+END_SRC - -**** table - -#+name: hub_format_and_sqlite_load_objects -#+BEGIN_SRC d -auto table(O)( - auto return ref const O obj, -) { - string[string] obj_txt = [ - "text": generic_munge_sanitize_text_for_search(obj.text), - "html": html_table(obj) - ]; - if (doc_matters.opt.action.debug_do) { - debug(sql_txt) { - writeln(obj_txt["text"]); - } - debug(sql_html) { - writeln(obj_txt["html"]); - } - } else { - // load sql - } - return obj_txt; -} -#+END_SRC - -** 4. loop, identify, load - loop template - -#+name: sqlite_objects_loop -#+BEGIN_SRC d -auto format_and_sqlite_load = SQLiteFormatAndLoadObject!()(doc_matters); -string[string] obj_txt; -string doc_text; -string[] _insert_doc_objects; -foreach (part; doc_parts) { - foreach (obj; doc_abstraction[part]) { - switch (obj.of_part) { - case "frontmatter": assert(part == "head", part); - switch (obj.is_of) { - case "para": - switch (obj.is_a) { - case "heading": - obj_txt = format_and_sqlite_load.heading(obj); - break; - default: - if ((doc_matters.opt.action.debug_do)) { - writeln(__FILE__, ":", __LINE__, ": ", obj.is_a); - } - break; - } - break; - default: - if ((doc_matters.opt.action.debug_do)) { - writeln(__FILE__, ":", __LINE__, ": ", obj.is_of); - } - break; - } - break; - case "body": // assert(part == "body", part); // TODO broken - switch (obj.is_of) { - case "para": - switch (obj.is_a) { - case "heading": - debug (asserts) { // TODO consider and fix or remove - if (part != "body") { - writeln(__LINE__, ": ", obj.text); - } - } - obj_txt = format_and_sqlite_load.heading(obj); - break; - case "para": - obj_txt = format_and_sqlite_load.para(obj); - break; - default: - if ((doc_matters.opt.action.debug_do)) { - writeln(__FILE__, ":", __LINE__, ": ", obj.is_a); - } - break; - } - break; - case "block": - switch (obj.is_a) { - case "quote": - obj_txt = format_and_sqlite_load.quote(obj); - break; - case "group": - obj_txt = format_and_sqlite_load.group(obj); - break; - case "block": - obj_txt = format_and_sqlite_load.block(obj); - break; - case "poem": // double check on keeping both poem & verse - break; - case "verse": - obj_txt = format_and_sqlite_load.verse(obj); - break; - case "code": - obj_txt = format_and_sqlite_load.code(obj); - break; - case "table": - obj_txt = format_and_sqlite_load.table(obj); - break; - default: - if ((doc_matters.opt.action.debug_do)) { - writeln(__FILE__, ":", __LINE__, ": ", obj.is_a); - } - break; - } - break; - default: - if ((doc_matters.opt.action.debug_do)) { - writeln(__FILE__, ":", __LINE__, ": ", obj.is_of); - } - break; - } - break; - case "backmatter": - assert(part == "endnotes" || "glossary" || "bibliography" || "bookindex_seg" || "blurb" || "tail", part); - switch (obj.is_of) { - case "para": - switch (obj.is_a) { - case "heading": - obj_txt = format_and_sqlite_load.heading(obj); - break; - case "endnote": assert(part == "endnotes", part); - obj_txt = format_and_sqlite_load.para(obj); - break; - case "glossary": assert(part == "glossary", part); - obj_txt = format_and_sqlite_load.para(obj); - break; - case "bibliography": assert(part == "bibliography", part); - obj_txt = format_and_sqlite_load.para(obj); - break; - case "bookindex": assert(part == "bookindex_seg", part); - obj_txt = format_and_sqlite_load.para(obj); - break; - case "blurb": assert(part == "blurb", part); - obj_txt = format_and_sqlite_load.para(obj); - break; - default: - if ((doc_matters.opt.action.debug_do)) { - writeln(__FILE__, ":", __LINE__, ": ", obj.is_a); - } - break; - } - break; - default: - if ((doc_matters.opt.action.debug_do)) { - writeln(__FILE__, ":", __LINE__, ": ", obj.is_of); - } - break; - } - break; - case "comment": - break; - default: - if ((doc_matters.opt.action.debug_do)) { - writeln(__FILE__, ":", __LINE__, ": ", obj.of_part); // check where empty value could come from - writeln(__FILE__, ":", __LINE__, ": ", obj.is_a); - writeln(__FILE__, ":", __LINE__, ": ", obj.text); // check where empty value could come from - } - break; - } - if (obj.is_a == "heading") { - if ((doc_matters.opt.action.verbose)) { - writeln( - "markup: ", obj.heading_lev_markup, - "> ", obj.dom_markedup, - "; collapsed: ", obj.heading_lev_collapsed, - "> ", obj.dom_collapsed, - "; ocn: ", obj.ocn, - " node: ", obj.node, - "; parent: ", obj.parent_lev_markup, - "; ocn: ", obj.parent_ocn, - "; ", - ); - } - } - if (!(obj.is_a == "comment")) { - _insert_doc_objects ~= insertDocObjectsRow(obj); - } - } // loop closes -} -return _insert_doc_objects.join.to!(char[]).toUTF8; -#+END_SRC - -** 5. SQL statements -*** create tables - -**** DROP INDEX IF EXISTS - -#+name: sqlite_statement_drop_existing_index -#+BEGIN_SRC sql -DROP INDEX IF EXISTS ocn; -DROP INDEX IF EXISTS digest_clean; -DROP INDEX IF EXISTS digest_all; -DROP INDEX IF EXISTS clean; -DROP INDEX IF EXISTS lev0; -DROP INDEX IF EXISTS lev1; -DROP INDEX IF EXISTS lev2; -DROP INDEX IF EXISTS lev3; -DROP INDEX IF EXISTS lev4; -DROP INDEX IF EXISTS lev5; -DROP INDEX IF EXISTS lev6; -DROP INDEX IF EXISTS title; -DROP INDEX IF EXISTS creator_author; -DROP INDEX IF EXISTS src_filename; -DROP INDEX IF EXISTS language_document_char; -DROP INDEX IF EXISTS classify_topic_register; -#+END_SRC - -**** DROP TABLE IF EXISTS - -#+name: sqlite_statement_drop_existing_tables -#+BEGIN_SRC sql -DROP TABLE IF EXISTS metadata_and_text; -DROP TABLE IF EXISTS doc_objects; -DROP TABLE IF EXISTS urls; -#+END_SRC - -**** CREATE TABLE metadata_and_text - -#+name: sqlite_statement_create_table_metadata_and_src_txt -#+BEGIN_SRC sql -CREATE TABLE metadata_and_text ( - tid INTEGER PRIMARY KEY, -/* src_filename_composite VARCHAR(256) NOT NULL UNIQUE, /* z pod name if any + src filename + language code */ - title VARCHAR(800) NOT NULL, - title_main VARCHAR(400) NOT NULL, - title_sub VARCHAR(400) NULL, - title_short VARCHAR(400) NULL, - title_edition VARCHAR(10) NULL, - title_note VARCHAR(2500) NULL, - title_language VARCHAR(100) NULL, - title_language_char VARCHAR(6) NULL, - creator_author VARCHAR(600) NOT NULL, - creator_author_email VARCHAR(100) NULL, - creator_author_hon VARCHAR(100) NULL, - creator_author_nationality VARCHAR(100) NULL, - creator_editor VARCHAR(600) NULL, - creator_contributor VARCHAR(600) NULL, - creator_illustrator VARCHAR(600) NULL, - creator_photographer VARCHAR(600) NULL, - creator_translator VARCHAR(600) NULL, - creator_prepared_by VARCHAR(600) NULL, - creator_digitized_by VARCHAR(600) NULL, - creator_audio VARCHAR(600) NULL, - creator_video VARCHAR(600) NULL, - language_document VARCHAR(100) NULL, - language_document_char VARCHAR(6) NOT NULL, - language_original VARCHAR(100) NULL, - language_original_char VARCHAR(6) NULL, - date_added_to_site VARCHAR(10) NULL, - date_available VARCHAR(10) NULL, - date_created VARCHAR(10) NULL, - date_issued VARCHAR(10) NULL, - date_modified VARCHAR(10) NULL, - date_published VARCHAR(10) NULL, - date_valid VARCHAR(10) NULL, - date_translated VARCHAR(10) NULL, - date_original_publication VARCHAR(10) NULL, - date_generated VARCHAR(10) NULL, - publisher VARCHAR(600) NULL, - original_publisher VARCHAR(600) NULL, - original_language VARCHAR(100) NULL, - original_language_char VARCHAR(6) NULL, - original_source VARCHAR(600) NULL, - original_institution VARCHAR(600) NULL, - original_nationality VARCHAR(100) NULL, - original_title VARCHAR(800) NULL, - rights_copyright VARCHAR(2500) NULL, - rights_copyright_audio VARCHAR(2500) NULL, - rights_copyright_cover VARCHAR(2500) NULL, - rights_copyright_illustrations VARCHAR(2500) NULL, - rights_copyright_photographs VARCHAR(2500) NULL, - rights_copyright_text VARCHAR(2500) NULL, - rights_copyright_translation VARCHAR(2500) NULL, - rights_copyright_video VARCHAR(2500) NULL, - rights_license VARCHAR(2500) NULL, - identifier_oclc VARCHAR(30) NULL, - identifier_isbn VARCHAR(16) NULL, - classify_topic_register VARCHAR(2500) NULL, - classify_subject VARCHAR(600) NULL, - classify_loc VARCHAR(30) NULL, - classify_dewey VARCHAR(30) NULL, - classify_keywords VARCHAR(600) NULL, - notes_abstract TEXT NULL, - notes_description TEXT NULL, - notes_comment TEXT NULL, - notes_coverage VARCHAR(200) NULL, - notes_relation VARCHAR(200) NULL, - notes_history VARCHAR(600) NULL, - notes_type VARCHAR(600) NULL, - notes_format VARCHAR(600) NULL, - notes_prefix TEXT NULL, - notes_prefix_a TEXT NULL, - notes_prefix_b TEXT NULL, - notes_suffix TEXT NULL, - src_filename VARCHAR(256) NOT NULL, - src_fingerprint VARCHAR(256) NULL, - src_filesize VARCHAR(10) NULL, - src_wordcount VARCHAR(10) NULL, - pod_name VARCHAR(256) NULL, /* zipped pod, work to be done here */ - pod_fingerprint VARCHAR(256) NULL, /* zipped pod, work to be done here */ - pod_size VARCHAR(10) NULL, /* zipped pod, work to be done here */ - src_text TEXT NULL, - fulltext TEXT NULL, - links TEXT NULL -); -#+END_SRC - -**** CREATE TABLE doc_objects - -#+name: sqlite_statement_create_table_objects -#+BEGIN_SRC sql -CREATE TABLE doc_objects ( - lid INTEGER PRIMARY KEY, - metadata_tid INTEGER REFERENCES metadata_and_text, - ocn SMALLINT, - ocnd VARCHAR(6), - ocns VARCHAR(6), - clean TEXT NULL, - body TEXT NULL, - book_idx TEXT NULL, - seg VARCHAR(256) NULL, - lev_an VARCHAR(1), - lev SMALLINT NULL, - lev0 SMALLINT, - lev1 SMALLINT, - lev2 SMALLINT, - lev3 SMALLINT, - lev4 SMALLINT, - lev5 SMALLINT, - lev6 SMALLINT, - lev7 SMALLINT, - en_a SMALLINT NULL, - en_z SMALLINT NULL, - en_a_asterisk SMALLINT NULL, - en_z_asterisk SMALLINT NULL, - en_a_plus SMALLINT NULL, - en_z_plus SMALLINT NULL, - t_of VARCHAR(16), - t_is VARCHAR(16), - node VARCHAR(16) NULL, - parent VARCHAR(16) NULL, - digest_clean CHAR(256), - digest_all CHAR(256), - types CHAR(1) NULL -); -#+END_SRC - -**** CREATE INDEX - -#+name: sqlite_statement_create_index -#+BEGIN_SRC sql -CREATE INDEX idx_ocn ON doc_objects(ocn); -CREATE INDEX idx_digest_clean ON doc_objects(digest_clean); -CREATE INDEX idx_digest_all ON doc_objects(digest_all); -CREATE INDEX idx_clean ON doc_objects(clean); -CREATE INDEX idx_lev0 ON doc_objects(lev0); -CREATE INDEX idx_lev1 ON doc_objects(lev1); -CREATE INDEX idx_lev2 ON doc_objects(lev2); -CREATE INDEX idx_lev3 ON doc_objects(lev3); -CREATE INDEX idx_lev4 ON doc_objects(lev4); -CREATE INDEX idx_lev5 ON doc_objects(lev5); -CREATE INDEX idx_lev6 ON doc_objects(lev6); -CREATE INDEX idx_title ON metadata_and_text(title); -CREATE INDEX idx_author ON metadata_and_text(creator_author); -CREATE INDEX idx_filename ON metadata_and_text(src_filename); -CREATE INDEX idx_language ON metadata_and_text(language_document_char); -CREATE INDEX idx_topics ON metadata_and_text(classify_topic_register); -#+END_SRC - -**** TODO local site link & info - -*** sql insert statement formatted doc objects - -lid unique, increment by 1 per object, not ocn - -metadata tid document number unique -either: -- increment by adding 1 for each document, -- make hash of document filename or url and use? - -**** sql statement: dlang format -#+name: sqlite_formatted_insertions_doc_objects -#+BEGIN_SRC d -string _insert_doc_objects_row; -_insert_doc_objects_row = format(q"¶ -#+END_SRC - -**** INSERT INTO - -#+name: sqlite_formatted_insertions_doc_objects -#+BEGIN_SRC sql - INSERT INTO doc_objects ( - metadata_tid, - ocn, - clean, - body, - lev, - t_of, - t_is - ) -#+END_SRC - -**** VALUES - -#+name: sqlite_formatted_insertions_doc_objects -#+BEGIN_SRC sql - VALUES ( - %d, - %s, - '%s', - '%s', - %s, - '%s', - '%s' - ); -#+END_SRC - -**** sql statement: dlang values for formatting - -#+name: sqlite_formatted_insertions_doc_objects -#+BEGIN_SRC d -¶", - 1, - obj.ocn, - SQLinsertDelimiter!()(obj_txt["text"]), - SQLinsertDelimiter!()(obj_txt["html"]), - obj.heading_lev_markup, - obj.is_of, - obj.is_a, -); -#+END_SRC - -*** sqlite insert statement formatted doc matters metadata -**** sql statement: dlang format -#+name: sqlite_formatted_insertions_doc_matters_metadata -#+BEGIN_SRC d -string _insert_metadata; -_insert_metadata = format(q"¶ -#+END_SRC - -**** INSERT INTO - -#+name: sqlite_formatted_insertions_doc_matters_metadata -#+BEGIN_SRC sql - INSERT INTO metadata_and_text ( - src_filename, - title, - title_main, - title_sub, - title_short, - title_edition, - title_language, - classify_dewey, - classify_keywords, - classify_loc, - classify_subject, - classify_topic_register, - creator_author, - creator_author_email, - creator_illustrator, - creator_translator, - date_added_to_site, - date_available, - date_created, - date_issued, - date_modified, - date_published, - date_valid, - identifier_isbn, - identifier_oclc, - language_document, - language_document_char, - notes_abstract, - notes_description, - original_publisher, - original_language, - original_language_char, - original_source, - original_title, - publisher, - rights_copyright, - rights_copyright_audio, - rights_copyright_cover, - rights_copyright_illustrations, - rights_copyright_photographs, - rights_copyright_text, - rights_copyright_translation, - rights_copyright_video, - rights_license - ) -#+END_SRC - -**** VALUES - -#+name: sqlite_formatted_insertions_doc_matters_metadata -#+BEGIN_SRC sql - VALUES ( - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s' - ); -#+END_SRC - -**** sql statement: values for formatting - -#+name: sqlite_formatted_insertions_doc_matters_metadata -#+BEGIN_SRC d -¶", - SQLinsertDelimiter!()(doc_matters.src.filename), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_full), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_main), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_subtitle), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_short), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_edition), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_language), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_dewey), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_keywords), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_loc), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_subject), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_topic_register), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_author), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_author_email), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_illustrator), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_translator), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_added_to_site), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_available), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_created), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_issued), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_modified), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_published), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_valid), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.identifier_isbn), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.identifier_oclc), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.language_document), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.language_document_char), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.notes_abstract), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.notes_description), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_publisher), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_language), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_language_char), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_source), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_title), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.publisher), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_audio), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_cover), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_illustrations), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_photographs), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_text), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_translation), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_video), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_license) -); -#+END_SRC - -* __END__ diff --git a/org/sdp.org b/org/sdp.org index 6b9052b..a3ddf40 100644 --- a/org/sdp.org +++ b/org/sdp.org @@ -26,7 +26,7 @@ struct Version { int minor; int patch; } -enum ver = Version(0, 26, 0); +enum ver = Version(0, 26, 1); #+END_SRC ** compilation restrictions (supported compilers) diff --git a/src/sdp/meta/rgx.d b/src/sdp/meta/rgx.d index 0f60847..6006122 100644 --- a/src/sdp/meta/rgx.d +++ b/src/sdp/meta/rgx.d @@ -199,7 +199,7 @@ static template SiSUrgxInit() { static nbsp_and_space = ctRegex!(` [ ]`, "mg"); static nbsp_char_and_space = ctRegex!(`░[ ]`, "mg"); static special_markup_chars = ctRegex!(`[【】〖〗┥┝┤├¤░┘┙┚┼┿╂┊┏┚┆■]`, "mg"); - static src_pth_sst_or_ssm = ctRegex!(`^(?P[/]?(?:[a-zA-Z0-9._-]+/)*)(?P[a-zA-Z0-9._-]+[.]ss[tm])$`); + static src_pth_sst_or_ssm = ctRegex!(`^(?P[/]?(?:[a-zA-Z0-9._-]+/)*)(?P[a-zA-Z0-9._-]+[.](?Pss[tm]))$`); static src_pth_pod_sst_or_ssm = ctRegex!(`^(?P[/]?(?:[a-zA-Z0-9._-]+/)*)media/text/[a-z]{2}/(?P[a-zA-Z0-9._-]+[.]ss[tm])$`); static src_pth_contents = ctRegex!(`^(?P[/]?(?:[a-zA-Z0-9._-]+/)*)(?P[a-zA-Z0-9._-]+)/sisupod[.]manifest$`); static src_pth_pod_root = ctRegex!(`^(?P(?:[/]?(?:[a-zA-Z0-9._-]+/)*)(sisupod))$`); diff --git a/src/sdp/output/hub.d b/src/sdp/output/hub.d index 373092b..d395269 100644 --- a/src/sdp/output/hub.d +++ b/src/sdp/output/hub.d @@ -8,7 +8,6 @@ template outputHub() { sdp.output.epub3, sdp.output.html, sdp.output.sqlite, - sdp.output.sqlite_discrete, sdp.output.xmls, sdp.output.source_sisupod, sdp.output.create_zip_file, @@ -16,6 +15,35 @@ template outputHub() { void outputHub(D,I)(D doc_abstraction, I doc_matters) { mixin SiSUoutputRgxInit; static auto rgx = Rgx(); + debug (substitutions) { + enum Substitute { match, markup, html, } + writeln(__LINE__, ":", __FILE__, ": DEBUG substitutions:"); + writeln("Doc Title: ", doc_matters.conf_make_meta.meta.title_full); + if (doc_matters.conf_make_meta.make.bold) { + writeln("regex to match: ", doc_matters.conf_make_meta.make.bold[Substitute.match]); + writeln("substitution to make: ", doc_matters.conf_make_meta.make.bold[Substitute.html]); + + auto _w = "1. Debian test string. Debian again. (the problem) do not use initialized only not repopulated" + .replaceAll( + regex(doc_matters.conf_make_meta.make.bold[Substitute.match]), + doc_matters.conf_make_meta.make.bold[Substitute.html] + ); + writeln(_w); + + writeln("2a. Debian test string. Debian again. (the problem) do not use initialized only not repopulated" + .replaceAll( + regex(doc_matters.conf_make_meta.make.bold[Substitute.match]), + doc_matters.conf_make_meta.make.bold[Substitute.html] + )); + + auto _v = regex(doc_matters.conf_make_meta.make.bold[Substitute.match]); + writeln("2b. Debian test string. Debian again. (the problem) do not use initialized only not repopulated" + .replaceAll( + _v, + doc_matters.conf_make_meta.make.bold[Substitute.html] + )); + } + } if (doc_matters.opt.action.verbose) { writeln(doc_matters.xml.keys_seq.seg); } @@ -109,13 +137,13 @@ template outputHub() { if ((doc_matters.opt.action.verbose)) { writeln("sqlite processing... "); } - SQLiteDiscreteBuildTablesAndPopulate!()(doc_abstraction, doc_matters); + SQLiteHubDiscreteBuildTablesAndPopulate!()(doc_abstraction, doc_matters); } if (doc_matters.opt.action.sqlite_update) { if ((doc_matters.opt.action.verbose)) { writeln("sqlite processing... "); } - SQLiteBuildTablesAndPopulate!()(doc_abstraction, doc_matters); + SQLiteHubBuildTablesAndPopulate!()(doc_abstraction, doc_matters); } if (doc_matters.opt.action.postgresql) { /+ mixin outputPostgreSQL; +/ @@ -128,7 +156,6 @@ template outputHubOp() { sdp.output.epub3, sdp.output.html, sdp.output.sqlite, - sdp.output.sqlite_discrete, sdp.output.xmls, sdp.output.source_sisupod, sdp.output.create_zip_file, diff --git a/src/sdp/output/paths_source.d b/src/sdp/output/paths_source.d index c420d7e..d04c9f6 100644 --- a/src/sdp/output/paths_source.d +++ b/src/sdp/output/paths_source.d @@ -151,6 +151,7 @@ template PathMatters() { return Pod_(); } auto src() { + string _sep = "␣"; struct SRC_ { auto is_pod() { auto _src_is_pod = (_manifest.pod_manifest_path.length > 0) ? true : false; @@ -159,19 +160,70 @@ template PathMatters() { auto path_and_fn() { return _fns; } + auto pod_name() { /+ work on +/ + auto _pod_name = (is_pod) + ? _manifest.pod_manifest_path + : ""; + return _pod_name; + } auto filename() { auto _fn = (path_and_fn).baseName; return _fn; } + auto filename_base() { + auto _fn = filename.stripExtension; + return _fn; + } + auto filename_extension() { + auto _ext = filename.match(rgx.src_pth_sst_or_ssm).captures["extension"]; + return _ext; + } auto lng() { string _k; - if (auto m = (path_and_fn).match(rgx.language_code_and_filename)) { + if (auto m = path_and_fn.match(rgx.language_code_and_filename)) { _k = m.captures[1]; } else { _k = "en"; } return _k; } + auto docname_composite_unique_per_src_doc() { + /+ + z pod name if any + src filename + lng code + filename ~ "." ~ lng + * unique per src doc + used by + - sqlite discrete index (multilingual, each language of a document) + +/ + string _fn; + if (pod_name.baseName == filename_base) { + _fn = filename_base ~ _sep ~ filename_extension ~ _sep ~ lng; + } else if (!(pod_name.empty)) { + _fn = pod_name.baseName ~ _sep ~ filename_base ~ _sep ~ filename_extension ~ _sep ~ lng; + } else { + _fn = "_" ~ _sep ~ filename_base ~ _sep ~ filename_extension ~ _sep ~ lng; + } + return _fn; + } + auto docname_composite_unique_per_src_pod() { + /+ + z pod name if any + src filename (without lng code) + filename ~ _sep ~ lng + * unique per src pod + used by + - sisupod (multilingual collection) + - sqlite discrete index (multilingual collection) + +/ + string _fn; + if (pod_name.baseName == filename_base) { + _fn = filename_base ~ _sep ~ filename_extension; + } else if (!(pod_name.empty)) { + _fn = pod_name.baseName ~ _sep ~ filename_base ~ _sep ~ filename_extension; + } else { + _fn = "_" ~ _sep ~ filename_base ~ _sep ~ filename_extension; + } + return _fn; + } auto language() { return lng(); } diff --git a/src/sdp/output/rgx.d b/src/sdp/output/rgx.d index d559491..4bfa72a 100644 --- a/src/sdp/output/rgx.d +++ b/src/sdp/output/rgx.d @@ -16,7 +16,7 @@ static template SiSUoutputRgxInit() { static nbsp_and_space = ctRegex!(` [ ]`, "mg"); static nbsp_char_and_space = ctRegex!(`░[ ]`, "mg"); static special_markup_chars = ctRegex!(`[【】〖〗┥┝┤├¤░┘┙┚┼┿╂┊┏┚┆■]`, "mg"); - static src_pth_sst_or_ssm = ctRegex!(`^(?P[/]?(?:[a-zA-Z0-9._-]+/)*)(?P[a-zA-Z0-9._-]+[.]ss[tm])$`); + static src_pth_sst_or_ssm = ctRegex!(`^(?P[/]?(?:[a-zA-Z0-9._-]+/)*)(?P[a-zA-Z0-9._-]+[.](?Pss[tm]))$`); static src_pth_pod_sst_or_ssm = ctRegex!(`^(?P[/]?(?:[a-zA-Z0-9._-]+/)*)media/text/[a-z]{2}/(?P[a-zA-Z0-9._-]+[.]ss[tm])$`); static src_pth_contents = ctRegex!(`^(?P[/]?(?:[a-zA-Z0-9._-]+/)*)(?P[a-zA-Z0-9._-]+)/sisupod[.]manifest$`); static src_pth_pod_root = ctRegex!(`^(?P(?:[/]?(?:[a-zA-Z0-9._-]+/)*)(sisupod))$`); diff --git a/src/sdp/output/sqlite.d b/src/sdp/output/sqlite.d index 2409e3b..fc1843f 100644 --- a/src/sdp/output/sqlite.d +++ b/src/sdp/output/sqlite.d @@ -3,7 +3,79 @@ import sdp.output; import std.file, std.conv : to; +import d2sqlite3; +import std.typecons : Nullable; +mixin SiSUoutputRgxInit; +static auto rgx = Rgx(); long _metadata_tid_lastrowid; +template SQLiteHubBuildTablesAndPopulate() { + void SQLiteHubBuildTablesAndPopulate(D,I)( + auto ref const D doc_abstraction, + auto ref I doc_matters, + ) { + auto pth_sqlite = SiSUpathsSQLite!()(doc_matters.output_path); + pth_sqlite.base.mkdirRecurse; + auto db = Database(pth_sqlite.sqlite_file(doc_matters.env.pwd.baseName)); + template SQLiteDbStatementComposite() { + void SQLiteDbStatementComposite(Db,D,I)( + auto ref Db db, + auto ref const D doc_abstraction, + auto ref I doc_matters, + ) { + { + string _db_statement; + { + if ((doc_matters.opt.action.sqlite_create)) { + auto pth_sqlite = SiSUpathsSQLite!()(doc_matters.output_path); + pth_sqlite.base.mkdirRecurse; + _db_statement ~= SQLiteTablesReCreate!()(); + SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "TABLE RE-CREATE"); + } + if ((doc_matters.opt.action.sqlite_update)) { // TODO + _db_statement ~= SQLiteInsertMetadata!()(doc_matters); + SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "table INSERT MetaData"); + /+ get tid (lastrowid or max) for use in doc_objects table +/ + _metadata_tid_lastrowid = db.lastInsertRowid(); + _db_statement ~= SQLiteInsertDocObjectsLoop!()(doc_abstraction, doc_matters, _metadata_tid_lastrowid); // FIX + SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "table INSERT DocObjects"); + } + } + db.close; + } + } + } + SQLiteDbStatementComposite!()(db, doc_abstraction, doc_matters); + } +} +template SQLiteHubDiscreteBuildTablesAndPopulate() { + void SQLiteHubDiscreteBuildTablesAndPopulate(D,I)( + auto ref const D doc_abstraction, + auto ref I doc_matters, + ) { + auto pth_sqlite = SiSUpathsSQLiteDiscrete!()(doc_matters.output_path, doc_matters.src.language); + pth_sqlite.base.mkdirRecurse; + auto db = Database(pth_sqlite.sqlite_file(doc_matters.src.filename)); + template SQLiteDiscreteDbStatementComposite() { + void SQLiteDiscreteDbStatementComposite(Db,D,I)( + auto ref Db db, + auto ref const D doc_abstraction, + auto ref I doc_matters, + ) { + { + string _db_statement; + { + _db_statement ~= SQLiteTablesReCreate!()(); + _db_statement ~= SQLiteInsertMetadata!()(doc_matters); + _db_statement ~= SQLiteInsertDocObjectsLoop!()(doc_abstraction, doc_matters, 1); // FIX + SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "table CREATE Tables, INSERT DocObjects"); + } + db.close; + } + } + } + SQLiteDiscreteDbStatementComposite!()(db, doc_abstraction, doc_matters); + } +} template SQLiteDbRun() { void SQLiteDbRun(Db,St,O)( auto ref Db db, @@ -20,7 +92,9 @@ template SQLiteDbRun() { db_statement ~ "\nCOMMIT;\n" ); - if (!(opt_action.sqlite_create)) { + if (!(opt_action.sqlite_discrete) + && !(opt_action.sqlite_create) + ) { _metadata_tid_lastrowid = db.lastInsertRowid(); writeln("last row id: ", _metadata_tid_lastrowid); } @@ -37,1005 +111,968 @@ template SQLiteDbRun() { } } } -template SQLiteBuildTablesAndPopulate() { - void SQLiteBuildTablesAndPopulate(D,I)( - auto ref const D doc_abstraction, +template SQLinsertDelimiter() { + auto SQLinsertDelimiter(string _txt) { + _txt = _txt + .replaceAll(rgx.quotation_mark_sql_insert_delimiter, "$0$0"); + return _txt; + } +} +template SQLiteFormatAndLoadObject() { + auto SQLiteFormatAndLoadObject(I)( auto ref I doc_matters, ) { - import d2sqlite3; - import std.typecons : Nullable; mixin SiSUoutputRgxInit; - static auto rgx = Rgx(); - auto pth_sqlite = SiSUpathsSQLite!()(doc_matters.output_path); - pth_sqlite.base.mkdirRecurse; - auto db = Database(pth_sqlite.sqlite_file(doc_matters.env.pwd.baseName)); - template SQLiteDbStatementComposite() { - void SQLiteDbStatementComposite(Db,D,I)( - auto ref Db db, - auto ref const D doc_abstraction, - auto ref I doc_matters, + struct sqlite_format_and_load_objects { + auto generic_munge_sanitize_text_for_search( + string _txt, ) { - { - { - string _db_statement; - { - if ((doc_matters.opt.action.sqlite_create)) { - auto pth_sqlite = SiSUpathsSQLiteDiscrete!()(doc_matters.output_path, doc_matters.src.language); - pth_sqlite.base.mkdirRecurse; - _db_statement ~= SQLiteTablesReCreate!()(); - SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "TABLE RE-CREATE"); - } - if ((doc_matters.opt.action.sqlite_update)) { // TODO - _db_statement ~= SQLiteInsertMetadata!()(doc_matters); - SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "table INSERT MetaData"); - /+ get tid (lastrowid or max) for use in doc_objects table +/ - _metadata_tid_lastrowid = db.lastInsertRowid(); - _db_statement ~= SQLiteInsertDocObjectsLoop!()(doc_matters.xml.keys_seq.sql, _metadata_tid_lastrowid); // FIX - SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "table INSERT DocObjects"); - } + string _notes; + string _urls; + if (_txt.matchFirst(rgx.inline_notes_al_gen)) { + foreach (m; _txt.matchAll(rgx.inline_notes_al_gen_text)) { + _notes ~= "\n" ~ m["text"]; + } + _txt = _txt.replaceAll(rgx.inline_notes_al_gen, ""); + } + if (_txt.matchFirst(rgx.inline_link)) { + foreach (m; _txt.matchAll(rgx.inline_link)) { + if (m["link"].match(rgx.url)) { + _urls ~= "\n" ~ m["link"]; } - db.close; } + _txt = _txt.replaceAll(rgx.inline_link_clean, ""); + } + if (_notes.length > 0) { + _txt ~= _notes; + } + if (_urls.length > 0) { + _txt ~= _urls; } + if (doc_matters.opt.action.debug_do) { + writeln(_txt, "\n"); + } + debug(sql_text_clean) { + writeln(_txt); + } + return _txt; } - } - template SQLinsertDelimiter() { - auto SQLinsertDelimiter(string _txt) { + auto munge_html(O)( + auto return ref const O obj, + ) { + string _html_special_characters(string _txt){ + _txt = _txt + .replaceAll(rgx.xhtml_ampersand, "&") + .replaceAll(rgx.xhtml_quotation, """) + .replaceAll(rgx.xhtml_less_than, "<") + .replaceAll(rgx.xhtml_greater_than, ">") + .replaceAll(rgx.nbsp_char, " ") + .replaceAll(rgx.xhtml_line_break, "
"); + return _txt; + } + string _html_font_face(string _txt){ + _txt = _txt + .replaceAll(rgx.inline_emphasis, "$1") + .replaceAll(rgx.inline_bold, "$1") + .replaceAll(rgx.inline_underscore, "$1") + .replaceAll(rgx.inline_italics, "$1") + .replaceAll(rgx.inline_superscript, "$1") + .replaceAll(rgx.inline_subscript, "$1") + .replaceAll(rgx.inline_strike, "$1") + .replaceAll(rgx.inline_insert, "$1") + .replaceAll(rgx.inline_mono, "$1") + .replaceAll(rgx.inline_cite, "$1"); + return _txt; + } + string _notes; + string _urls; + string _txt = _html_font_face(_html_special_characters(obj.text)); + if (_txt.matchFirst(rgx.inline_notes_al_gen)) { + foreach (m; _txt.matchAll(rgx.inline_notes_al_gen_text)) { + _notes ~= "\n" ~ m["text"]; + } + _txt = _txt.replaceAll(rgx.inline_notes_al_gen_ref, "$1 "); + } + if (_txt.matchFirst(rgx.inline_link)) { + foreach (m; _txt.matchAll(rgx.inline_link)) { + } + _txt = _txt.replaceAll(rgx.inline_link_clean, ""); + } + if (_notes.length > 0) { + _txt ~= _notes; + } + if (doc_matters.opt.action.debug_do) { + writeln(_txt, "\n"); + } + return _txt; + } + string html_special_characters(string _txt){ _txt = _txt - .replaceAll(rgx.quotation_mark_sql_insert_delimiter, "$0$0"); + .replaceAll(rgx.xhtml_ampersand, "&") + .replaceAll(rgx.xhtml_quotation, """) + .replaceAll(rgx.xhtml_less_than, "<") + .replaceAll(rgx.xhtml_greater_than, ">") + .replaceAll(rgx.nbsp_char, " ") + .replaceAll(rgx.xhtml_line_break, "
"); return _txt; } - } - template SQLiteFormatAndLoadObject() { - auto SQLiteFormatAndLoadObject(I)( - auto ref I doc_matters, + string html_special_characters_code(string _txt){ + _txt = _txt + .replaceAll(rgx.xhtml_ampersand, "&") + .replaceAll(rgx.xhtml_quotation, """) + .replaceAll(rgx.xhtml_less_than, "<") + .replaceAll(rgx.xhtml_greater_than, ">") + .replaceAll(rgx.nbsp_char, " "); + return _txt; + } + string html_font_face(string _txt){ + _txt = _txt + .replaceAll(rgx.inline_emphasis, "$1") + .replaceAll(rgx.inline_bold, "$1") + .replaceAll(rgx.inline_underscore, "$1") + .replaceAll(rgx.inline_italics, "$1") + .replaceAll(rgx.inline_superscript, "$1") + .replaceAll(rgx.inline_subscript, "$1") + .replaceAll(rgx.inline_strike, "$1") + .replaceAll(rgx.inline_insert, "$1") + .replaceAll(rgx.inline_mono, "$1") + .replaceAll(rgx.inline_cite, "$1"); + return _txt; + } + auto html_heading(O)( + auto return ref const O obj, ) { - mixin SiSUoutputRgxInit; - struct sqlite_format_and_load_objects { - auto generic_munge_sanitize_text_for_search( - string _txt, - ) { - string _notes; - string _urls; - if (_txt.matchFirst(rgx.inline_notes_al_gen)) { - foreach (m; _txt.matchAll(rgx.inline_notes_al_gen_text)) { - _notes ~= "\n" ~ m["text"]; - } - _txt = _txt.replaceAll(rgx.inline_notes_al_gen, ""); - } - if (_txt.matchFirst(rgx.inline_link)) { - foreach (m; _txt.matchAll(rgx.inline_link)) { - if (m["link"].match(rgx.url)) { - _urls ~= "\n" ~ m["link"]; + string _txt = munge_html(obj); + string o = format(q"¶

+ %s +

¶", + obj.is_a, + _txt, + ); + return o; + } + auto html_para(O)( + auto return ref const O obj, + ) { + string _txt = munge_html(obj); + _txt = (obj.bullet) ? ("●  " ~ _txt) : _txt; + string o = format(q"¶

+ %s +

¶", + obj.is_a, + obj.indent_hang, + obj.indent_base, + _txt + ); + return o; + } + auto html_quote(O)( + auto return ref const O obj, + ) { + string _txt = munge_html(obj); + string o = format(q"¶

+ %s +

¶", + obj.is_a, + _txt + ); + return o; + } + auto html_group(O)( + auto return ref const O obj, + ) { + string _txt = munge_html(obj); + string o = format(q"¶

+ %s +

¶", + obj.is_a, + _txt + ); + return o; + } + auto html_block(O)( + auto return ref const O obj, + ) { + string _txt = munge_html(obj); + string o = format(q"¶ +

%s

¶", + obj.is_a, + _txt.stripRight + ); + return o; + } + auto html_verse(O)( + auto return ref const O obj, + ) { + string _txt = munge_html(obj); + string o = format(q"¶

%s

¶", + obj.is_a, + _txt + ); + return o; + } + auto html_code(O)( + auto return ref const O obj, + ) { + string _txt = html_special_characters_code(obj.text); + string o = format(q"¶

%s

¶", + obj.is_a, + _txt + ); + return o; + } + auto html_table(O)( + auto return ref const O obj, + ) { + auto _tablarize(O)( + auto return ref const O obj, + string _txt, + ) { + string[] _table_rows = _txt.split(rgx.table_delimiter_row); + string[] _table_cols; + string _table; + string _tablenote; + foreach(row_idx, row; _table_rows) { + _table_cols = row.split(rgx.table_delimiter_col); + _table ~= ""; + foreach(col_idx, cell; _table_cols) { + if ((_table_cols.length == 1) + && (_table_rows.length <= row_idx+2)) { // check row_idx+2 (rather than == ++row_idx) + _tablenote ~= cell; + } else { + string _col_is = (row_idx == 0 && obj.table_heading) ? "th" : "td"; + string _align = ("style=\"text-align:" + ~ ((obj.table_column_aligns[col_idx] == "l") + ? "left\"" : "right\"")); + _table ~= "<" + ~ _col_is + ~ " width=\"" + ~ obj.table_column_widths[col_idx].to!string + ~ "%\" " + ~ _align + ~ ">"; + _table ~= cell; + _table ~= ""; } } - _txt = _txt.replaceAll(rgx.inline_link_clean, ""); - } - if (_notes.length > 0) { - _txt ~= _notes; - } - if (_urls.length > 0) { - _txt ~= _urls; - } - if (doc_matters.opt.action.debug_do) { - writeln(_txt, "\n"); - } - debug(sql_text_clean) { - writeln(_txt); - } - return _txt; - } - auto munge_html(O)( - auto return ref const O obj, - ) { - string _html_special_characters(string _txt){ - _txt = _txt - .replaceAll(rgx.xhtml_ampersand, "&") - .replaceAll(rgx.xhtml_quotation, """) - .replaceAll(rgx.xhtml_less_than, "<") - .replaceAll(rgx.xhtml_greater_than, ">") - .replaceAll(rgx.nbsp_char, " ") - .replaceAll(rgx.xhtml_line_break, "
"); - return _txt; - } - string _html_font_face(string _txt){ - _txt = _txt - .replaceAll(rgx.inline_emphasis, "$1") - .replaceAll(rgx.inline_bold, "$1") - .replaceAll(rgx.inline_underscore, "$1") - .replaceAll(rgx.inline_italics, "$1") - .replaceAll(rgx.inline_superscript, "$1") - .replaceAll(rgx.inline_subscript, "$1") - .replaceAll(rgx.inline_strike, "$1") - .replaceAll(rgx.inline_insert, "$1") - .replaceAll(rgx.inline_mono, "$1") - .replaceAll(rgx.inline_cite, "$1"); - return _txt; + _table ~= ""; } - string _notes; - string _urls; - string _txt = _html_font_face(_html_special_characters(obj.text)); - if (_txt.matchFirst(rgx.inline_notes_al_gen)) { - foreach (m; _txt.matchAll(rgx.inline_notes_al_gen_text)) { - _notes ~= "\n" ~ m["text"]; - } - _txt = _txt.replaceAll(rgx.inline_notes_al_gen_ref, "$1 "); - } - if (_txt.matchFirst(rgx.inline_link)) { - foreach (m; _txt.matchAll(rgx.inline_link)) { - } - _txt = _txt.replaceAll(rgx.inline_link_clean, ""); - } - if (_notes.length > 0) { - _txt ~= _notes; - } - if (doc_matters.opt.action.debug_do) { - writeln(_txt, "\n"); - } - return _txt; - } - string html_special_characters(string _txt){ - _txt = _txt - .replaceAll(rgx.xhtml_ampersand, "&") - .replaceAll(rgx.xhtml_quotation, """) - .replaceAll(rgx.xhtml_less_than, "<") - .replaceAll(rgx.xhtml_greater_than, ">") - .replaceAll(rgx.nbsp_char, " ") - .replaceAll(rgx.xhtml_line_break, "
"); - return _txt; - } - string html_special_characters_code(string _txt){ - _txt = _txt - .replaceAll(rgx.xhtml_ampersand, "&") - .replaceAll(rgx.xhtml_quotation, """) - .replaceAll(rgx.xhtml_less_than, "<") - .replaceAll(rgx.xhtml_greater_than, ">") - .replaceAll(rgx.nbsp_char, " "); - return _txt; - } - string html_font_face(string _txt){ - _txt = _txt - .replaceAll(rgx.inline_emphasis, "$1") - .replaceAll(rgx.inline_bold, "$1") - .replaceAll(rgx.inline_underscore, "$1") - .replaceAll(rgx.inline_italics, "$1") - .replaceAll(rgx.inline_superscript, "$1") - .replaceAll(rgx.inline_subscript, "$1") - .replaceAll(rgx.inline_strike, "$1") - .replaceAll(rgx.inline_insert, "$1") - .replaceAll(rgx.inline_mono, "$1") - .replaceAll(rgx.inline_cite, "$1"); - return _txt; - } - auto html_heading(O)( - auto return ref const O obj, - ) { - string _txt = munge_html(obj); - string o = format(q"¶

- %s -

¶", - obj.is_a, - _txt, - ); - return o; - } - auto html_para(O)( - auto return ref const O obj, - ) { - string _txt = munge_html(obj); - _txt = (obj.bullet) ? ("●  " ~ _txt) : _txt; - string o = format(q"¶

- %s -

¶", - obj.is_a, - obj.indent_hang, - obj.indent_base, - _txt - ); - return o; - } - auto html_quote(O)( - auto return ref const O obj, - ) { - string _txt = munge_html(obj); - string o = format(q"¶

- %s -

¶", - obj.is_a, - _txt - ); - return o; - } - auto html_group(O)( - auto return ref const O obj, - ) { - string _txt = munge_html(obj); - string o = format(q"¶

- %s -

¶", - obj.is_a, - _txt - ); - return o; + auto t = tuple( + _table, + _tablenote, + ); + return t; + } + string _txt = munge_html(obj); + auto t = _tablarize(obj, _txt); + _txt = t[0]; + string _note = t[1]; + string o = format(q"¶

+ + %s +
+ %s +

¶", + obj.is_a, + _txt, + _note + ); + return o; + } + auto sqlite_load_string(O,Dm)( + auto return ref const O obj, + auto return ref Dm doc_matters, + ) { + string o; + return o; + } + auto postgresql_load_string(O,Dm)( + auto return ref const O obj, + auto return ref Dm doc_matters, + ) { + string o; + return o; + } + string sqlite_statement(O)( + auto return ref const O obj, + string _txt, + string _html, + ) { + void _sql_exe(O)( + string _sql, + ) { + writeln(_html); + writeln(_sql); + } + string _sql; + return _sql; + } + auto heading(O)( + auto return ref const O obj, + ) { + string[string] obj_txt = [ + "text": generic_munge_sanitize_text_for_search(obj.text), + "html": html_heading(obj) + ]; + if (doc_matters.opt.action.debug_do) { + debug(sql_txt) { + writeln(obj_txt["text"]); } - auto html_block(O)( - auto return ref const O obj, - ) { - string _txt = munge_html(obj); - string o = format(q"¶ -

%s

¶", - obj.is_a, - _txt.stripRight - ); - return o; + debug(sql_html) { + writeln(obj_txt["html"]); } - auto html_verse(O)( - auto return ref const O obj, - ) { - string _txt = munge_html(obj); - string o = format(q"¶

%s

¶", - obj.is_a, - _txt - ); - return o; + } else { + // load sql + } + return obj_txt; + } + auto para(O)( + auto return ref const O obj, + ) { + string[string] obj_txt = [ + "text": generic_munge_sanitize_text_for_search(obj.text), + "html": html_para(obj) + ]; + if (doc_matters.opt.action.debug_do) { + debug(sql_txt) { + writeln(obj_txt["text"]); } - auto html_code(O)( - auto return ref const O obj, - ) { - string _txt = html_special_characters_code(obj.text); - string o = format(q"¶

%s

¶", - obj.is_a, - _txt - ); - return o; + debug(sql_html) { + writeln(obj_txt["html"]); } - auto html_table(O)( - auto return ref const O obj, - ) { - auto _tablarize(O)( - auto return ref const O obj, - string _txt, - ) { - string[] _table_rows = _txt.split(rgx.table_delimiter_row); - string[] _table_cols; - string _table; - string _tablenote; - foreach(row_idx, row; _table_rows) { - _table_cols = row.split(rgx.table_delimiter_col); - _table ~= ""; - foreach(col_idx, cell; _table_cols) { - if ((_table_cols.length == 1) - && (_table_rows.length <= row_idx+2)) { // check row_idx+2 (rather than == ++row_idx) - _tablenote ~= cell; - } else { - string _col_is = (row_idx == 0 && obj.table_heading) ? "th" : "td"; - string _align = ("style=\"text-align:" - ~ ((obj.table_column_aligns[col_idx] == "l") - ? "left\"" : "right\"")); - _table ~= "<" - ~ _col_is - ~ " width=\"" - ~ obj.table_column_widths[col_idx].to!string - ~ "%\" " - ~ _align - ~ ">"; - _table ~= cell; - _table ~= ""; - } - } - _table ~= ""; - } - auto t = tuple( - _table, - _tablenote, - ); - return t; - } - string _txt = munge_html(obj); - auto t = _tablarize(obj, _txt); - _txt = t[0]; - string _note = t[1]; - string o = format(q"¶

- - %s -
- %s -

¶", - obj.is_a, - _txt, - _note - ); - return o; + } else { + // load sql + } + return obj_txt; + } + auto quote(O)( + auto return ref const O obj, + ) { + string[string] obj_txt = [ + "text": generic_munge_sanitize_text_for_search(obj.text), + "html": html_quote(obj) + ]; + if (doc_matters.opt.action.debug_do) { + debug(sql_txt) { + writeln(obj_txt["text"]); } - auto sqlite_load_string(O,Dm)( - auto return ref const O obj, - auto return ref Dm doc_matters, - ) { - string o; - return o; + debug(sql_html) { + writeln(obj_txt["html"]); } - auto postgresql_load_string(O,Dm)( - auto return ref const O obj, - auto return ref Dm doc_matters, - ) { - string o; - return o; + } else { + // load sql + } + return obj_txt; + } + auto group(O)( + auto return ref const O obj, + ) { + string[string] obj_txt = [ + "text": generic_munge_sanitize_text_for_search(obj.text), + "html": html_group(obj) + ]; + if (doc_matters.opt.action.debug_do) { + debug(sql_txt) { + writeln(obj_txt["text"]); } - string sqlite_statement(O)( - auto return ref const O obj, - string _txt, - string _html, - ) { - void _sql_exe(O)( - string _sql, - ) { - writeln(_html); - writeln(_sql); - } - string _sql; - return _sql; + debug(sql_html) { + writeln(obj_txt["html"]); } - auto heading(O)( - auto return ref const O obj, - ) { - string[string] obj_txt = [ - "text": generic_munge_sanitize_text_for_search(obj.text), - "html": html_heading(obj) - ]; - if (doc_matters.opt.action.debug_do) { - debug(sql_txt) { - writeln(obj_txt["text"]); - } - debug(sql_html) { - writeln(obj_txt["html"]); - } - } else { - // load sql - } - return obj_txt; + } else { + // load sql + } + return obj_txt; + } + auto block(O)( + auto return ref const O obj, + ) { + string[string] obj_txt = [ + "text": generic_munge_sanitize_text_for_search(obj.text), + "html": html_block(obj) + ]; + if (doc_matters.opt.action.debug_do) { + debug(sql_txt) { + writeln(obj_txt["text"]); } - auto para(O)( - auto return ref const O obj, - ) { - string[string] obj_txt = [ - "text": generic_munge_sanitize_text_for_search(obj.text), - "html": html_para(obj) - ]; - if (doc_matters.opt.action.debug_do) { - debug(sql_txt) { - writeln(obj_txt["text"]); - } - debug(sql_html) { - writeln(obj_txt["html"]); - } - } else { - // load sql - } - return obj_txt; + debug(sql_html) { + writeln(obj_txt["html"]); } - auto quote(O)( - auto return ref const O obj, - ) { - string[string] obj_txt = [ - "text": generic_munge_sanitize_text_for_search(obj.text), - "html": html_quote(obj) - ]; - if (doc_matters.opt.action.debug_do) { - debug(sql_txt) { - writeln(obj_txt["text"]); - } - debug(sql_html) { - writeln(obj_txt["html"]); - } - } else { - // load sql - } - return obj_txt; + } else { + // load sql + } + return obj_txt; + } + auto verse(O)( + auto return ref const O obj, + ) { + string[string] obj_txt = [ + "text": generic_munge_sanitize_text_for_search(obj.text), + "html": html_verse(obj) + ]; + if (doc_matters.opt.action.debug_do) { + debug(sql_txt) { + writeln(obj_txt["text"]); } - auto group(O)( - auto return ref const O obj, - ) { - string[string] obj_txt = [ - "text": generic_munge_sanitize_text_for_search(obj.text), - "html": html_group(obj) - ]; - if (doc_matters.opt.action.debug_do) { - debug(sql_txt) { - writeln(obj_txt["text"]); - } - debug(sql_html) { - writeln(obj_txt["html"]); - } - } else { - // load sql - } - return obj_txt; + debug(sql_html) { + writeln(obj_txt["html"]); } - auto block(O)( - auto return ref const O obj, - ) { - string[string] obj_txt = [ - "text": generic_munge_sanitize_text_for_search(obj.text), - "html": html_block(obj) - ]; - if (doc_matters.opt.action.debug_do) { - debug(sql_txt) { - writeln(obj_txt["text"]); - } - debug(sql_html) { - writeln(obj_txt["html"]); - } - } else { - // load sql - } - return obj_txt; + } else { + // load sql + } + return obj_txt; + } + auto code(O)( + auto return ref const O obj, + ) { + string[string] obj_txt = [ + "text": generic_munge_sanitize_text_for_search(obj.text), + "html": html_code(obj) + ]; + if (doc_matters.opt.action.debug_do) { + debug(sql_txt) { + writeln(obj_txt["text"]); } - auto verse(O)( - auto return ref const O obj, - ) { - string[string] obj_txt = [ - "text": generic_munge_sanitize_text_for_search(obj.text), - "html": html_verse(obj) - ]; - if (doc_matters.opt.action.debug_do) { - debug(sql_txt) { - writeln(obj_txt["text"]); - } - debug(sql_html) { - writeln(obj_txt["html"]); - } - } else { - // load sql - } - return obj_txt; + debug(sql_html) { + writeln(obj_txt["html"]); } - auto code(O)( - auto return ref const O obj, - ) { - string[string] obj_txt = [ - "text": generic_munge_sanitize_text_for_search(obj.text), - "html": html_code(obj) - ]; - if (doc_matters.opt.action.debug_do) { - debug(sql_txt) { - writeln(obj_txt["text"]); - } - debug(sql_html) { - writeln(obj_txt["html"]); - } - } else { - // load sql - } - return obj_txt; + } else { + // load sql + } + return obj_txt; + } + auto table(O)( + auto return ref const O obj, + ) { + string[string] obj_txt = [ + "text": generic_munge_sanitize_text_for_search(obj.text), + "html": html_table(obj) + ]; + if (doc_matters.opt.action.debug_do) { + debug(sql_txt) { + writeln(obj_txt["text"]); } - auto table(O)( - auto return ref const O obj, - ) { - string[string] obj_txt = [ - "text": generic_munge_sanitize_text_for_search(obj.text), - "html": html_table(obj) - ]; - if (doc_matters.opt.action.debug_do) { - debug(sql_txt) { - writeln(obj_txt["text"]); - } - debug(sql_html) { - writeln(obj_txt["html"]); - } - } else { - // load sql - } - return obj_txt; + debug(sql_html) { + writeln(obj_txt["html"]); } + } else { + // load sql } - return sqlite_format_and_load_objects(); + return obj_txt; } } - template SQLiteTablesReCreate() { - string SQLiteTablesReCreate()() { - string _sql_instruct; - _sql_instruct = format(q"¶ - DROP INDEX IF EXISTS ocn; - DROP INDEX IF EXISTS digest_clean; - DROP INDEX IF EXISTS digest_all; - DROP INDEX IF EXISTS clean; - DROP INDEX IF EXISTS lev0; - DROP INDEX IF EXISTS lev1; - DROP INDEX IF EXISTS lev2; - DROP INDEX IF EXISTS lev3; - DROP INDEX IF EXISTS lev4; - DROP INDEX IF EXISTS lev5; - DROP INDEX IF EXISTS lev6; - DROP INDEX IF EXISTS title; - DROP INDEX IF EXISTS creator_author; - DROP INDEX IF EXISTS src_filename; - DROP INDEX IF EXISTS language_document_char; - DROP INDEX IF EXISTS classify_topic_register; - DROP TABLE IF EXISTS metadata_and_text; - DROP TABLE IF EXISTS doc_objects; - DROP TABLE IF EXISTS urls; - CREATE TABLE metadata_and_text ( - tid INTEGER PRIMARY KEY, - /* src_filename_composite VARCHAR(256) NOT NULL UNIQUE, /* z pod name if any + src filename + language code */ - title VARCHAR(800) NOT NULL, - title_main VARCHAR(400) NOT NULL, - title_sub VARCHAR(400) NULL, - title_short VARCHAR(400) NULL, - title_edition VARCHAR(10) NULL, - title_note VARCHAR(2500) NULL, - title_language VARCHAR(100) NULL, - title_language_char VARCHAR(6) NULL, - creator_author VARCHAR(600) NOT NULL, - creator_author_email VARCHAR(100) NULL, - creator_author_hon VARCHAR(100) NULL, - creator_author_nationality VARCHAR(100) NULL, - creator_editor VARCHAR(600) NULL, - creator_contributor VARCHAR(600) NULL, - creator_illustrator VARCHAR(600) NULL, - creator_photographer VARCHAR(600) NULL, - creator_translator VARCHAR(600) NULL, - creator_prepared_by VARCHAR(600) NULL, - creator_digitized_by VARCHAR(600) NULL, - creator_audio VARCHAR(600) NULL, - creator_video VARCHAR(600) NULL, - language_document VARCHAR(100) NULL, - language_document_char VARCHAR(6) NOT NULL, - language_original VARCHAR(100) NULL, - language_original_char VARCHAR(6) NULL, - date_added_to_site VARCHAR(10) NULL, - date_available VARCHAR(10) NULL, - date_created VARCHAR(10) NULL, - date_issued VARCHAR(10) NULL, - date_modified VARCHAR(10) NULL, - date_published VARCHAR(10) NULL, - date_valid VARCHAR(10) NULL, - date_translated VARCHAR(10) NULL, - date_original_publication VARCHAR(10) NULL, - date_generated VARCHAR(10) NULL, - publisher VARCHAR(600) NULL, - original_publisher VARCHAR(600) NULL, - original_language VARCHAR(100) NULL, - original_language_char VARCHAR(6) NULL, - original_source VARCHAR(600) NULL, - original_institution VARCHAR(600) NULL, - original_nationality VARCHAR(100) NULL, - original_title VARCHAR(800) NULL, - rights_copyright VARCHAR(2500) NULL, - rights_copyright_audio VARCHAR(2500) NULL, - rights_copyright_cover VARCHAR(2500) NULL, - rights_copyright_illustrations VARCHAR(2500) NULL, - rights_copyright_photographs VARCHAR(2500) NULL, - rights_copyright_text VARCHAR(2500) NULL, - rights_copyright_translation VARCHAR(2500) NULL, - rights_copyright_video VARCHAR(2500) NULL, - rights_license VARCHAR(2500) NULL, - identifier_oclc VARCHAR(30) NULL, - identifier_isbn VARCHAR(16) NULL, - classify_topic_register VARCHAR(2500) NULL, - classify_subject VARCHAR(600) NULL, - classify_loc VARCHAR(30) NULL, - classify_dewey VARCHAR(30) NULL, - classify_keywords VARCHAR(600) NULL, - notes_abstract TEXT NULL, - notes_description TEXT NULL, - notes_comment TEXT NULL, - notes_coverage VARCHAR(200) NULL, - notes_relation VARCHAR(200) NULL, - notes_history VARCHAR(600) NULL, - notes_type VARCHAR(600) NULL, - notes_format VARCHAR(600) NULL, - notes_prefix TEXT NULL, - notes_prefix_a TEXT NULL, - notes_prefix_b TEXT NULL, - notes_suffix TEXT NULL, - src_filename VARCHAR(256) NOT NULL, - src_fingerprint VARCHAR(256) NULL, - src_filesize VARCHAR(10) NULL, - src_wordcount VARCHAR(10) NULL, - pod_name VARCHAR(256) NULL, /* zipped pod, work to be done here */ - pod_fingerprint VARCHAR(256) NULL, /* zipped pod, work to be done here */ - pod_size VARCHAR(10) NULL, /* zipped pod, work to be done here */ - src_text TEXT NULL, - fulltext TEXT NULL, - links TEXT NULL - ); - CREATE TABLE doc_objects ( - lid INTEGER PRIMARY KEY, - metadata_tid INTEGER REFERENCES metadata_and_text, - ocn SMALLINT, - ocnd VARCHAR(6), - ocns VARCHAR(6), - clean TEXT NULL, - body TEXT NULL, - book_idx TEXT NULL, - seg VARCHAR(256) NULL, - lev_an VARCHAR(1), - lev SMALLINT NULL, - lev0 SMALLINT, - lev1 SMALLINT, - lev2 SMALLINT, - lev3 SMALLINT, - lev4 SMALLINT, - lev5 SMALLINT, - lev6 SMALLINT, - lev7 SMALLINT, - en_a SMALLINT NULL, - en_z SMALLINT NULL, - en_a_asterisk SMALLINT NULL, - en_z_asterisk SMALLINT NULL, - en_a_plus SMALLINT NULL, - en_z_plus SMALLINT NULL, - t_of VARCHAR(16), - t_is VARCHAR(16), - node VARCHAR(16) NULL, - parent VARCHAR(16) NULL, - digest_clean CHAR(256), - digest_all CHAR(256), - types CHAR(1) NULL - ); - CREATE INDEX idx_ocn ON doc_objects(ocn); - CREATE INDEX idx_digest_clean ON doc_objects(digest_clean); - CREATE INDEX idx_digest_all ON doc_objects(digest_all); - CREATE INDEX idx_clean ON doc_objects(clean); - CREATE INDEX idx_lev0 ON doc_objects(lev0); - CREATE INDEX idx_lev1 ON doc_objects(lev1); - CREATE INDEX idx_lev2 ON doc_objects(lev2); - CREATE INDEX idx_lev3 ON doc_objects(lev3); - CREATE INDEX idx_lev4 ON doc_objects(lev4); - CREATE INDEX idx_lev5 ON doc_objects(lev5); - CREATE INDEX idx_lev6 ON doc_objects(lev6); - CREATE INDEX idx_title ON metadata_and_text(title); - CREATE INDEX idx_author ON metadata_and_text(creator_author); - CREATE INDEX idx_filename ON metadata_and_text(src_filename); - CREATE INDEX idx_language ON metadata_and_text(language_document_char); - CREATE INDEX idx_topics ON metadata_and_text(classify_topic_register); - ¶",); - return _sql_instruct; - } - } - template SQLiteInsertMetadata() { - string SQLiteInsertMetadata(I)( - auto ref I doc_matters, - ) { - string _insert_metadata; - _insert_metadata = format(q"¶ - INSERT INTO metadata_and_text ( - src_filename, - title, - title_main, - title_sub, - title_short, - title_edition, - title_language, - classify_dewey, - classify_keywords, - classify_loc, - classify_subject, - classify_topic_register, - creator_author, - creator_author_email, - creator_illustrator, - creator_translator, - date_added_to_site, - date_available, - date_created, - date_issued, - date_modified, - date_published, - date_valid, - identifier_isbn, - identifier_oclc, - language_document, - language_document_char, - notes_abstract, - notes_description, - original_publisher, - original_language, - original_language_char, - original_source, - original_title, - publisher, - rights_copyright, - rights_copyright_audio, - rights_copyright_cover, - rights_copyright_illustrations, - rights_copyright_photographs, - rights_copyright_text, - rights_copyright_translation, - rights_copyright_video, - rights_license - ) - VALUES ( - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s' - ); - ¶", - SQLinsertDelimiter!()(doc_matters.src.filename), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_full), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_main), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_subtitle), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_short), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_edition), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_language), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_dewey), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_keywords), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_loc), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_subject), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_topic_register), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_author), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_author_email), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_illustrator), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_translator), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_added_to_site), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_available), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_created), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_issued), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_modified), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_published), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_valid), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.identifier_isbn), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.identifier_oclc), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.language_document), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.language_document_char), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.notes_abstract), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.notes_description), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_publisher), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_language), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_language_char), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_source), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_title), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.publisher), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_audio), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_cover), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_illustrations), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_photographs), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_text), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_translation), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_video), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_license) + return sqlite_format_and_load_objects(); + } +} +template SQLiteTablesReCreate() { + string SQLiteTablesReCreate()() { + string _sql_instruct; + _sql_instruct = format(q"¶ + DROP INDEX IF EXISTS ocn; + DROP INDEX IF EXISTS digest_clean; + DROP INDEX IF EXISTS digest_all; + DROP INDEX IF EXISTS clean; + DROP INDEX IF EXISTS lev0; + DROP INDEX IF EXISTS lev1; + DROP INDEX IF EXISTS lev2; + DROP INDEX IF EXISTS lev3; + DROP INDEX IF EXISTS lev4; + DROP INDEX IF EXISTS lev5; + DROP INDEX IF EXISTS lev6; + DROP INDEX IF EXISTS title; + DROP INDEX IF EXISTS creator_author; + DROP INDEX IF EXISTS src_filename; + DROP INDEX IF EXISTS language_document_char; + DROP INDEX IF EXISTS classify_topic_register; + DROP TABLE IF EXISTS metadata_and_text; + DROP TABLE IF EXISTS doc_objects; + DROP TABLE IF EXISTS urls; + CREATE TABLE metadata_and_text ( + tid INTEGER PRIMARY KEY, + src_composite_id_per_txt VARCHAR(256) NOT NULL UNIQUE, /* z pod name if any + src filename + language code */ + src_composite_id_per_pod VARCHAR(256) NOT NULL UNIQUE, /* z pod name if any + src filename + language code */ + title VARCHAR(800) NOT NULL, + title_main VARCHAR(400) NOT NULL, + title_sub VARCHAR(400) NULL, + title_short VARCHAR(400) NULL, + title_edition VARCHAR(10) NULL, + title_note VARCHAR(2500) NULL, + title_language VARCHAR(100) NULL, + title_language_char VARCHAR(6) NULL, + creator_author VARCHAR(600) NOT NULL, + creator_author_email VARCHAR(100) NULL, + creator_author_hon VARCHAR(100) NULL, + creator_author_nationality VARCHAR(100) NULL, + creator_editor VARCHAR(600) NULL, + creator_contributor VARCHAR(600) NULL, + creator_illustrator VARCHAR(600) NULL, + creator_photographer VARCHAR(600) NULL, + creator_translator VARCHAR(600) NULL, + creator_prepared_by VARCHAR(600) NULL, + creator_digitized_by VARCHAR(600) NULL, + creator_audio VARCHAR(600) NULL, + creator_video VARCHAR(600) NULL, + language_document VARCHAR(100) NULL, + language_document_char VARCHAR(6) NOT NULL, + language_original VARCHAR(100) NULL, + language_original_char VARCHAR(6) NULL, + date_added_to_site VARCHAR(10) NULL, + date_available VARCHAR(10) NULL, + date_created VARCHAR(10) NULL, + date_issued VARCHAR(10) NULL, + date_modified VARCHAR(10) NULL, + date_published VARCHAR(10) NULL, + date_valid VARCHAR(10) NULL, + date_translated VARCHAR(10) NULL, + date_original_publication VARCHAR(10) NULL, + date_generated VARCHAR(10) NULL, + publisher VARCHAR(600) NULL, + original_publisher VARCHAR(600) NULL, + original_language VARCHAR(100) NULL, + original_language_char VARCHAR(6) NULL, + original_source VARCHAR(600) NULL, + original_institution VARCHAR(600) NULL, + original_nationality VARCHAR(100) NULL, + original_title VARCHAR(800) NULL, + rights_copyright VARCHAR(2500) NULL, + rights_copyright_audio VARCHAR(2500) NULL, + rights_copyright_cover VARCHAR(2500) NULL, + rights_copyright_illustrations VARCHAR(2500) NULL, + rights_copyright_photographs VARCHAR(2500) NULL, + rights_copyright_text VARCHAR(2500) NULL, + rights_copyright_translation VARCHAR(2500) NULL, + rights_copyright_video VARCHAR(2500) NULL, + rights_license VARCHAR(2500) NULL, + identifier_oclc VARCHAR(30) NULL, + identifier_isbn VARCHAR(16) NULL, + classify_topic_register VARCHAR(2500) NULL, + classify_subject VARCHAR(600) NULL, + classify_loc VARCHAR(30) NULL, + classify_dewey VARCHAR(30) NULL, + classify_keywords VARCHAR(600) NULL, + notes_abstract TEXT NULL, + notes_description TEXT NULL, + notes_comment TEXT NULL, + notes_coverage VARCHAR(200) NULL, + notes_relation VARCHAR(200) NULL, + notes_history VARCHAR(600) NULL, + notes_type VARCHAR(600) NULL, + notes_format VARCHAR(600) NULL, + notes_prefix TEXT NULL, + notes_prefix_a TEXT NULL, + notes_prefix_b TEXT NULL, + notes_suffix TEXT NULL, + src_filename VARCHAR(256) NOT NULL, + src_fingerprint VARCHAR(256) NULL, + src_filesize VARCHAR(10) NULL, + src_wordcount VARCHAR(10) NULL, + pod_name VARCHAR(256) NULL, /* zipped pod, work to be done here */ + pod_fingerprint VARCHAR(256) NULL, /* zipped pod, work to be done here */ + pod_size VARCHAR(10) NULL, /* zipped pod, work to be done here */ + src_text TEXT NULL, + fulltext TEXT NULL, + links TEXT NULL + ); + CREATE TABLE doc_objects ( + lid INTEGER PRIMARY KEY, + metadata_tid INTEGER REFERENCES metadata_and_text, + ocn SMALLINT, + ocnd VARCHAR(6), + ocns VARCHAR(6), + clean TEXT NULL, + body TEXT NULL, + book_idx TEXT NULL, + seg VARCHAR(256) NULL, + lev_an VARCHAR(1), + lev SMALLINT NULL, + lev0 SMALLINT, + lev1 SMALLINT, + lev2 SMALLINT, + lev3 SMALLINT, + lev4 SMALLINT, + lev5 SMALLINT, + lev6 SMALLINT, + lev7 SMALLINT, + en_a SMALLINT NULL, + en_z SMALLINT NULL, + en_a_asterisk SMALLINT NULL, + en_z_asterisk SMALLINT NULL, + en_a_plus SMALLINT NULL, + en_z_plus SMALLINT NULL, + t_of VARCHAR(16), + t_is VARCHAR(16), + node VARCHAR(16) NULL, + parent VARCHAR(16) NULL, + digest_clean CHAR(256), + digest_all CHAR(256), + types CHAR(1) NULL + ); + CREATE INDEX idx_ocn ON doc_objects(ocn); + CREATE INDEX idx_digest_clean ON doc_objects(digest_clean); + CREATE INDEX idx_digest_all ON doc_objects(digest_all); + CREATE INDEX idx_clean ON doc_objects(clean); + CREATE INDEX idx_lev0 ON doc_objects(lev0); + CREATE INDEX idx_lev1 ON doc_objects(lev1); + CREATE INDEX idx_lev2 ON doc_objects(lev2); + CREATE INDEX idx_lev3 ON doc_objects(lev3); + CREATE INDEX idx_lev4 ON doc_objects(lev4); + CREATE INDEX idx_lev5 ON doc_objects(lev5); + CREATE INDEX idx_lev6 ON doc_objects(lev6); + CREATE INDEX idx_title ON metadata_and_text(title); + CREATE INDEX idx_author ON metadata_and_text(creator_author); + CREATE INDEX idx_filename ON metadata_and_text(src_filename); + CREATE INDEX idx_language ON metadata_and_text(language_document_char); + CREATE INDEX idx_topics ON metadata_and_text(classify_topic_register); + ¶",); + return _sql_instruct; + } +} +template SQLiteInsertMetadata() { + string SQLiteInsertMetadata(I)( + auto ref I doc_matters, + ) { + string _insert_metadata; + _insert_metadata = format(q"¶ + INSERT INTO metadata_and_text ( + src_filename, + src_composite_id_per_txt, + src_composite_id_per_pod, + title, + title_main, + title_sub, + title_short, + title_edition, + title_language, + classify_dewey, + classify_keywords, + classify_loc, + classify_subject, + classify_topic_register, + creator_author, + creator_author_email, + creator_illustrator, + creator_translator, + date_added_to_site, + date_available, + date_created, + date_issued, + date_modified, + date_published, + date_valid, + identifier_isbn, + identifier_oclc, + language_document, + language_document_char, + notes_abstract, + notes_description, + original_publisher, + original_language, + original_language_char, + original_source, + original_title, + publisher, + rights_copyright, + rights_copyright_audio, + rights_copyright_cover, + rights_copyright_illustrations, + rights_copyright_photographs, + rights_copyright_text, + rights_copyright_translation, + rights_copyright_video, + rights_license + ) + VALUES ( + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s' + ); + ¶", + SQLinsertDelimiter!()(doc_matters.src.filename), + SQLinsertDelimiter!()(doc_matters.src.docname_composite_unique_per_src_doc), + SQLinsertDelimiter!()(doc_matters.src.docname_composite_unique_per_src_pod), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_full), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_main), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_subtitle), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_short), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_edition), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_language), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_dewey), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_keywords), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_loc), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_subject), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_topic_register), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_author), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_author_email), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_illustrator), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_translator), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_added_to_site), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_available), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_created), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_issued), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_modified), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_published), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_valid), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.identifier_isbn), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.identifier_oclc), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.language_document), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.language_document_char), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.notes_abstract), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.notes_description), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_publisher), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_language), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_language_char), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_source), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_title), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.publisher), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_audio), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_cover), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_illustrations), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_photographs), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_text), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_translation), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_video), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_license) + ); + return _insert_metadata; + } +} +template SQLiteInsertDocObjectsLoop() { + string SQLiteInsertDocObjectsLoop(D,I,X)( + auto ref const D doc_abstraction, + auto ref I doc_matters, + auto ref X _metadata_tid, + ) { + string insertDocObjectsRow(O)(O obj) { + string _insert_doc_objects_row; + _insert_doc_objects_row = format(q"¶ + INSERT INTO doc_objects ( + metadata_tid, + ocn, + clean, + body, + lev, + t_of, + t_is + ) + VALUES ( + %d, + %s, + '%s', + '%s', + %s, + '%s', + '%s' ); - return _insert_metadata; - } + ¶", + _metadata_tid, + obj.ocn, + SQLinsertDelimiter!()(obj_txt["text"]), + SQLinsertDelimiter!()(obj_txt["html"]), + obj.heading_lev_markup, + obj.is_of, + obj.is_a, + ); + return _insert_doc_objects_row; } - template SQLiteInsertDocObjectsLoop() { - string SQLiteInsertDocObjectsLoop(P,I)( - auto ref P doc_parts, - auto ref I _metadata_tid, - ) { - string insertDocObjectsRow(O)(O obj) { - string _insert_doc_objects_row; - _insert_doc_objects_row = format(q"¶ - INSERT INTO doc_objects ( - metadata_tid, - ocn, - clean, - body, - lev, - t_of, - t_is - ) - VALUES ( - %d, - %s, - '%s', - '%s', - %s, - '%s', - '%s' - ); - ¶", - _metadata_tid, - obj.ocn, - SQLinsertDelimiter!()(obj_txt["text"]), - SQLinsertDelimiter!()(obj_txt["html"]), - obj.heading_lev_markup, - obj.is_of, - obj.is_a, - ); - return _insert_doc_objects_row; - } - auto format_and_sqlite_load = SQLiteFormatAndLoadObject!()(doc_matters); - string[string] obj_txt; - string doc_text; - string[] _insert_doc_objects; - foreach (part; doc_parts) { - foreach (obj; doc_abstraction[part]) { - switch (obj.of_part) { - case "frontmatter": assert(part == "head", part); - switch (obj.is_of) { - case "para": - switch (obj.is_a) { - case "heading": - obj_txt = format_and_sqlite_load.heading(obj); - break; - default: - if ((doc_matters.opt.action.debug_do)) { - writeln(__FILE__, ":", __LINE__, ": ", obj.is_a); - } - break; - } - break; - default: - if ((doc_matters.opt.action.debug_do)) { - writeln(__FILE__, ":", __LINE__, ": ", obj.is_of); - } - break; + auto format_and_sqlite_load = SQLiteFormatAndLoadObject!()(doc_matters); + string[string] obj_txt; + string doc_text; + string[] _insert_doc_objects; + foreach (part; doc_matters.xml.keys_seq.sql) { + foreach (obj; doc_abstraction[part]) { + switch (obj.of_part) { + case "frontmatter": assert(part == "head", part); + switch (obj.is_of) { + case "para": + switch (obj.is_a) { + case "heading": + obj_txt = format_and_sqlite_load.heading(obj); + break; + default: + if ((doc_matters.opt.action.debug_do)) { + writeln(__FILE__, ":", __LINE__, ": ", obj.is_a); } break; - case "body": // assert(part == "body", part); // TODO broken - switch (obj.is_of) { - case "para": - switch (obj.is_a) { - case "heading": - debug (asserts) { // TODO consider and fix or remove - if (part != "body") { - writeln(__LINE__, ": ", obj.text); - } - } - obj_txt = format_and_sqlite_load.heading(obj); - break; - case "para": - obj_txt = format_and_sqlite_load.para(obj); - break; - default: - if ((doc_matters.opt.action.debug_do)) { - writeln(__FILE__, ":", __LINE__, ": ", obj.is_a); - } - break; - } - break; - case "block": - switch (obj.is_a) { - case "quote": - obj_txt = format_and_sqlite_load.quote(obj); - break; - case "group": - obj_txt = format_and_sqlite_load.group(obj); - break; - case "block": - obj_txt = format_and_sqlite_load.block(obj); - break; - case "poem": // double check on keeping both poem & verse - break; - case "verse": - obj_txt = format_and_sqlite_load.verse(obj); - break; - case "code": - obj_txt = format_and_sqlite_load.code(obj); - break; - case "table": - obj_txt = format_and_sqlite_load.table(obj); - break; - default: - if ((doc_matters.opt.action.debug_do)) { - writeln(__FILE__, ":", __LINE__, ": ", obj.is_a); - } - break; - } - break; - default: - if ((doc_matters.opt.action.debug_do)) { - writeln(__FILE__, ":", __LINE__, ": ", obj.is_of); + } + break; + default: + if ((doc_matters.opt.action.debug_do)) { + writeln(__FILE__, ":", __LINE__, ": ", obj.is_of); + } + break; + } + break; + case "body": // assert(part == "body", part); // TODO broken + switch (obj.is_of) { + case "para": + switch (obj.is_a) { + case "heading": + debug (asserts) { // TODO consider and fix or remove + if (part != "body") { + writeln(__LINE__, ": ", obj.text); } - break; } + obj_txt = format_and_sqlite_load.heading(obj); break; - case "backmatter": - assert(part == "endnotes" || "glossary" || "bibliography" || "bookindex_seg" || "blurb" || "tail", part); - switch (obj.is_of) { - case "para": - switch (obj.is_a) { - case "heading": - obj_txt = format_and_sqlite_load.heading(obj); - break; - case "endnote": assert(part == "endnotes", part); - obj_txt = format_and_sqlite_load.para(obj); - break; - case "glossary": assert(part == "glossary", part); - obj_txt = format_and_sqlite_load.para(obj); - break; - case "bibliography": assert(part == "bibliography", part); - obj_txt = format_and_sqlite_load.para(obj); - break; - case "bookindex": assert(part == "bookindex_seg", part); - obj_txt = format_and_sqlite_load.para(obj); - break; - case "blurb": assert(part == "blurb", part); - obj_txt = format_and_sqlite_load.para(obj); - break; - default: - if ((doc_matters.opt.action.debug_do)) { - writeln(__FILE__, ":", __LINE__, ": ", obj.is_a); - } - break; - } - break; - default: - if ((doc_matters.opt.action.debug_do)) { - writeln(__FILE__, ":", __LINE__, ": ", obj.is_of); - } - break; + case "para": + obj_txt = format_and_sqlite_load.para(obj); + break; + default: + if ((doc_matters.opt.action.debug_do)) { + writeln(__FILE__, ":", __LINE__, ": ", obj.is_a); } break; - case "comment": + } + break; + case "block": + switch (obj.is_a) { + case "quote": + obj_txt = format_and_sqlite_load.quote(obj); + break; + case "group": + obj_txt = format_and_sqlite_load.group(obj); + break; + case "block": + obj_txt = format_and_sqlite_load.block(obj); + break; + case "poem": // double check on keeping both poem & verse + break; + case "verse": + obj_txt = format_and_sqlite_load.verse(obj); + break; + case "code": + obj_txt = format_and_sqlite_load.code(obj); + break; + case "table": + obj_txt = format_and_sqlite_load.table(obj); break; default: if ((doc_matters.opt.action.debug_do)) { - writeln(__FILE__, ":", __LINE__, ": ", obj.of_part); // check where empty value could come from writeln(__FILE__, ":", __LINE__, ": ", obj.is_a); - writeln(__FILE__, ":", __LINE__, ": ", obj.text); // check where empty value could come from } break; } - if (obj.is_a == "heading") { - if ((doc_matters.opt.action.verbose)) { - writeln( - "markup: ", obj.heading_lev_markup, - "> ", obj.dom_markedup, - "; collapsed: ", obj.heading_lev_collapsed, - "> ", obj.dom_collapsed, - "; ocn: ", obj.ocn, - " node: ", obj.node, - "; parent: ", obj.parent_lev_markup, - "; ocn: ", obj.parent_ocn, - "; ", - ); + break; + default: + if ((doc_matters.opt.action.debug_do)) { + writeln(__FILE__, ":", __LINE__, ": ", obj.is_of); + } + break; + } + break; + case "backmatter": + assert(part == "endnotes" || "glossary" || "bibliography" || "bookindex_seg" || "blurb" || "tail", part); + switch (obj.is_of) { + case "para": + switch (obj.is_a) { + case "heading": + obj_txt = format_and_sqlite_load.heading(obj); + break; + case "endnote": assert(part == "endnotes", part); + obj_txt = format_and_sqlite_load.para(obj); + break; + case "glossary": assert(part == "glossary", part); + obj_txt = format_and_sqlite_load.para(obj); + break; + case "bibliography": assert(part == "bibliography", part); + obj_txt = format_and_sqlite_load.para(obj); + break; + case "bookindex": assert(part == "bookindex_seg", part); + obj_txt = format_and_sqlite_load.para(obj); + break; + case "blurb": assert(part == "blurb", part); + obj_txt = format_and_sqlite_load.para(obj); + break; + default: + if ((doc_matters.opt.action.debug_do)) { + writeln(__FILE__, ":", __LINE__, ": ", obj.is_a); } + break; } - if (!(obj.is_a == "comment")) { - _insert_doc_objects ~= insertDocObjectsRow(obj); + break; + default: + if ((doc_matters.opt.action.debug_do)) { + writeln(__FILE__, ":", __LINE__, ": ", obj.is_of); } - } // loop closes + break; + } + break; + case "comment": + break; + default: + if ((doc_matters.opt.action.debug_do)) { + writeln(__FILE__, ":", __LINE__, ": ", obj.of_part); // check where empty value could come from + writeln(__FILE__, ":", __LINE__, ": ", obj.is_a); + writeln(__FILE__, ":", __LINE__, ": ", obj.text); // check where empty value could come from + } + break; } - return _insert_doc_objects.join.to!(char[]).toUTF8; - } + if (obj.is_a == "heading") { + if ((doc_matters.opt.action.verbose)) { + writeln( + "markup: ", obj.heading_lev_markup, + "> ", obj.dom_markedup, + "; collapsed: ", obj.heading_lev_collapsed, + "> ", obj.dom_collapsed, + "; ocn: ", obj.ocn, + " node: ", obj.node, + "; parent: ", obj.parent_lev_markup, + "; ocn: ", obj.parent_ocn, + "; ", + ); + } + } + if (!(obj.is_a == "comment")) { + _insert_doc_objects ~= insertDocObjectsRow(obj); + } + } // loop closes } - SQLiteDbStatementComposite!()(db, doc_abstraction, doc_matters); + return _insert_doc_objects.join.to!(char[]).toUTF8; } } template SQLiteTablesCreate() { @@ -1066,7 +1103,8 @@ template SQLiteTablesCreate() { DROP TABLE IF EXISTS urls; CREATE TABLE metadata_and_text ( tid INTEGER PRIMARY KEY, - /* src_filename_composite VARCHAR(256) NOT NULL UNIQUE, /* z pod name if any + src filename + language code */ + src_composite_id_per_txt VARCHAR(256) NOT NULL UNIQUE, /* z pod name if any + src filename + language code */ + src_composite_id_per_pod VARCHAR(256) NOT NULL UNIQUE, /* z pod name if any + src filename + language code */ title VARCHAR(800) NOT NULL, title_main VARCHAR(400) NOT NULL, title_sub VARCHAR(400) NULL, diff --git a/src/sdp/output/sqlite_discrete.d b/src/sdp/output/sqlite_discrete.d deleted file mode 100644 index 40e0b15..0000000 --- a/src/sdp/output/sqlite_discrete.d +++ /dev/null @@ -1,1023 +0,0 @@ -module sdp.output.sqlite_discrete; -import sdp.output; -import - std.file, - std.conv : to; -template SQLiteDiscreteDbRun() { - void SQLiteDiscreteDbRun(Db,St,O)( - auto ref Db db, - auto ref St db_statement, - auto ref O opt_action, - string note, - ) { - debug(sql_statement) { - writeln(db_statement); - } - try { - db.run( - "\nBEGIN;\n" ~ - db_statement ~ - "\nCOMMIT;\n" - ); - } catch (ErrnoException ex) { - writeln("ERROR SQLite : ", ex); - } catch (Exception ex) { - writeln("ERROR SQLite : ", ex); - } - if (opt_action.debug_do) { - writeln(note); - if (opt_action.verbose) { - writeln(db_statement); - } - } - } -} -template SQLiteDiscreteBuildTablesAndPopulate() { - void SQLiteDiscreteBuildTablesAndPopulate(D,I)( - auto ref const D doc_abstraction, - auto ref I doc_matters, - ) { - import d2sqlite3; - import std.typecons : Nullable; - mixin SiSUoutputRgxInit; - static auto rgx = Rgx(); - auto pth_sqlite = SiSUpathsSQLiteDiscrete!()(doc_matters.output_path, doc_matters.src.language); - pth_sqlite.base.mkdirRecurse; - auto db = Database(pth_sqlite.sqlite_file(doc_matters.src.filename)); - template SQLiteDiscreteDbStatementComposite() { - void SQLiteDiscreteDbStatementComposite(Db,D,I)( - auto ref Db db, - auto ref const D doc_abstraction, - auto ref I doc_matters, - ) { - { - string _db_statement; - { - _db_statement ~= SQLiteDiscreteTablesReCreate!()(); - _db_statement ~= SQLiteDiscreteInsertMetadata!()(doc_matters); - _db_statement ~= SQLiteDiscreteInsertDocObjectsLoop!()(doc_matters.xml.keys_seq.sql, 1); // FIX - SQLiteDiscreteDbRun!()(db, _db_statement, doc_matters.opt.action, "table CREATE Tables, INSERT DocObjects"); - } - db.close; - } - } - } - template SQLinsertDelimiter() { - auto SQLinsertDelimiter(string _txt) { - _txt = _txt - .replaceAll(rgx.quotation_mark_sql_insert_delimiter, "$0$0"); - return _txt; - } - } - template SQLiteFormatAndLoadObject() { - auto SQLiteFormatAndLoadObject(I)( - auto ref I doc_matters, - ) { - mixin SiSUoutputRgxInit; - struct sqlite_format_and_load_objects { - auto generic_munge_sanitize_text_for_search( - string _txt, - ) { - string _notes; - string _urls; - if (_txt.matchFirst(rgx.inline_notes_al_gen)) { - foreach (m; _txt.matchAll(rgx.inline_notes_al_gen_text)) { - _notes ~= "\n" ~ m["text"]; - } - _txt = _txt.replaceAll(rgx.inline_notes_al_gen, ""); - } - if (_txt.matchFirst(rgx.inline_link)) { - foreach (m; _txt.matchAll(rgx.inline_link)) { - if (m["link"].match(rgx.url)) { - _urls ~= "\n" ~ m["link"]; - } - } - _txt = _txt.replaceAll(rgx.inline_link_clean, ""); - } - if (_notes.length > 0) { - _txt ~= _notes; - } - if (_urls.length > 0) { - _txt ~= _urls; - } - if (doc_matters.opt.action.debug_do) { - writeln(_txt, "\n"); - } - debug(sql_text_clean) { - writeln(_txt); - } - return _txt; - } - auto munge_html(O)( - auto return ref const O obj, - ) { - string _html_special_characters(string _txt){ - _txt = _txt - .replaceAll(rgx.xhtml_ampersand, "&") - .replaceAll(rgx.xhtml_quotation, """) - .replaceAll(rgx.xhtml_less_than, "<") - .replaceAll(rgx.xhtml_greater_than, ">") - .replaceAll(rgx.nbsp_char, " ") - .replaceAll(rgx.xhtml_line_break, "
"); - return _txt; - } - string _html_font_face(string _txt){ - _txt = _txt - .replaceAll(rgx.inline_emphasis, "$1") - .replaceAll(rgx.inline_bold, "$1") - .replaceAll(rgx.inline_underscore, "$1") - .replaceAll(rgx.inline_italics, "$1") - .replaceAll(rgx.inline_superscript, "$1") - .replaceAll(rgx.inline_subscript, "$1") - .replaceAll(rgx.inline_strike, "$1") - .replaceAll(rgx.inline_insert, "$1") - .replaceAll(rgx.inline_mono, "$1") - .replaceAll(rgx.inline_cite, "$1"); - return _txt; - } - string _notes; - string _urls; - string _txt = _html_font_face(_html_special_characters(obj.text)); - if (_txt.matchFirst(rgx.inline_notes_al_gen)) { - foreach (m; _txt.matchAll(rgx.inline_notes_al_gen_text)) { - _notes ~= "\n" ~ m["text"]; - } - _txt = _txt.replaceAll(rgx.inline_notes_al_gen_ref, "$1 "); - } - if (_txt.matchFirst(rgx.inline_link)) { - foreach (m; _txt.matchAll(rgx.inline_link)) { - } - _txt = _txt.replaceAll(rgx.inline_link_clean, ""); - } - if (_notes.length > 0) { - _txt ~= _notes; - } - if (doc_matters.opt.action.debug_do) { - writeln(_txt, "\n"); - } - return _txt; - } - string html_special_characters(string _txt){ - _txt = _txt - .replaceAll(rgx.xhtml_ampersand, "&") - .replaceAll(rgx.xhtml_quotation, """) - .replaceAll(rgx.xhtml_less_than, "<") - .replaceAll(rgx.xhtml_greater_than, ">") - .replaceAll(rgx.nbsp_char, " ") - .replaceAll(rgx.xhtml_line_break, "
"); - return _txt; - } - string html_special_characters_code(string _txt){ - _txt = _txt - .replaceAll(rgx.xhtml_ampersand, "&") - .replaceAll(rgx.xhtml_quotation, """) - .replaceAll(rgx.xhtml_less_than, "<") - .replaceAll(rgx.xhtml_greater_than, ">") - .replaceAll(rgx.nbsp_char, " "); - return _txt; - } - string html_font_face(string _txt){ - _txt = _txt - .replaceAll(rgx.inline_emphasis, "$1") - .replaceAll(rgx.inline_bold, "$1") - .replaceAll(rgx.inline_underscore, "$1") - .replaceAll(rgx.inline_italics, "$1") - .replaceAll(rgx.inline_superscript, "$1") - .replaceAll(rgx.inline_subscript, "$1") - .replaceAll(rgx.inline_strike, "$1") - .replaceAll(rgx.inline_insert, "$1") - .replaceAll(rgx.inline_mono, "$1") - .replaceAll(rgx.inline_cite, "$1"); - return _txt; - } - auto html_heading(O)( - auto return ref const O obj, - ) { - string _txt = munge_html(obj); - string o = format(q"¶

- %s -

¶", - obj.is_a, - _txt, - ); - return o; - } - auto html_para(O)( - auto return ref const O obj, - ) { - string _txt = munge_html(obj); - _txt = (obj.bullet) ? ("●  " ~ _txt) : _txt; - string o = format(q"¶

- %s -

¶", - obj.is_a, - obj.indent_hang, - obj.indent_base, - _txt - ); - return o; - } - auto html_quote(O)( - auto return ref const O obj, - ) { - string _txt = munge_html(obj); - string o = format(q"¶

- %s -

¶", - obj.is_a, - _txt - ); - return o; - } - auto html_group(O)( - auto return ref const O obj, - ) { - string _txt = munge_html(obj); - string o = format(q"¶

- %s -

¶", - obj.is_a, - _txt - ); - return o; - } - auto html_block(O)( - auto return ref const O obj, - ) { - string _txt = munge_html(obj); - string o = format(q"¶ -

%s

¶", - obj.is_a, - _txt.stripRight - ); - return o; - } - auto html_verse(O)( - auto return ref const O obj, - ) { - string _txt = munge_html(obj); - string o = format(q"¶

%s

¶", - obj.is_a, - _txt - ); - return o; - } - auto html_code(O)( - auto return ref const O obj, - ) { - string _txt = html_special_characters_code(obj.text); - string o = format(q"¶

%s

¶", - obj.is_a, - _txt - ); - return o; - } - auto html_table(O)( - auto return ref const O obj, - ) { - auto _tablarize(O)( - auto return ref const O obj, - string _txt, - ) { - string[] _table_rows = _txt.split(rgx.table_delimiter_row); - string[] _table_cols; - string _table; - string _tablenote; - foreach(row_idx, row; _table_rows) { - _table_cols = row.split(rgx.table_delimiter_col); - _table ~= ""; - foreach(col_idx, cell; _table_cols) { - if ((_table_cols.length == 1) - && (_table_rows.length <= row_idx+2)) { // check row_idx+2 (rather than == ++row_idx) - _tablenote ~= cell; - } else { - string _col_is = (row_idx == 0 && obj.table_heading) ? "th" : "td"; - string _align = ("style=\"text-align:" - ~ ((obj.table_column_aligns[col_idx] == "l") - ? "left\"" : "right\"")); - _table ~= "<" - ~ _col_is - ~ " width=\"" - ~ obj.table_column_widths[col_idx].to!string - ~ "%\" " - ~ _align - ~ ">"; - _table ~= cell; - _table ~= ""; - } - } - _table ~= ""; - } - auto t = tuple( - _table, - _tablenote, - ); - return t; - } - string _txt = munge_html(obj); - auto t = _tablarize(obj, _txt); - _txt = t[0]; - string _note = t[1]; - string o = format(q"¶

- - %s -
- %s -

¶", - obj.is_a, - _txt, - _note - ); - return o; - } - auto sqlite_load_string(O,Dm)( - auto return ref const O obj, - auto return ref Dm doc_matters, - ) { - string o; - return o; - } - auto postgresql_load_string(O,Dm)( - auto return ref const O obj, - auto return ref Dm doc_matters, - ) { - string o; - return o; - } - string sqlite_statement(O)( - auto return ref const O obj, - string _txt, - string _html, - ) { - void _sql_exe(O)( - string _sql, - ) { - writeln(_html); - writeln(_sql); - } - string _sql; - return _sql; - } - auto heading(O)( - auto return ref const O obj, - ) { - string[string] obj_txt = [ - "text": generic_munge_sanitize_text_for_search(obj.text), - "html": html_heading(obj) - ]; - if (doc_matters.opt.action.debug_do) { - debug(sql_txt) { - writeln(obj_txt["text"]); - } - debug(sql_html) { - writeln(obj_txt["html"]); - } - } else { - // load sql - } - return obj_txt; - } - auto para(O)( - auto return ref const O obj, - ) { - string[string] obj_txt = [ - "text": generic_munge_sanitize_text_for_search(obj.text), - "html": html_para(obj) - ]; - if (doc_matters.opt.action.debug_do) { - debug(sql_txt) { - writeln(obj_txt["text"]); - } - debug(sql_html) { - writeln(obj_txt["html"]); - } - } else { - // load sql - } - return obj_txt; - } - auto quote(O)( - auto return ref const O obj, - ) { - string[string] obj_txt = [ - "text": generic_munge_sanitize_text_for_search(obj.text), - "html": html_quote(obj) - ]; - if (doc_matters.opt.action.debug_do) { - debug(sql_txt) { - writeln(obj_txt["text"]); - } - debug(sql_html) { - writeln(obj_txt["html"]); - } - } else { - // load sql - } - return obj_txt; - } - auto group(O)( - auto return ref const O obj, - ) { - string[string] obj_txt = [ - "text": generic_munge_sanitize_text_for_search(obj.text), - "html": html_group(obj) - ]; - if (doc_matters.opt.action.debug_do) { - debug(sql_txt) { - writeln(obj_txt["text"]); - } - debug(sql_html) { - writeln(obj_txt["html"]); - } - } else { - // load sql - } - return obj_txt; - } - auto block(O)( - auto return ref const O obj, - ) { - string[string] obj_txt = [ - "text": generic_munge_sanitize_text_for_search(obj.text), - "html": html_block(obj) - ]; - if (doc_matters.opt.action.debug_do) { - debug(sql_txt) { - writeln(obj_txt["text"]); - } - debug(sql_html) { - writeln(obj_txt["html"]); - } - } else { - // load sql - } - return obj_txt; - } - auto verse(O)( - auto return ref const O obj, - ) { - string[string] obj_txt = [ - "text": generic_munge_sanitize_text_for_search(obj.text), - "html": html_verse(obj) - ]; - if (doc_matters.opt.action.debug_do) { - debug(sql_txt) { - writeln(obj_txt["text"]); - } - debug(sql_html) { - writeln(obj_txt["html"]); - } - } else { - // load sql - } - return obj_txt; - } - auto code(O)( - auto return ref const O obj, - ) { - string[string] obj_txt = [ - "text": generic_munge_sanitize_text_for_search(obj.text), - "html": html_code(obj) - ]; - if (doc_matters.opt.action.debug_do) { - debug(sql_txt) { - writeln(obj_txt["text"]); - } - debug(sql_html) { - writeln(obj_txt["html"]); - } - } else { - // load sql - } - return obj_txt; - } - auto table(O)( - auto return ref const O obj, - ) { - string[string] obj_txt = [ - "text": generic_munge_sanitize_text_for_search(obj.text), - "html": html_table(obj) - ]; - if (doc_matters.opt.action.debug_do) { - debug(sql_txt) { - writeln(obj_txt["text"]); - } - debug(sql_html) { - writeln(obj_txt["html"]); - } - } else { - // load sql - } - return obj_txt; - } - } - return sqlite_format_and_load_objects(); - } - } - template SQLiteDiscreteTablesReCreate() { - string SQLiteDiscreteTablesReCreate()() { - string _sql_instruct; - _sql_instruct = format(q"¶ - DROP INDEX IF EXISTS ocn; - DROP INDEX IF EXISTS digest_clean; - DROP INDEX IF EXISTS digest_all; - DROP INDEX IF EXISTS clean; - DROP INDEX IF EXISTS lev0; - DROP INDEX IF EXISTS lev1; - DROP INDEX IF EXISTS lev2; - DROP INDEX IF EXISTS lev3; - DROP INDEX IF EXISTS lev4; - DROP INDEX IF EXISTS lev5; - DROP INDEX IF EXISTS lev6; - DROP INDEX IF EXISTS title; - DROP INDEX IF EXISTS creator_author; - DROP INDEX IF EXISTS src_filename; - DROP INDEX IF EXISTS language_document_char; - DROP INDEX IF EXISTS classify_topic_register; - DROP TABLE IF EXISTS metadata_and_text; - DROP TABLE IF EXISTS doc_objects; - DROP TABLE IF EXISTS urls; - CREATE TABLE metadata_and_text ( - tid INTEGER PRIMARY KEY, - /* src_filename_composite VARCHAR(256) NOT NULL UNIQUE, /* z pod name if any + src filename + language code */ - title VARCHAR(800) NOT NULL, - title_main VARCHAR(400) NOT NULL, - title_sub VARCHAR(400) NULL, - title_short VARCHAR(400) NULL, - title_edition VARCHAR(10) NULL, - title_note VARCHAR(2500) NULL, - title_language VARCHAR(100) NULL, - title_language_char VARCHAR(6) NULL, - creator_author VARCHAR(600) NOT NULL, - creator_author_email VARCHAR(100) NULL, - creator_author_hon VARCHAR(100) NULL, - creator_author_nationality VARCHAR(100) NULL, - creator_editor VARCHAR(600) NULL, - creator_contributor VARCHAR(600) NULL, - creator_illustrator VARCHAR(600) NULL, - creator_photographer VARCHAR(600) NULL, - creator_translator VARCHAR(600) NULL, - creator_prepared_by VARCHAR(600) NULL, - creator_digitized_by VARCHAR(600) NULL, - creator_audio VARCHAR(600) NULL, - creator_video VARCHAR(600) NULL, - language_document VARCHAR(100) NULL, - language_document_char VARCHAR(6) NOT NULL, - language_original VARCHAR(100) NULL, - language_original_char VARCHAR(6) NULL, - date_added_to_site VARCHAR(10) NULL, - date_available VARCHAR(10) NULL, - date_created VARCHAR(10) NULL, - date_issued VARCHAR(10) NULL, - date_modified VARCHAR(10) NULL, - date_published VARCHAR(10) NULL, - date_valid VARCHAR(10) NULL, - date_translated VARCHAR(10) NULL, - date_original_publication VARCHAR(10) NULL, - date_generated VARCHAR(10) NULL, - publisher VARCHAR(600) NULL, - original_publisher VARCHAR(600) NULL, - original_language VARCHAR(100) NULL, - original_language_char VARCHAR(6) NULL, - original_source VARCHAR(600) NULL, - original_institution VARCHAR(600) NULL, - original_nationality VARCHAR(100) NULL, - original_title VARCHAR(800) NULL, - rights_copyright VARCHAR(2500) NULL, - rights_copyright_audio VARCHAR(2500) NULL, - rights_copyright_cover VARCHAR(2500) NULL, - rights_copyright_illustrations VARCHAR(2500) NULL, - rights_copyright_photographs VARCHAR(2500) NULL, - rights_copyright_text VARCHAR(2500) NULL, - rights_copyright_translation VARCHAR(2500) NULL, - rights_copyright_video VARCHAR(2500) NULL, - rights_license VARCHAR(2500) NULL, - identifier_oclc VARCHAR(30) NULL, - identifier_isbn VARCHAR(16) NULL, - classify_topic_register VARCHAR(2500) NULL, - classify_subject VARCHAR(600) NULL, - classify_loc VARCHAR(30) NULL, - classify_dewey VARCHAR(30) NULL, - classify_keywords VARCHAR(600) NULL, - notes_abstract TEXT NULL, - notes_description TEXT NULL, - notes_comment TEXT NULL, - notes_coverage VARCHAR(200) NULL, - notes_relation VARCHAR(200) NULL, - notes_history VARCHAR(600) NULL, - notes_type VARCHAR(600) NULL, - notes_format VARCHAR(600) NULL, - notes_prefix TEXT NULL, - notes_prefix_a TEXT NULL, - notes_prefix_b TEXT NULL, - notes_suffix TEXT NULL, - src_filename VARCHAR(256) NOT NULL, - src_fingerprint VARCHAR(256) NULL, - src_filesize VARCHAR(10) NULL, - src_wordcount VARCHAR(10) NULL, - pod_name VARCHAR(256) NULL, /* zipped pod, work to be done here */ - pod_fingerprint VARCHAR(256) NULL, /* zipped pod, work to be done here */ - pod_size VARCHAR(10) NULL, /* zipped pod, work to be done here */ - src_text TEXT NULL, - fulltext TEXT NULL, - links TEXT NULL - ); - CREATE TABLE doc_objects ( - lid INTEGER PRIMARY KEY, - metadata_tid INTEGER REFERENCES metadata_and_text, - ocn SMALLINT, - ocnd VARCHAR(6), - ocns VARCHAR(6), - clean TEXT NULL, - body TEXT NULL, - book_idx TEXT NULL, - seg VARCHAR(256) NULL, - lev_an VARCHAR(1), - lev SMALLINT NULL, - lev0 SMALLINT, - lev1 SMALLINT, - lev2 SMALLINT, - lev3 SMALLINT, - lev4 SMALLINT, - lev5 SMALLINT, - lev6 SMALLINT, - lev7 SMALLINT, - en_a SMALLINT NULL, - en_z SMALLINT NULL, - en_a_asterisk SMALLINT NULL, - en_z_asterisk SMALLINT NULL, - en_a_plus SMALLINT NULL, - en_z_plus SMALLINT NULL, - t_of VARCHAR(16), - t_is VARCHAR(16), - node VARCHAR(16) NULL, - parent VARCHAR(16) NULL, - digest_clean CHAR(256), - digest_all CHAR(256), - types CHAR(1) NULL - ); - CREATE INDEX idx_ocn ON doc_objects(ocn); - CREATE INDEX idx_digest_clean ON doc_objects(digest_clean); - CREATE INDEX idx_digest_all ON doc_objects(digest_all); - CREATE INDEX idx_clean ON doc_objects(clean); - CREATE INDEX idx_lev0 ON doc_objects(lev0); - CREATE INDEX idx_lev1 ON doc_objects(lev1); - CREATE INDEX idx_lev2 ON doc_objects(lev2); - CREATE INDEX idx_lev3 ON doc_objects(lev3); - CREATE INDEX idx_lev4 ON doc_objects(lev4); - CREATE INDEX idx_lev5 ON doc_objects(lev5); - CREATE INDEX idx_lev6 ON doc_objects(lev6); - CREATE INDEX idx_title ON metadata_and_text(title); - CREATE INDEX idx_author ON metadata_and_text(creator_author); - CREATE INDEX idx_filename ON metadata_and_text(src_filename); - CREATE INDEX idx_language ON metadata_and_text(language_document_char); - CREATE INDEX idx_topics ON metadata_and_text(classify_topic_register); - ¶",); - return _sql_instruct; - } - } - template SQLiteDiscreteInsertMetadata() { - string SQLiteDiscreteInsertMetadata(I)( - auto ref I doc_matters, - ) { - string _insert_metadata; - _insert_metadata = format(q"¶ - INSERT INTO metadata_and_text ( - src_filename, - title, - title_main, - title_sub, - title_short, - title_edition, - title_language, - classify_dewey, - classify_keywords, - classify_loc, - classify_subject, - classify_topic_register, - creator_author, - creator_author_email, - creator_illustrator, - creator_translator, - date_added_to_site, - date_available, - date_created, - date_issued, - date_modified, - date_published, - date_valid, - identifier_isbn, - identifier_oclc, - language_document, - language_document_char, - notes_abstract, - notes_description, - original_publisher, - original_language, - original_language_char, - original_source, - original_title, - publisher, - rights_copyright, - rights_copyright_audio, - rights_copyright_cover, - rights_copyright_illustrations, - rights_copyright_photographs, - rights_copyright_text, - rights_copyright_translation, - rights_copyright_video, - rights_license - ) - VALUES ( - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s', - '%s' - ); - ¶", - SQLinsertDelimiter!()(doc_matters.src.filename), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_full), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_main), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_subtitle), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_short), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_edition), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_language), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_dewey), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_keywords), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_loc), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_subject), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_topic_register), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_author), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_author_email), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_illustrator), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_translator), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_added_to_site), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_available), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_created), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_issued), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_modified), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_published), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_valid), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.identifier_isbn), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.identifier_oclc), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.language_document), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.language_document_char), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.notes_abstract), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.notes_description), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_publisher), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_language), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_language_char), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_source), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_title), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.publisher), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_audio), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_cover), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_illustrations), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_photographs), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_text), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_translation), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_video), - SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_license) - ); - return _insert_metadata; - } - } - template SQLiteDiscreteInsertDocObjectsLoop() { - string SQLiteDiscreteInsertDocObjectsLoop(P,I)( - auto ref P doc_parts, - auto ref I tid, - ) { - string insertDocObjectsRow(O)(O obj) { - string _insert_doc_objects_row; - _insert_doc_objects_row = format(q"¶ - INSERT INTO doc_objects ( - metadata_tid, - ocn, - clean, - body, - lev, - t_of, - t_is - ) - VALUES ( - %d, - %s, - '%s', - '%s', - %s, - '%s', - '%s' - ); - ¶", - 1, - obj.ocn, - SQLinsertDelimiter!()(obj_txt["text"]), - SQLinsertDelimiter!()(obj_txt["html"]), - obj.heading_lev_markup, - obj.is_of, - obj.is_a, - ); - return _insert_doc_objects_row; - } - auto format_and_sqlite_load = SQLiteFormatAndLoadObject!()(doc_matters); - string[string] obj_txt; - string doc_text; - string[] _insert_doc_objects; - foreach (part; doc_parts) { - foreach (obj; doc_abstraction[part]) { - switch (obj.of_part) { - case "frontmatter": assert(part == "head", part); - switch (obj.is_of) { - case "para": - switch (obj.is_a) { - case "heading": - obj_txt = format_and_sqlite_load.heading(obj); - break; - default: - if ((doc_matters.opt.action.debug_do)) { - writeln(__FILE__, ":", __LINE__, ": ", obj.is_a); - } - break; - } - break; - default: - if ((doc_matters.opt.action.debug_do)) { - writeln(__FILE__, ":", __LINE__, ": ", obj.is_of); - } - break; - } - break; - case "body": // assert(part == "body", part); // TODO broken - switch (obj.is_of) { - case "para": - switch (obj.is_a) { - case "heading": - debug (asserts) { // TODO consider and fix or remove - if (part != "body") { - writeln(__LINE__, ": ", obj.text); - } - } - obj_txt = format_and_sqlite_load.heading(obj); - break; - case "para": - obj_txt = format_and_sqlite_load.para(obj); - break; - default: - if ((doc_matters.opt.action.debug_do)) { - writeln(__FILE__, ":", __LINE__, ": ", obj.is_a); - } - break; - } - break; - case "block": - switch (obj.is_a) { - case "quote": - obj_txt = format_and_sqlite_load.quote(obj); - break; - case "group": - obj_txt = format_and_sqlite_load.group(obj); - break; - case "block": - obj_txt = format_and_sqlite_load.block(obj); - break; - case "poem": // double check on keeping both poem & verse - break; - case "verse": - obj_txt = format_and_sqlite_load.verse(obj); - break; - case "code": - obj_txt = format_and_sqlite_load.code(obj); - break; - case "table": - obj_txt = format_and_sqlite_load.table(obj); - break; - default: - if ((doc_matters.opt.action.debug_do)) { - writeln(__FILE__, ":", __LINE__, ": ", obj.is_a); - } - break; - } - break; - default: - if ((doc_matters.opt.action.debug_do)) { - writeln(__FILE__, ":", __LINE__, ": ", obj.is_of); - } - break; - } - break; - case "backmatter": - assert(part == "endnotes" || "glossary" || "bibliography" || "bookindex_seg" || "blurb" || "tail", part); - switch (obj.is_of) { - case "para": - switch (obj.is_a) { - case "heading": - obj_txt = format_and_sqlite_load.heading(obj); - break; - case "endnote": assert(part == "endnotes", part); - obj_txt = format_and_sqlite_load.para(obj); - break; - case "glossary": assert(part == "glossary", part); - obj_txt = format_and_sqlite_load.para(obj); - break; - case "bibliography": assert(part == "bibliography", part); - obj_txt = format_and_sqlite_load.para(obj); - break; - case "bookindex": assert(part == "bookindex_seg", part); - obj_txt = format_and_sqlite_load.para(obj); - break; - case "blurb": assert(part == "blurb", part); - obj_txt = format_and_sqlite_load.para(obj); - break; - default: - if ((doc_matters.opt.action.debug_do)) { - writeln(__FILE__, ":", __LINE__, ": ", obj.is_a); - } - break; - } - break; - default: - if ((doc_matters.opt.action.debug_do)) { - writeln(__FILE__, ":", __LINE__, ": ", obj.is_of); - } - break; - } - break; - case "comment": - break; - default: - if ((doc_matters.opt.action.debug_do)) { - writeln(__FILE__, ":", __LINE__, ": ", obj.of_part); // check where empty value could come from - writeln(__FILE__, ":", __LINE__, ": ", obj.is_a); - writeln(__FILE__, ":", __LINE__, ": ", obj.text); // check where empty value could come from - } - break; - } - if (obj.is_a == "heading") { - if ((doc_matters.opt.action.verbose)) { - writeln( - "markup: ", obj.heading_lev_markup, - "> ", obj.dom_markedup, - "; collapsed: ", obj.heading_lev_collapsed, - "> ", obj.dom_collapsed, - "; ocn: ", obj.ocn, - " node: ", obj.node, - "; parent: ", obj.parent_lev_markup, - "; ocn: ", obj.parent_ocn, - "; ", - ); - } - } - if (!(obj.is_a == "comment")) { - _insert_doc_objects ~= insertDocObjectsRow(obj); - } - } // loop closes - } - return _insert_doc_objects.join.to!(char[]).toUTF8; - } - } - SQLiteDiscreteDbStatementComposite!()(db, doc_abstraction, doc_matters); - } -} diff --git a/views/version.txt b/views/version.txt index b3b647a..e143575 100644 --- a/views/version.txt +++ b/views/version.txt @@ -4,7 +4,7 @@ struct Version { int minor; int patch; } -enum ver = Version(0, 26, 0); +enum ver = Version(0, 26, 1); version (Posix) { version (DigitalMars) { } else version (LDC) { -- cgit v1.2.3