diff options
41 files changed, 3760 insertions, 2243 deletions
diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..8a834da --- /dev/null +++ b/.editorconfig @@ -0,0 +1,28 @@ +# EditorConfig : https://editorconfig.org + +# top-most EditorConfig file +root = true + +# Unix-style newlines with a newline ending every file +[*] +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true + +# Matches multiple files with brace expansion notation +# Set default charset +[*.{org,d,nix}] +charset = utf-8 + +# Indentation override for all D under src directory +[src/sisudoc/**.d] +indent_style = space +indent_size = 2 + +# Tab indentation (no size specified) +[Makefile] +indent_style = tab + +[{dub.selections.json,dub_describe.json}] +indent_style = space +indent_size = 2 @@ -1,8 +1,11 @@ +# -*- mode: sh -*- if [ -f .envrc-git-init ]; then source_env_if_exists .envrc-git-init || source .envrc-git-init fi if [ -f .envrc-local ]; then source_env_if_exists .envrc-local || source .envrc-local +elif [ -f .envrc-local_ ]; then + source_env_if_exists .envrc-local_ || source .envrc-local_ fi if [ -f .envrc-nix ]; then source_env_if_exists .envrc-nix || source .envrc-nix diff --git a/.envrc-git-init b/.envrc-git-init index ca74ec3..d78e46b 100644 --- a/.envrc-git-init +++ b/.envrc-git-init @@ -1,3 +1,4 @@ +# -*- mode: sh -*- if [[ ! -d ./.git ]]; then git init git add . diff --git a/.envrc-local b/.envrc-local index 392dd6e..1bafe75 100644 --- a/.envrc-local +++ b/.envrc-local @@ -1,5 +1,5 @@ export SpineVER=$(git describe --long --tags | sed 's/^[ a-z_-]\+\([0-9.]\+\)/\1/;s/\([^-]*-g\)/r\1/;s/-/./g') -export SpineSearchBIN=result/share/cgi-bin +export SpineSearchBIN=result/cgi-bin # ❯❯ cgi search form name: export SpineCGIform='spine_search' # ❯❯ configuration cgi search form path: diff --git a/.envrc-local_ b/.envrc-local_ index 392dd6e..a2671b4 100644 --- a/.envrc-local_ +++ b/.envrc-local_ @@ -1,6 +1,11 @@ +# -*- mode: sh -*- export SpineVER=$(git describe --long --tags | sed 's/^[ a-z_-]\+\([0-9.]\+\)/\1/;s/\([^-]*-g\)/r\1/;s/-/./g') -export SpineSearchBIN=result/share/cgi-bin +export SpineSearchBIN=result/cgi-bin # ❯❯ cgi search form name: export SpineCGIform='spine_search' +# ❯❯ search form db name: +export SpineSQLdb='spine.search.db' # ❯❯ configuration cgi search form path: export SpineCGIbin=/var/www/cgi/cgi-bin +# ❯❯ configuration db path: +export SpineDBpath=/var/www/sqlite @@ -1,10 +1,11 @@ +# -*- mode: nix -*- NIX_ENFORCE_PURITY=1 -# - https://github.com/nix-community/nix-direnv -NixDirEnvVersion="3.0.7" -NixDirEnvSHA="sha256-bn8WANE5a91RusFmRI7kS751ApelG02nMcwRekC/qzc=" -if ! has nix_direnv_version || ! nix_direnv_version ${NixDirEnvVersion}; then - source_url "https://raw.githubusercontent.com/nix-community/nix-direnv/${NixDirEnvVersion}/direnvrc" "${NixDirEnvSHA}" -fi +## - https://github.com/nix-community/nix-direnv +#NixDirEnvVersion="3.1.0" +#NixDirEnvSHA="sha256-yMJ2OVMzrFaDPn7q8nCBZFRYpL/f0RcHzhmw/i6btJM=" +#if ! has nix_direnv_version || ! nix_direnv_version ${NixDirEnvVersion}; then +# source_url "https://raw.githubusercontent.com/nix-community/nix-direnv/${NixDirEnvVersion}/direnvrc" "${NixDirEnvSHA}" +#fi watch_file flake.lock watch_file flake.nix watch_file shell.nix @@ -24,11 +25,11 @@ echo ' • nix develop ❯❯ nix develop ".#" --print-build-logs -c zsh • nix develop using nixpkgs - ❯❯ nix develop ".#dsh-nixpkgs-dmd-dub" --print-build-logs -c zsh - ❯❯ nix develop ".#dsh-nixpkgs-ldc-dub" --print-build-logs -c zsh - • nix develop using package overlays: (dmd-2.111.0, ldc-1.41.0, dub-1.39.0) - ❯❯ nix develop ".#dsh-overlay-dmd-dub" --print-build-logs -c zsh - ❯❯ nix develop ".#dsh-overlay-ldc-dub" --print-build-logs -c zsh + ❯❯ nix develop ".#dsh-nixpkgs-dmd" --print-build-logs -c zsh + ❯❯ nix develop ".#dsh-nixpkgs-ldc" --print-build-logs -c zsh + • nix develop using package overlays: (dmd-2.111.0, ldc-1.42.0, dub-1.40.0) + ❯❯ nix develop ".#dsh-overlay-dmd" --print-build-logs -c zsh + ❯❯ nix develop ".#dsh-overlay-ldc" --print-build-logs -c zsh ❯❯ nix flake show • to build project: @@ -40,20 +41,21 @@ echo ' • nix build using nixpkgs ❯❯ nix build ".#spine-search-nixpkgs-dmd" --print-build-logs ❯❯ nix build ".#spine-search-nixpkgs-ldc" --print-build-logs - • nix build using package overlays: (dmd-2.111.0, ldc-1.41.0, dub-1.39.0) + • nix build using package overlays: (dmd-2.111.0, ldc-1.42.0, dub-1.40.0) ❯❯ nix build ".#spine-search-overlay-dmd" --print-build-logs ❯❯ nix build ".#spine-search-overlay-ldc" --print-build-logs • to build using dub on nix (get dependencies by setting your development environment): - ❯❯ nix develop ".#dsh-nixpkgs-dmd-dub" --print-build-logs -c zsh + ❯❯ nix develop ".#dsh-nixpkgs-dmd" --print-build-logs -c zsh ❯❯ dub run --compiler=dmd --config=dmd --combined --skip-registry=all - ❯❯ nix develop ".#dsh-nixpkgs-ldc-dub" --print-build-logs -c zsh + ❯❯ nix develop ".#dsh-nixpkgs-ldc" --print-build-logs -c zsh ❯❯ dub run --compiler=ldmd2 --config=ldmd2 --combined --skip-registry=all • for develop environment & build options - ❯❯ nix flake show + ❯❯ nix flake update && nix flake check && nix flake show + ❯❯ nix flake show ' @@ -1,4 +1,5 @@ # git ls-files --others --exclude-from=.git/info/exclude +# git ls-files --others --ignored --exclude-standard --directory # git check-ignore -v flake.lock # git clean -ix # find . -name "*_" | xargs rm -rf @@ -6,16 +7,14 @@ * !.gitignore !.gitattributes +!.editorconfig !nix/** !*.nix !flake.lock !nixDevEnv.sh !.envrc -!.envrc-local !.envrc-nix !.envrc-git-init -!.env -!.env/** !nix-overlays !nix-overlays/** !README.md @@ -32,6 +31,8 @@ !*.org !org !org/*.org +!*.md +!*.txt !*.d !*.rb !*.txt @@ -50,16 +51,17 @@ tmp/** *_.org *_.nix *_.d +*_.md *_.txt *_ -*.swp -*~ -\#* -*.\#* !.envrc-local_ !src/sisudoc/spine_search.d_ !views/configuration.txt_remotehost_ !views/configuration.txt_localhost_ +*.swp +*~ +\#* +*.\#* #!*/ #\#* #*.\#* @@ -5,7 +5,7 @@ - Author: Ralph Amissah [ralph.amissah@gmail.com] - - Copyright: (C) 2015 - 2025 Ralph Amissah + - Copyright: (C) 2015 - 2026 Ralph Amissah - code under src/* src/sisudoc/* - License: AGPL 3 or later: @@ -4,14 +4,14 @@ #+FILETAGS: :spine:build:tools: #+AUTHOR: Ralph Amissah #+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] -#+COPYRIGHT: Copyright (C) 2015 - 2025 Ralph Amissah +#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah #+LANGUAGE: en #+STARTUP: content hideblocks hidestars noindent entitiespretty #+OPTIONS: H:3 num:nil toc:t \n:nil @:t ::t |:t ^:nil _:nil -:t f:t *:t <:t #+PROPERTY: header-args :exports code #+PROPERTY: header-args+ :noweb yes #+PROPERTY: header-args+ :eval no -#+PROPERTY: header-args+ :results no +#+PROPERTY: header-args+ :results silent #+PROPERTY: header-args+ :cache no #+PROPERTY: header-args+ :padline no @@ -28,7 +28,7 @@ author: name: "Ralph Amissah" email: ralph.amissah@gmail.com -copyright: "(C) 2015 - 2025 Ralph Amissah, All Rights Reserved." +copyright: "(C) 2015 - 2026 Ralph Amissah, All Rights Reserved." license: - "project code: AGPL 3 or later" @@ -101,9 +101,9 @@ Make a directory and clone the sisudoc-spine project mkdir ~/git.sisudoc cd ~/git.sisudoc - git clone --depth=1 git://git.sisudoc.org/software/sisudoc-spine && \ - git clone --depth=1 git://git.sisudoc.org/software/sisudoc-spine-search-cgi && \ - git clone --depth=1 git://git.sisudoc.org/markup/sisudoc-spine-samples + git clone --depth 1 git://git.sisudoc.org/software/sisudoc-spine && \ + git clone --depth 1 git://git.sisudoc.org/software/sisudoc-spine-search-cgi && \ + git clone --depth 1 git://git.sisudoc.org/markup/sisudoc-spine-samples such a relative directory layout will be assumed in the examples that provided @@ -153,7 +153,7 @@ mkdir -p ~/.config/nix && echo "experimental-features = flakes nix-command" >> ~ - to use the D compiler ldc2: # on nix (get dependencies by setting your development environment): - nix develop ".#dsh-nixpkgs-ldc-dub" --print-build-logs -c zsh + nix develop ".#dsh-nixpkgs-ldc" --print-build-logs -c zsh # assuming you have ldc2 & dub installed on your system: dub run --compiler=ldmd2 --config=ldmd2 --combined --skip-registry=all @@ -162,7 +162,7 @@ mkdir -p ~/.config/nix && echo "experimental-features = flakes nix-command" >> ~ - to use the D compiler dmd: # on nix (get dependencies by setting your development environment): - nix develop ".#dsh-nixpkgs-dmd-dub" --print-build-logs -c zsh + nix develop ".#dsh-nixpkgs-dmd" --print-build-logs -c zsh # assuming you have dmd & dub installed on your system: dub run --compiler=dmd --config=dmd --combined --skip-registry=all @@ -11,7 +11,7 @@ author: name: "Ralph Amissah" email: ralph.amissah@gmail.com -copyright: "(C) 2015 - 2025 Ralph Amissah, All Rights Reserved." +copyright: "(C) 2015 - 2026 Ralph Amissah, All Rights Reserved." license: - "project code: AGPL 3 or later" @@ -84,9 +84,9 @@ Make a directory and clone the sisudoc-spine project mkdir ~/git.sisudoc cd ~/git.sisudoc - git clone --depth=1 git://git.sisudoc.org/software/sisudoc-spine && \ - git clone --depth=1 git://git.sisudoc.org/software/sisudoc-spine-search-cgi && \ - git clone --depth=1 git://git.sisudoc.org/markup/sisudoc-spine-samples + git clone --depth 1 git://git.sisudoc.org/software/sisudoc-spine && \ + git clone --depth 1 git://git.sisudoc.org/software/sisudoc-spine-search-cgi && \ + git clone --depth 1 git://git.sisudoc.org/markup/sisudoc-spine-samples such a relative directory layout will be assumed in the examples that provided @@ -136,7 +136,7 @@ mkdir -p ~/.config/nix && echo "experimental-features = flakes nix-command" >> ~ - to use the D compiler ldc2: # on nix (get dependencies by setting your development environment): - nix develop ".#dsh-nixpkgs-ldc-dub" --print-build-logs -c zsh + nix develop ".#dsh-nixpkgs-ldc" --print-build-logs -c zsh # assuming you have ldc2 & dub installed on your system: dub run --compiler=ldmd2 --config=ldmd2 --combined --skip-registry=all @@ -145,7 +145,7 @@ mkdir -p ~/.config/nix && echo "experimental-features = flakes nix-command" >> ~ - to use the D compiler dmd: # on nix (get dependencies by setting your development environment): - nix develop ".#dsh-nixpkgs-dmd-dub" --print-build-logs -c zsh + nix develop ".#dsh-nixpkgs-dmd" --print-build-logs -c zsh # assuming you have dmd & dub installed on your system: dub run --compiler=dmd --config=dmd --combined --skip-registry=all diff --git a/default.nix b/default.nix deleted file mode 100755 index 922b291..0000000 --- a/default.nix +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env -S nix-build -{pkgs ? import <nixpkgs> {}}: -pkgs.callPackage ./devenv.nix {} diff --git a/derivation.nix b/derivation.nix index efe8ebf..95797b4 100644..100755 --- a/derivation.nix +++ b/derivation.nix @@ -1,3 +1,4 @@ +#!/usr/bin/env -S nix-build derivation.nix { pkgs ? import <nixpkgs> {}, stdenv ? pkgs.stdenv, @@ -17,7 +18,7 @@ with ( ); targetOf = package: "${package.targetPath or "."}/${package.targetName or package.name}"; # remove reference to build tools and library sources - disallowedReferences = deps: [dcompiler dub]; + disallowedRefs = [dcompiler dub]; removeExpr = refs: ''remove-references-to ${lib.concatMapStrings (ref: " -t ${ref}") refs}''; in { mkDubDerivation = lib.makeOverridable ({ @@ -30,9 +31,9 @@ with ( } @ attrs: stdenv.mkDerivation (attrs // { - pname = package.name; + pname = attrs.pname or package.name; nativeBuildInputs = [dcompiler dub pkgs.removeReferencesTo] ++ nativeBuildInputs; - disallowedReferences = disallowedReferences deps; + disallowedReferences = disallowedRefs; passthru = passthru // { @@ -43,7 +44,7 @@ with ( src = lib.cleanSource src; }; preFixup = '' - find $out/share/cgi-bin -type f -exec ${removeExpr (disallowedReferences deps)} '{}' + || true + find $out/cgi-bin -type f -exec ${removeExpr disallowedRefs} '{}' + || true ''; buildPhase = '' runHook preBuild @@ -57,7 +58,8 @@ with ( fi done if [ "$DC" == "" ]; then - exit "Error: could not find D compiler" + echo "Error: could not find D compiler" >&2 + exit 1 fi echo "$DC_ used as D compiler to build $pname" dub run --compiler=$DC --build=release --combined --skip-registry=all @@ -71,14 +73,12 @@ with ( ''; installPhase = '' runHook preInstall - mkdir -p $out/share/cgi-bin - cp -r "${targetOf package}" $out/share/cgi-bin - install -m755 -D $out/share/cgi-bin/spine_search spine_search + mkdir -p $out/cgi-bin + install -m755 -D "${targetOf package}" "$out/cgi-bin/spine_search" runHook postInstall ''; postInstall = '' - echo "HERE ${targetOf package} $out/share/cgi-bin" - echo `ls -la $out/share/cgi-bin/spine_search` + echo `ls -la $out/cgi-bin/spine_search` ''; meta = lib.optionalAttrs (package ? description) { @@ -92,34 +92,32 @@ with ( } ); mkDubDerivation rec { - name = "spine-search-0.18.0"; - #name = "spine-search-${version}"; + pname = "spine-search"; + version = "0.18.0"; src = ./.; buildInputs = [ pkgs.sqlite ( - with pkgs-nix; [ + with pkgs; [ nixVersions.latest #nixVersions.latest #nixVersions.git ## package manager dub ## compiler ldc - rund + #rund sqlite ] ) ]; - meta = with pkgs-nix.lib; { - pname = "spine-search"; - version = "0.18.0"; + meta = with pkgs.lib; { homepage = "https://sisudoc.org"; description = "cgi sqlite search form for document object search"; longDescription = '' a sisu like parser & document generator ''; - homepage = "https://sisudoc.org"; license = licenses.agpl3Plus; platforms = platforms.linux; maintainers = ["RalphAmissah"]; + mainProgram = "spine_search"; }; } diff --git a/devenv.nix b/devenv.nix deleted file mode 100644 index 60a373e..0000000 --- a/devenv.nix +++ /dev/null @@ -1,46 +0,0 @@ -{ - pkgs ? import <nixpkgs> {}, - name ? "user-env", -}: -with pkgs; - buildEnv { - inherit name; - extraOutputsToInstall = ["out" "man" "lib"]; # to get all needed symlinks - paths = [ - # ❯❯❯ nix_related - #nix # if not on NixOS, this is needed - direnv - nixVersions.latest #nixVersions.latest #nixVersions.git - nix-prefetch-git - validatePkgConfig - nix-output-monitor - nix-tree - jq #gx - #nixfmt-rfc-style - git - # ❯❯❯ dev - gnumake - ps - # ❯❯❯ d_build_related - # ❯❯ package manager - dub - # ❯❯ compiler - #dmd - ldc - # ❯❯❯ sqlite search related - sqlite - # this will create a script that will rebuild and upgrade your setup; using shell script syntax - (writeScriptBin "nix-rebuild" '' - #!${stdenv.shell} - #cd <path-to-flake> || exit 1 - nix flake update - nix profile upgrade '.*' - '') - # puts in your root the nixpkgs version - (writeTextFile { - name = "nixpkgs-version"; - destination = "/nixpkgs-version"; - text = lib.version; - }) - ]; - } @@ -2,10 +2,10 @@ "authors": [ "Ralph Amissah" ], - "copyright": "Copyright © 2002, 2025 Ralph Amissah", + "copyright": "Copyright © 2002, 2026 Ralph Amissah", "name": "spine_search", "version": "0.18.0", - "description": "cgi search for spine, a sisu like document parser", + "description": "cgi search for spine, a sisu-like document parser", "homepage": "https://sisudoc.org", "license": "AGPL-3.0+", "targetPath": "./cgi-bin", diff --git a/dub_describe.json b/dub_describe.json index e87b668..b034366 100644 --- a/dub_describe.json +++ b/dub_describe.json @@ -1,7 +1,7 @@ { "rootPackage": "spine_search", "configuration": "default", - "buildType": "$DFLAGS", + "buildType": "debug", "compiler": "ldc", "architecture": [ "x86_64" @@ -14,13 +14,13 @@ { "path": "./", "name": "spine_search", - "version": "0.12.0", + "version": "0.18.0", "description": "cgi search for spine, a sisu like document parser", "homepage": "https://sisudoc.org", "authors": [ "Ralph Amissah" ], - "copyright": "Copyright © 2002, 2023 Ralph Amissah", + "copyright": "Copyright © 2002, 2025 Ralph Amissah", "license": "AGPL-3.0+", "dependencies": [ "spine_search:arsd.cgi", @@ -43,8 +43,9 @@ "versions": [], "debugVersions": [], "importPaths": [ - "src/" + "./views" ], + "cImportPaths": [], "stringImportPaths": [ "./views" ], @@ -70,66 +71,42 @@ "files": [ { "role": "source", - "path": "src/ext_depends/arsd/cgi.d" - }, - { - "role": "source", - "path": "src/ext_depends/arsd/core.d" - }, - { - "role": "source", - "path": "src/ext_depends/d2sqlite3/source/d2sqlite3/database.d" - }, - { - "role": "source", - "path": "src/ext_depends/d2sqlite3/source/d2sqlite3/internal/memory.d" - }, - { - "role": "source", - "path": "src/ext_depends/d2sqlite3/source/d2sqlite3/internal/util.d" - }, - { - "role": "source", - "path": "src/ext_depends/d2sqlite3/source/d2sqlite3/library.d" - }, - { - "role": "source", - "path": "src/ext_depends/d2sqlite3/source/d2sqlite3/package.d" + "path": "src/sisudoc/spine_search.d" }, { - "role": "source", - "path": "src/ext_depends/d2sqlite3/source/d2sqlite3/results.d" + "role": "stringImport", + "path": "views/configuration.txt" }, { - "role": "source", - "path": "src/ext_depends/d2sqlite3/source/d2sqlite3/sqlite3.d" + "role": "stringImport", + "path": "views/configuration.txt_" }, { - "role": "source", - "path": "src/ext_depends/d2sqlite3/source/d2sqlite3/statement.d" + "role": "stringImport", + "path": "views/configuration.txt_localhost_" }, { - "role": "source", - "path": "src/ext_depends/d2sqlite3/source/tests.d" + "role": "stringImport", + "path": "views/configuration.txt_remotehost_" }, { - "role": "source", - "path": "src/spine_search.d" + "role": "stringImport", + "path": "views/configuration_example.txt" }, { "role": "stringImport", - "path": "views/configuration.txt" + "path": "views/version.txt" }, { "role": "stringImport", - "path": "views/version.txt" + "path": "views/version.txt_" } ] }, { "path": "./", "name": "spine_search:arsd.cgi", - "version": "0.12.0", + "version": "0.18.0", "description": "cgi", "homepage": "https://code.dlang.org/packages/arsd-official", "authors": [ @@ -157,6 +134,7 @@ "importPaths": [ "./src/ext_depends/arsd" ], + "cImportPaths": [], "stringImportPaths": [ "views" ], @@ -192,14 +170,34 @@ }, { "role": "stringImport", + "path": "views/configuration.txt_" + }, + { + "role": "stringImport", + "path": "views/configuration.txt_localhost_" + }, + { + "role": "stringImport", + "path": "views/configuration.txt_remotehost_" + }, + { + "role": "stringImport", + "path": "views/configuration_example.txt" + }, + { + "role": "stringImport", "path": "views/version.txt" + }, + { + "role": "stringImport", + "path": "views/version.txt_" } ] }, { "path": "./", "name": "spine_search:d2sqlite3", - "version": "0.12.0", + "version": "0.18.0", "description": "A thin wrapper around SQLite 3", "homepage": "https://github.com/dlang-community/d2sqlite3", "authors": [ @@ -230,6 +228,7 @@ "importPaths": [ "./src/ext_depends/d2sqlite3/source" ], + "cImportPaths": [], "stringImportPaths": [ "views" ], @@ -293,7 +292,27 @@ }, { "role": "stringImport", + "path": "views/configuration.txt_" + }, + { + "role": "stringImport", + "path": "views/configuration.txt_localhost_" + }, + { + "role": "stringImport", + "path": "views/configuration.txt_remotehost_" + }, + { + "role": "stringImport", + "path": "views/configuration_example.txt" + }, + { + "role": "stringImport", "path": "views/version.txt" + }, + { + "role": "stringImport", + "path": "views/version.txt_" } ] } @@ -311,12 +330,7 @@ "targetName": "spine_search", "workingDirectory": "", "mainSourceFile": "", - "dflags": [ - "\"-O2", - "-inline", - "-boundscheck=on", - "-color=on\"" - ], + "dflags": [], "lflags": [], "libs": [ "sqlite3" @@ -326,18 +340,7 @@ "./libspine_search_d2sqlite3.a" ], "sourceFiles": [ - "./src/ext_depends/arsd/cgi.d", - "./src/ext_depends/arsd/core.d", - "./src/ext_depends/d2sqlite3/source/d2sqlite3/database.d", - "./src/ext_depends/d2sqlite3/source/d2sqlite3/internal/memory.d", - "./src/ext_depends/d2sqlite3/source/d2sqlite3/internal/util.d", - "./src/ext_depends/d2sqlite3/source/d2sqlite3/library.d", - "./src/ext_depends/d2sqlite3/source/d2sqlite3/package.d", - "./src/ext_depends/d2sqlite3/source/d2sqlite3/results.d", - "./src/ext_depends/d2sqlite3/source/d2sqlite3/sqlite3.d", - "./src/ext_depends/d2sqlite3/source/d2sqlite3/statement.d", - "./src/ext_depends/d2sqlite3/source/tests.d", - "./src/spine_search.d" + "./src/sisudoc/spine_search.d" ], "injectSourceFiles": [], "copyFiles": [], @@ -351,17 +354,23 @@ "versionFilters": [], "debugVersionFilters": [], "importPaths": [ - "./src/", + "./views", "./src/ext_depends/arsd", "./src/ext_depends/d2sqlite3/source" ], + "cImportPaths": [], "stringImportPaths": [ "./views" ], "importFiles": [], "stringImportFiles": [ + "./views/configuration.txt_localhost_", + "./views/configuration.txt_", + "./views/configuration.txt_remotehost_", + "./views/version.txt_", + "./views/version.txt", "./views/configuration.txt", - "./views/version.txt" + "./views/configuration_example.txt" ], "preGenerateCommands": [], "postGenerateCommands": [], @@ -378,9 +387,16 @@ "postBuildEnvironments": {}, "preRunEnvironments": {}, "postRunEnvironments": {}, - "requirements": {}, - "options": {} + "requirements": [ + "allowWarnings" + ], + "options": [ + "debugMode", + "debugInfo", + "warnings" + ] }, + "cacheArtifactPath": "/home/ralph/.dub/cache/spine_search/0.18.0/build/default-debug-ad4Jr5KYHL-vWkDoM8eUmQ/spine_search", "dependencies": [ "spine_search:arsd.cgi", "spine_search:d2sqlite3" @@ -402,12 +418,7 @@ "targetName": "spine_search_arsd.cgi", "workingDirectory": "", "mainSourceFile": "", - "dflags": [ - "\"-O2", - "-inline", - "-boundscheck=on", - "-color=on\"" - ], + "dflags": [], "lflags": [], "libs": [], "linkerFiles": [], @@ -427,13 +438,19 @@ "importPaths": [ "./src/ext_depends/arsd" ], + "cImportPaths": [], "stringImportPaths": [ "./views" ], "importFiles": [], "stringImportFiles": [ + "./views/configuration.txt_localhost_", + "./views/configuration.txt_", + "./views/configuration.txt_remotehost_", + "./views/version.txt_", + "./views/version.txt", "./views/configuration.txt", - "./views/version.txt" + "./views/configuration_example.txt" ], "preGenerateCommands": [], "postGenerateCommands": [], @@ -450,9 +467,14 @@ "postBuildEnvironments": {}, "preRunEnvironments": {}, "postRunEnvironments": {}, - "requirements": {}, - "options": {} + "requirements": [], + "options": [ + "debugMode", + "debugInfo", + "warningsAsErrors" + ] }, + "cacheArtifactPath": "/home/ralph/.dub/cache/spine_search/0.18.0/+arsd.cgi/build/cgi-debug-v2DAX8HeTpjjJrCrVfJhng/libspine_search_arsd.cgi.a", "dependencies": [], "linkDependencies": [] }, @@ -468,12 +490,7 @@ "targetName": "spine_search_d2sqlite3", "workingDirectory": "", "mainSourceFile": "", - "dflags": [ - "\"-O2", - "-inline", - "-boundscheck=on", - "-color=on\"" - ], + "dflags": [], "lflags": [], "libs": [ "sqlite3" @@ -502,13 +519,19 @@ "importPaths": [ "./src/ext_depends/d2sqlite3/source" ], + "cImportPaths": [], "stringImportPaths": [ "./views" ], "importFiles": [], "stringImportFiles": [ + "./views/configuration.txt_localhost_", + "./views/configuration.txt_", + "./views/configuration.txt_remotehost_", + "./views/version.txt_", + "./views/version.txt", "./views/configuration.txt", - "./views/version.txt" + "./views/configuration_example.txt" ], "preGenerateCommands": [], "postGenerateCommands": [], @@ -525,9 +548,14 @@ "postBuildEnvironments": {}, "preRunEnvironments": {}, "postRunEnvironments": {}, - "requirements": {}, - "options": {} + "requirements": [], + "options": [ + "debugMode", + "debugInfo", + "warningsAsErrors" + ] }, + "cacheArtifactPath": "/home/ralph/.dub/cache/spine_search/0.18.0/+d2sqlite3/build/d2sqlite3-debug-WjZayd7TAWeyfdiH60vT-A/libspine_search_d2sqlite3.a", "dependencies": [], "linkDependencies": [] } @@ -2,11 +2,11 @@ "nodes": { "nixpkgs": { "locked": { - "lastModified": 1754651824, - "narHash": "sha256-aB7ft6njy9EJfuW+rdToNChfRrHNRw/yTg5cSEnG+HI=", + "lastModified": 1777826146, + "narHash": "sha256-wQ/iN5Zp5VIa3ebBibijPnLyKhor+xEbDy4d0goa9Zs=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "b069b7c1e2fe1a3a24221428558bf44128d3d5c8", + "rev": "73c703c22422b8951895a960959dbbaca7296492", "type": "github" }, "original": { @@ -29,11 +29,11 @@ ] }, "locked": { - "lastModified": 1754707636, - "narHash": "sha256-np3xPi0/PPEe4OWPPrTNV+eu8QIgSp83NaKMukE0hJE=", + "lastModified": 1775877366, + "narHash": "sha256-W33lhJ9+/KEl1xoGKw/BYHIZpuqjLpDHjlue2CVQrcM=", "ref": "refs/heads/main", - "rev": "d393ad85676436fef1e37e712405d6d10593977d", - "revCount": 23, + "rev": "40ccbcf2cf53b1fd1476e7ced6b70c29d491d464", + "revCount": 30, "type": "git", "url": "git://git.sisudoc.org/package/dlang-nix-overlays-and-flake" }, @@ -1,35 +1,28 @@ { description = "a sisu like document parser search form"; inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; - #inputs.flake-utils.url = "github:numtide/flake-utils"; inputs.spine-d-overlay = { url = "git://git.sisudoc.org/package/dlang-nix-overlays-and-flake"; inputs.nixpkgs.follows = "nixpkgs"; - flake = true; }; outputs = { self, nixpkgs, - #flake-utils, spine-d-overlay, ... }@inputs: let pname = "spine_search"; version = "0.18.0"; - shell = ./shell.nix; - devEnv = ./nixDevEnv.sh; + #shell = ./shell.nix; + #devEnv = ./nixDevEnv.sh; supportedSystems = ["x86_64-linux"]; forAllSystems = nixpkgs.lib.genAttrs supportedSystems; nixpkgsFor = forAllSystems (system: import nixpkgs {inherit system;}); # nixpkgs instantiated for supported system types - checkPhase = '' - runHook preCheck - dub test --combined --skip-registry=all - runHook postCheck - ''; localOverlay = (final: prev: { - ldc = prev.callPackage ./nix-overlays/ldc { }; - dmd = prev.callPackage ./nix-overlays/dmd { }; - dub = prev.callPackage ./nix-overlays/dub { }; + ldc = prev.callPackage ./nix-overlays/ldc { }; # -> ok 1.42.0 + # dmd overlay: revert to GCC14; (current) GCC 15 introduced nullptr in its headers, and DMD's ImportC parser needs update to handle it, monitor and update: + dmd = prev.callPackage ./nix-overlays/dmd { stdenv = prev.gcc14Stdenv; }; # -> ok 2.111.0 + dub = prev.callPackage ./nix-overlays/dub { }; # -> ? 1.40.0 #gdc = prev.callPackage ./nix-overlays/gdc { }; }); pkgsForSystem = system: import nixpkgs { @@ -41,165 +34,79 @@ preBuild = '' export HOME=$(pwd) ''; + checkPhase = '' + runHook preCheck + dub test --combined --skip-registry=all + runHook postCheck + ''; installPhase = '' runHook preInstall - mkdir -p $out/share/cgi-bin - install -m755 -D ./cgi-bin/spine_search $out/share/cgi-bin/spine_search + mkdir -p $out/cgi-bin + install -m755 -D ./cgi-bin/spine_search $out/cgi-bin/spine_search runHook postInstall ''; postInstall = '' - echo `ls -la $out/share/cgi-bin/spine_search` + echo `ls -la $out/cgi-bin/spine_search` ''; + # Helper to reduce duplication across build variants + mkSpineSearch = {pkgs, compiler, compilerCmd, dubConfig}: pkgs.stdenv.mkDerivation { + inherit pname version; + meta.mainProgram = "spine_search"; + src = self; + buildInputs = [pkgs.sqlite]; + nativeBuildInputs = [pkgs.dub compiler pkgs.gnumake]; + buildPhase = '' + runHook preBuild + buildCMD="dub run --cache=local --compiler=$(type -P ${compilerCmd}) --build=${dubConfig} --combined --skip-registry=all" + echo $buildCMD + $buildCMD + echo $buildCMD + runHook postBuild + ''; + inherit preBuild; + inherit checkPhase; + inherit installPhase; + inherit postInstall; + }; in { packages = forAllSystems (system: let pkgs-ovl = pkgsForSystem system; pkgs-nix = nixpkgsFor.${system}; - in - with pkgs-nix; { - default = stdenv.mkDerivation { - inherit pname; - inherit version; - meta.mainProgram = "spine-search"; - executable = true; - src = self; - inherit shell; - inherit devEnv; - buildInputs = [sqlite]; - nativeBuildInputs = [dub ldc gnumake]; - buildPhase = '' - runHook preBuild - for DC_ in dmd ldmd2 ldc2 gdc gdmd; do - echo "- check for D compiler $DC_" - DC=$(type -P $DC_ || echo "") - if [ ! "$DC" == "" ]; then break; fi - done - if [ "$DC" == "" ]; then exit "Error: could not find D compiler"; fi - echo "$DC_ used as D compiler to build $pname" - buildCMD="dub run --cache=local --compiler=$DC --build=$DC_ --combined --skip-registry=all" - echo $buildCMD - $buildCMD - echo $buildCMD - runHook postBuild - ''; - inherit preBuild; - inherit checkPhase; - inherit installPhase; - inherit postInstall; - }; - spine-search-nixpkgs-dmd = stdenv.mkDerivation { - inherit pname; - inherit version; - meta.mainProgram = "spine_search-dmd"; - executable = true; - src = self; - inherit shell; - inherit devEnv; - buildInputs = with pkgs-nix; [sqlite]; - nativeBuildInputs = with pkgs-nix; [dub dmd gnumake]; - buildPhase = '' - runHook preBuild - buildCMD="dub run --cache=local --compiler=$(type -P dmd) --build=dmd --combined --skip-registry=all" - echo $buildCMD - $buildCMD - echo $buildCMD - runHook postBuild - ''; - inherit preBuild; - inherit checkPhase; - inherit installPhase; - inherit postInstall; - }; - spine-search-nixpkgs-ldc = stdenv.mkDerivation { - inherit pname; - inherit version; - meta.mainProgram = "spine_search-ldc"; - executable = true; - src = self; - inherit shell; - inherit devEnv; - buildInputs = with pkgs-nix; [sqlite]; - nativeBuildInputs = with pkgs-nix; [dub ldc gnumake]; - buildPhase = '' - runHook preBuild - buildCMD="dub run --cache=local --compiler=$(type -P ldmd2) --build=ldmd2 --combined --skip-registry=all" - echo $buildCMD - $buildCMD - echo $buildCMD - runHook postBuild - ''; - inherit preBuild; - inherit checkPhase; - inherit installPhase; - inherit postInstall; - }; - spine-search-overlay-dmd = stdenv.mkDerivation { - inherit pname; - inherit version; - meta.mainProgram = "spine_search-dmd"; - executable = true; - src = self; - inherit shell; - inherit devEnv; - buildInputs = with pkgs-ovl; [sqlite]; - nativeBuildInputs = with pkgs-ovl; [dub dmd gnumake]; - buildPhase = '' - runHook preBuild - buildCMD="dub run --cache=local --compiler=$(type -P dmd) --build=dmd --combined --skip-registry=all" - echo $buildCMD - $buildCMD - echo $buildCMD - runHook postBuild - ''; - inherit preBuild; - inherit checkPhase; - inherit installPhase; - inherit postInstall; - }; - spine-search-overlay-ldc = stdenv.mkDerivation { - inherit pname; - inherit version; - meta.mainProgram = "spine_search-ldc"; - executable = true; - src = self; - inherit shell; - inherit devEnv; - buildInputs = with pkgs-ovl; [sqlite]; - nativeBuildInputs = with pkgs-ovl; [dub ldc gnumake]; - buildPhase = '' - runHook preBuild - buildCMD="dub run --cache=local --compiler=$(type -P ldmd2) --build=ldmd2 --combined --skip-registry=all" - echo $buildCMD - $buildCMD - echo $buildCMD - runHook postBuild - ''; - inherit preBuild; - inherit checkPhase; - inherit installPhase; - inherit postInstall; - }; - #spine-search-overlay-gdc = stdenv.mkDerivation { - # inherit pname; - # inherit version; - # meta.mainProgram = "spine_search-gdc"; - # executable = true; - # src = self; - # inherit shell; - # inherit devEnv; - # buildInputs = with pkgs-ovl; [ sqlite ]; - # nativeBuildInputs = with pkgs-ovl; [ dub gdc gnumake ]; - # buildPhase = '' - # runHook preBuild - # dub run --cache=local --compiler=$(type -P gdc) --build=gdc --combined --skip-registry=all - # runHook postBuild - # ''; - # inherit preBuild; - # inherit checkPhase; - # inherit installPhase; - # inherit postInstall; - #}; - #vendorSha256 = "sha256-0Q00000000000000000000000000000000000000000="; - }); + in { + default = self.packages.${system}.spine-search-nixpkgs-ldc; + spine-search-nixpkgs-dmd = mkSpineSearch { + pkgs = pkgs-nix; + # nixpkgs dmd: revert to GCC14 (same nullptr/ImportC fix as overlay dmd) + compiler = pkgs-nix.dmd.override { stdenv = pkgs-nix.gcc14Stdenv; }; + #compiler = pkgs-nix.dmd; + compilerCmd = "dmd"; + dubConfig = "dmd"; + }; + spine-search-nixpkgs-ldc = mkSpineSearch { + pkgs = pkgs-nix; + compiler = pkgs-nix.ldc; + compilerCmd = "ldmd2"; + dubConfig = "ldmd2"; + }; + spine-search-overlay-dmd = mkSpineSearch { + pkgs = pkgs-ovl; + compiler = pkgs-ovl.dmd; + compilerCmd = "dmd"; + dubConfig = "dmd"; + }; + spine-search-overlay-ldc = mkSpineSearch { + pkgs = pkgs-ovl; + compiler = pkgs-ovl.ldc; + compilerCmd = "ldmd2"; + dubConfig = "ldmd2"; + }; + #spine-search-overlay-gdc = mkSpineSearch { + # pkgs = pkgs-ovl; + # compiler = pkgs-ovl.gdc; + # compilerCmd = "gdc"; + # dubConfig = "gdc"; + #}; + }); devShells = forAllSystems (system: let pkgs-ovl = pkgsForSystem system; pkgs-nix = nixpkgsFor.${system}; @@ -211,9 +118,7 @@ with pkgs-nix; { dsh-d-overlay = spine-d-overlay.devShells.${system}.default; dsh-overlay = mkShell { - name = "spine-0.18.0 base dev shell, ldc-1.41.0, dub-1.39.0"; - inherit shell; - inherit devEnv; + name = "${pname}-${version} dev shell (overlay)"; packages = with pkgs-ovl; [ ldc #dmd @@ -223,10 +128,8 @@ ]; inherit shellHook; }; - dsh-nixpkgs-dmd-dub = mkShell { - name = "spine-0.18.0 base dev shell"; - inherit shell; - inherit devEnv; + dsh-nixpkgs-dmd = mkShell { + name = "${pname}-${version} dev shell (nixpkgs dmd)"; packages = [ dmd dub @@ -235,10 +138,8 @@ ]; inherit shellHook; }; - dsh-nixpkgs-ldc-dub = mkShell { - name = "spine-0.18.0 base dev shell"; - inherit shell; - inherit devEnv; + dsh-nixpkgs-ldc = mkShell { + name = "${pname}-${version} dev shell (nixpkgs ldc)"; packages = [ ldc dub @@ -247,10 +148,9 @@ ]; inherit shellHook; }; - dsh-overlay-dmd-dub = mkShell { - name = "spine-0.18.0 base dev shell, dmd-2.111.0, dub-1.39.0"; - inherit shell; - inherit devEnv; + dsh-overlay-dmd = mkShell { + name = "${pname}-${version} dev shell (overlay dmd)"; + # "spine-0.18.0 base dev shell, dmd-2.111.0, dub-1.40.0"; packages = with pkgs-ovl; [ dmd dub @@ -259,10 +159,9 @@ ]; inherit shellHook; }; - dsh-overlay-ldc-dub = mkShell { - name = "spine-0.18.0 base dev shell, ldc-1.41.0, dub-1.39.0"; - inherit shell; - inherit devEnv; + dsh-overlay-ldc = mkShell { + name = "${pname}-${version} dev shell (overlay ldc)"; + # "spine-0.18.0 base dev shell, ldc-1.42.0, dub-1.40.0"; packages = with pkgs-ovl; [ ldc dub @@ -272,9 +171,7 @@ inherit shellHook; }; dsh-sqlite = mkShell { - name = "spine dev shell for latex & pdf output"; - inherit shell; - inherit devEnv; + name = "${pname}-${version} dev shell (sqlite)"; packages = [ ldc #dmd @@ -284,7 +181,8 @@ ]; inherit shellHook; }; - default = import ./shell.nix {inherit pkgs-nix;}; + default = self.devShells.${system}.dsh-nixpkgs-ldc; + #default = import ./shell.nix {pkgs = pkgs-nix;}; }); }; } @@ -83,7 +83,7 @@ set_depends: get_depends # cp cgi search form cp-cgi: sudo cp -vi /var/www/cgi/cgi-bin/spine_search /var/www/cgi/cgi-bin/spine_search_old && \ - sudo cp -vi result/share/cgi-bin/spine_search /var/www/cgi/cgi-bin/spine_search + sudo cp -vi result/cgi-bin/spine_search /var/www/cgi/cgi-bin/spine_search # git archive gitZip: diff --git a/nix-overlays/README b/nix-overlays/README index 75b498b..de281c7 100644 --- a/nix-overlays/README +++ b/nix-overlays/README @@ -4,7 +4,7 @@ #+FILETAGS: :dlang:build:tools: #+AUTHOR: Ralph Amissah #+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] -#+COPYRIGHT: Copyright (C) 2015 - 2025 Ralph Amissah +#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah #+LANGUAGE: en #+STARTUP: content hideblocks hidestars noindent entitiespretty @@ -16,7 +16,7 @@ compiler and build tools: ldc, dub and dtools, than exist in nixpkgs at the time of publishing, in some cases with additional fixes. nix overlays for (updates nixpkgs, more recent versions of): - - ldc ( 1.30.0 -> 1.41.0 ) + - ldc ( 1.30.0 -> 1.42.0 ) - dub ( 1.23.0 -> 1.40.0 ) [for nix versions 1.31.0 ... 1.33.0 broken] - dtools ( 2.095.1 -> 2.110.0 ) @@ -47,13 +47,13 @@ search nixpkgs here: - https://search.nixos.org/packages?channel=unstable&from=0&size=100&sort=relevance&query= ** compilers -*** ldc OK ✓ ( 1.30.0 -> 1.41.0 ) +*** ldc OK ✓ ( 1.30.0 -> 1.42.0 ) - https://wiki.dlang.org/LDC - https://github.com/ldc-developers/ldc - https://github.com/ldc-developers/ldc/releases - nix-prefetch-url --unpack https://github.com/ldc-developers/ldc/archive/refs/tags/v1.41.0.tar.gz + nix-prefetch-url --unpack https://github.com/ldc-developers/ldc/archive/refs/tags/v1.42.0.tar.gz - https://github.com/ldc-developers/ldc/issues - version in nixpkgs: diff --git a/nix-overlays/dtools/disabled-tests.diff b/nix-overlays/dtools/disabled-tests.diff index a22825a..79eb1cb 100644 --- a/nix-overlays/dtools/disabled-tests.diff +++ b/nix-overlays/dtools/disabled-tests.diff @@ -3,7 +3,7 @@ @@ -616,7 +616,7 @@ void runTests(string rdmdApp, string compiler, string model) enforce(res.status == 1, res.output); } - + - version (Posix) + version (none) { diff --git a/nix-overlays/dtools/fix-ldc-arm64.diff b/nix-overlays/dtools/fix-ldc-arm64.diff index 66eaa22..1c650a6 100644 --- a/nix-overlays/dtools/fix-ldc-arm64.diff +++ b/nix-overlays/dtools/fix-ldc-arm64.diff @@ -82,21 +82,21 @@ Adapted from https://github.com/dlang/tools/commit/6c6a042d1b08e3ec1790bd07a7f69 @@ -9,9 +9,8 @@ DUB=dub WITH_DOC = no DOC = ../dlang.org - + -# Load operating system $(OS) (e.g. linux, osx, ...) and $(MODEL) (e.g. 32, 64) detection Makefile from dmd -$(shell [ ! -d $(DMD_DIR) ] && git clone --depth=1 https://github.com/dlang/dmd $(DMD_DIR)) -include $(DMD_DIR)/compiler/src/osmodel.mak +# Load operating system $(OS) (e.g. linux, osx, ...) and $(MODEL) (e.g. 32, 64) detection Makefile +include osmodel.mak - + ifeq (windows,$(OS)) DOTEXE:=.exe @@ -30,7 +29,7 @@ DFLAGS = $(MODEL_FLAG) $(if $(findstring windows,$(OS)),,-fPIC) -preview=dip1000 DFLAGS += $(WARNINGS) - + # Default DUB flags (DUB uses a different architecture format) -DUBFLAGS = --arch=$(subst 32,x86,$(subst 64,x86_64,$(MODEL))) +DUBFLAGS = --arch=$(ARCH) - + TOOLS = \ $(ROOT)/catdoc$(DOTEXE) \ diff --git a/nix-overlays/ldc/package.nix b/nix-overlays/ldc/package.nix index 1767785..b5daa75 100644 --- a/nix-overlays/ldc/package.nix +++ b/nix-overlays/ldc/package.nix @@ -11,7 +11,7 @@ targetPackages, cmake, ninja, - llvm_20, + llvm_21, curl, tzdata, lit, @@ -32,13 +32,13 @@ in stdenv.mkDerivation (finalAttrs: { pname = "ldc"; - version = "1.41.0"; + version = "1.42.0"; src = fetchFromGitHub { owner = "ldc-developers"; repo = "ldc"; rev = "v${finalAttrs.version}"; - hash = "sha256-6LcpY3LSFK4KgEiGrFp/LONu5Vr+/+vI04wEEpF3s+s="; + hash = "sha256-adA14tx/bruGvHVoODz13f8h/U2ol1lK0ytxnypsLv8="; fetchSubmodules = true; }; @@ -64,7 +64,7 @@ stdenv.mkDerivation (finalAttrs: { ''; nativeBuildInputs = [ - cmake ldcBootstrap lit lit.python llvm_20.dev makeWrapper ninja unzip + cmake ldcBootstrap lit lit.python llvm_21.dev makeWrapper ninja unzip ] ++ lib.optionals (!stdenv.hostPlatform.isDarwin) [ # https://github.com/NixOS/nixpkgs/pull/36378#issuecomment-385034818 gdb @@ -138,6 +138,8 @@ stdenv.mkDerivation (finalAttrs: { find $out/bin -type f -exec ${removeReferencesTo}/bin/remove-references-to -t ${ldcBootstrap} '{}' + ''; + # doCheck = false; + disallowedReferences = [ ldcBootstrap ]; meta = with lib; { @@ -181,6 +183,6 @@ stdenv.mkDerivation (finalAttrs: { no-references-to-compiler = helloWorld.overrideAttrs { disallowedReferences = [ ldc ]; dFlags = ["-g"]; + }; }; - }; }) diff --git a/org/COPYRIGHT b/org/COPYRIGHT index b1cdad5..1678968 100644 --- a/org/COPYRIGHT +++ b/org/COPYRIGHT @@ -5,7 +5,7 @@ - Author: Ralph Amissah [ralph.amissah@gmail.com] - - Copyright: (C) 2015 - 2025 Ralph Amissah + - Copyright: (C) 2015 - 2026 Ralph Amissah - Spine, Doc Reform (related to SiSU) uses standard: - docReform markup syntax (based on SiSU markup) diff --git a/org/config_d_cfte.org b/org/config_d_cfte.org index df29cc5..ef471f8 100644 --- a/org/config_d_cfte.org +++ b/org/config_d_cfte.org @@ -4,16 +4,16 @@ #+FILETAGS: :spine:hub: #+AUTHOR: Ralph Amissah #+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] -#+COPYRIGHT: Copyright (C) 2015 - 2025 Ralph Amissah +#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah #+LANGUAGE: en #+STARTUP: content hideblocks hidestars noindent entitiespretty -#+PROPERTY: header-args :exports code -#+PROPERTY: header-args+ :noweb yes -#+PROPERTY: header-args+ :results no -#+PROPERTY: header-args+ :cache no -#+PROPERTY: header-args+ :padline no +#+PROPERTY: header-args+ :eval never-export :exports code +#+PROPERTY: header-args+ :noweb yes :padline no +#+PROPERTY: header-args+ :results silent :cache no #+PROPERTY: header-args+ :mkdirp yes #+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t +- magic single double-quote → " ← FIX changes hilighting behavior (occuring + after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX - [[./doc-reform.org][doc-reform.org]] [[../org/][org/]] diff --git a/org/config_dub.org b/org/config_dub.org index 60a8a43..6d91118 100644 --- a/org/config_dub.org +++ b/org/config_dub.org @@ -4,16 +4,16 @@ #+FILETAGS: :spine:hub: #+AUTHOR: Ralph Amissah #+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] -#+COPYRIGHT: Copyright (C) 2015 - 2025 Ralph Amissah +#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah #+LANGUAGE: en #+STARTUP: content hideblocks hidestars noindent entitiespretty -#+PROPERTY: header-args :exports code -#+PROPERTY: header-args+ :noweb yes -#+PROPERTY: header-args+ :results no -#+PROPERTY: header-args+ :cache no -#+PROPERTY: header-args+ :padline no +#+PROPERTY: header-args+ :eval never-export :exports code +#+PROPERTY: header-args+ :noweb yes :padline no +#+PROPERTY: header-args+ :results silent :cache no #+PROPERTY: header-args+ :mkdirp yes #+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t +- magic single double-quote → " ← FIX changes hilighting behavior (occuring + after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX - [[./doc-reform.org][doc-reform.org]] [[./][org/]] @@ -47,10 +47,10 @@ "authors": [ "Ralph Amissah" ], - "copyright": "Copyright © 2002, 2025 Ralph Amissah", + "copyright": "Copyright © 2002, 2026 Ralph Amissah", "name": "spine_search", "version": "<<spine_version>>", - "description": "cgi search for spine, a sisu like document parser", + "description": "cgi search for spine, a sisu-like document parser", "homepage": "https://sisudoc.org", "license": "AGPL-3.0+", "targetPath": "./cgi-bin", diff --git a/org/config_env.org b/org/config_env.org index e5f2a43..42d99bc 100644 --- a/org/config_env.org +++ b/org/config_env.org @@ -4,16 +4,16 @@ #+FILETAGS: :spine:build:tools: #+AUTHOR: Ralph Amissah #+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] -#+COPYRIGHT: Copyright (C) 2015 - 2025 Ralph Amissah +#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah #+LANGUAGE: en #+STARTUP: content hideblocks hidestars noindent entitiespretty -#+PROPERTY: header-args :exports code -#+PROPERTY: header-args+ :noweb yes -#+PROPERTY: header-args+ :results no -#+PROPERTY: header-args+ :cache no -#+PROPERTY: header-args+ :padline no +#+PROPERTY: header-args+ :eval never-export :exports code +#+PROPERTY: header-args+ :noweb yes :padline no +#+PROPERTY: header-args+ :results silent :cache no #+PROPERTY: header-args+ :mkdirp yes #+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t +- magic single double-quote → " ← FIX changes hilighting behavior (occuring + after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX - [[./doc-reform.org][doc-reform.org]] [[../org/][org/]] - [[./config_nix.org][config_nix.org]] @@ -33,12 +33,16 @@ fi #+NAME: envrc #+HEADER: :tangle ../.envrc +#+HEADER: :tangle-mode (identity #o644) +#+HEADER: :shebang "# -*- mode: sh -*-" #+BEGIN_SRC shell if [ -f .envrc-git-init ]; then source_env_if_exists .envrc-git-init || source .envrc-git-init fi if [ -f .envrc-local ]; then source_env_if_exists .envrc-local || source .envrc-local +elif [ -f .envrc-local_ ]; then + source_env_if_exists .envrc-local_ || source .envrc-local_ fi if [ -f .envrc-nix ]; then source_env_if_exists .envrc-nix || source .envrc-nix @@ -53,14 +57,16 @@ fi - ${NixDirEnvVersion} #+HEADER: :tangle ../.envrc-nix +#+HEADER: :tangle-mode (identity #o644) +#+HEADER: :shebang "# -*- mode: nix -*-" #+BEGIN_SRC shell NIX_ENFORCE_PURITY=1 -# - https://github.com/nix-community/nix-direnv -NixDirEnvVersion="<<direnv_version>>" -NixDirEnvSHA="<<direnv_hash>>" -if ! has nix_direnv_version || ! nix_direnv_version ${NixDirEnvVersion}; then - source_url "https://raw.githubusercontent.com/nix-community/nix-direnv/${NixDirEnvVersion}/direnvrc" "${NixDirEnvSHA}" -fi +## - https://github.com/nix-community/nix-direnv +#NixDirEnvVersion="<<direnv_version>>" +#NixDirEnvSHA="<<direnv_hash>>" +#if ! has nix_direnv_version || ! nix_direnv_version ${NixDirEnvVersion}; then +# source_url "https://raw.githubusercontent.com/nix-community/nix-direnv/${NixDirEnvVersion}/direnvrc" "${NixDirEnvSHA}" +#fi watch_file flake.lock watch_file flake.nix watch_file shell.nix @@ -80,11 +86,11 @@ echo ' • nix develop ❯❯ nix develop ".#" --print-build-logs -c zsh • nix develop using nixpkgs - ❯❯ nix develop ".#dsh-nixpkgs-dmd-dub" --print-build-logs -c zsh - ❯❯ nix develop ".#dsh-nixpkgs-ldc-dub" --print-build-logs -c zsh + ❯❯ nix develop ".#dsh-nixpkgs-dmd" --print-build-logs -c zsh + ❯❯ nix develop ".#dsh-nixpkgs-ldc" --print-build-logs -c zsh • nix develop using package overlays: (dmd-<<dmd_version>>, ldc-<<ldc_version>>, dub-<<dub_version>>) - ❯❯ nix develop ".#dsh-overlay-dmd-dub" --print-build-logs -c zsh - ❯❯ nix develop ".#dsh-overlay-ldc-dub" --print-build-logs -c zsh + ❯❯ nix develop ".#dsh-overlay-dmd" --print-build-logs -c zsh + ❯❯ nix develop ".#dsh-overlay-ldc" --print-build-logs -c zsh ❯❯ nix flake show • to build project: @@ -102,15 +108,16 @@ echo ' • to build using dub on nix (get dependencies by setting your development environment): - ❯❯ nix develop ".#dsh-nixpkgs-dmd-dub" --print-build-logs -c zsh + ❯❯ nix develop ".#dsh-nixpkgs-dmd" --print-build-logs -c zsh ❯❯ dub run --compiler=dmd --config=dmd --combined --skip-registry=all - ❯❯ nix develop ".#dsh-nixpkgs-ldc-dub" --print-build-logs -c zsh + ❯❯ nix develop ".#dsh-nixpkgs-ldc" --print-build-logs -c zsh ❯❯ dub run --compiler=ldmd2 --config=ldmd2 --combined --skip-registry=all • for develop environment & build options - ❯❯ nix flake show + ❯❯ nix flake update && nix flake check && nix flake show + ❯❯ nix flake show ' #+END_SRC @@ -118,6 +125,8 @@ echo ' ** .envrc-git-init #+HEADER: :tangle ../.envrc-git-init +#+HEADER: :tangle-mode (identity #o644) +#+HEADER: :shebang "# -*- mode: sh -*-" #+HEADER: :noweb yes #+BEGIN_SRC shell if [[ ! -d ./.git ]]; then @@ -130,18 +139,58 @@ fi - bespoke modify appropriately and generate if needed -#+HEADER: :tangle ../.envrc-local +#+HEADER: :tangle ../.envrc-local_ +#+HEADER: :tangle-mode (identity #o644) +#+HEADER: :shebang "# -*- mode: sh -*-" #+HEADER: :noweb yes #+BEGIN_SRC shell export SpineVER=$(git describe --long --tags | sed 's/^[ a-z_-]\+\([0-9.]\+\)/\1/;s/\([^-]*-g\)/r\1/;s/-/./g') -export SpineSearchBIN=result/share/cgi-bin +export SpineSearchBIN=result/cgi-bin # ❯❯ cgi search form name: export SpineCGIform='spine_search' +# ❯❯ search form db name: +export SpineSQLdb='spine.search.db' # ❯❯ configuration cgi search form path: export SpineCGIbin=/var/www/cgi/cgi-bin +# ❯❯ configuration db path: +export SpineDBpath=/var/www/sqlite #+END_SRC -* versions GET +* editorconfig + +#+HEADER: :tangle "../.editorconfig" +#+BEGIN_SRC gitignore +# EditorConfig : https://editorconfig.org + +# top-most EditorConfig file +root = true + +# Unix-style newlines with a newline ending every file +[*] +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true + +# Matches multiple files with brace expansion notation +# Set default charset +[*.{org,d,nix}] +charset = utf-8 + +# Indentation override for all D under src directory +[src/sisudoc/**.d] +indent_style = space +indent_size = 2 + +# Tab indentation (no size specified) +[Makefile] +indent_style = tab + +[{dub.selections.json,dub_describe.json}] +indent_style = space +indent_size = 2 +#+END_SRC + +* org includes - versions GET ** direnv #+NAME: direnv_version @@ -156,14 +205,6 @@ export SpineCGIbin=/var/www/cgi/cgi-bin <<./nix-develop-dlang-shared.org:direnv-hash()>> #+END_SRC -** spine project VERSION :version:set:project: - -#+NAME: spine_version -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>> -#+END_SRC - ** dlang overlays *** ldc @@ -197,4 +238,26 @@ export SpineCGIbin=/var/www/cgi/cgi-bin <<./nix-develop-dlang-shared.org:dtools-version()>> #+END_SRC +** spine project VERSION :version:set:project: + +#+NAME: spine_version +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>> +#+END_SRC + +#+NAME: year +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>> +#+END_SRC + +** document header including copyright & license + +#+NAME: doc_header_including_copyright_and_license +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>> +#+END_SRC + * __END__ diff --git a/org/config_git.org b/org/config_git.org index 778dce1..0c82c4e 100644 --- a/org/config_git.org +++ b/org/config_git.org @@ -4,23 +4,24 @@ #+FILETAGS: :spine:hub: #+AUTHOR: Ralph Amissah #+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] -#+COPYRIGHT: Copyright (C) 2015 - 2025 Ralph Amissah +#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah #+LANGUAGE: en #+STARTUP: content hideblocks hidestars noindent entitiespretty -#+PROPERTY: header-args :exports code -#+PROPERTY: header-args+ :noweb yes -#+PROPERTY: header-args+ :results no -#+PROPERTY: header-args+ :cache no -#+PROPERTY: header-args+ :padline no +#+PROPERTY: header-args+ :eval never-export :exports code +#+PROPERTY: header-args+ :noweb yes :padline no +#+PROPERTY: header-args+ :results silent :cache no #+PROPERTY: header-args+ :mkdirp yes #+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t +- magic single double-quote → " ← FIX changes hilighting behavior (occuring + after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX -* spine +* git config spine search ** .gitignore :gitignore: #+HEADER: :tangle "../.gitignore" #+BEGIN_SRC gitignore # git ls-files --others --exclude-from=.git/info/exclude +# git ls-files --others --ignored --exclude-standard --directory # git check-ignore -v flake.lock # git clean -ix # find . -name "*_" | xargs rm -rf @@ -28,16 +29,14 @@ ,* !.gitignore !.gitattributes +!.editorconfig !nix/** !*.nix !flake.lock !nixDevEnv.sh !.envrc -!.envrc-local !.envrc-nix !.envrc-git-init -!.env -!.env/** !nix-overlays !nix-overlays/** !README.md @@ -54,6 +53,8 @@ !*.org !org !org/*.org +!*.md +!*.txt !*.d !*.rb !*.txt @@ -72,16 +73,17 @@ tmp/** ,*_.org ,*_.nix ,*_.d +,*_.md ,*_.txt ,*_ -,*.swp -,*~ -\#* -,*.\#* !.envrc-local_ !src/sisudoc/spine_search.d_ !views/configuration.txt_remotehost_ !views/configuration.txt_localhost_ +,*.swp +,*~ +\#* +,*.\#* #!*/ #\#* #*.\#* diff --git a/org/config_make.org b/org/config_make.org index 3aec827..f4a290f 100644 --- a/org/config_make.org +++ b/org/config_make.org @@ -4,15 +4,16 @@ #+FILETAGS: :spine:build:tools: #+AUTHOR: Ralph Amissah #+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] -#+COPYRIGHT: Copyright (C) 2015 - 2025 Ralph Amissah +#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah #+LANGUAGE: en #+STARTUP: content hideblocks hidestars noindent entitiespretty -#+PROPERTY: header-args :exports code -#+PROPERTY: header-args+ :noweb yes -#+PROPERTY: header-args+ :results no -#+PROPERTY: header-args+ :cache no -#+PROPERTY: header-args+ :padline no +#+PROPERTY: header-args+ :eval never-export :exports code +#+PROPERTY: header-args+ :noweb yes :padline no +#+PROPERTY: header-args+ :results silent :cache no #+PROPERTY: header-args+ :mkdirp yes +#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t +- magic single double-quote → " ← FIX changes hilighting behavior (occuring + after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX - [[./doc-reform.org][doc-reform.org]] [[../org/][org/]] @@ -134,7 +135,7 @@ set_depends: get_depends # cp cgi search form cp-cgi: sudo cp -vi /var/www/cgi/cgi-bin/spine_search /var/www/cgi/cgi-bin/spine_search_old && \ - sudo cp -vi result/share/cgi-bin/spine_search /var/www/cgi/cgi-bin/spine_search + sudo cp -vi result/cgi-bin/spine_search /var/www/cgi/cgi-bin/spine_search # git archive gitZip: diff --git a/org/config_nix.org b/org/config_nix.org index 195114a..365757e 100644 --- a/org/config_nix.org +++ b/org/config_nix.org @@ -4,16 +4,16 @@ #+FILETAGS: :spine:build:tools: #+AUTHOR: Ralph Amissah #+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] -#+COPYRIGHT: Copyright (C) 2015 - 2025 Ralph Amissah +#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah #+LANGUAGE: en -#+STARTUP: show4levels hideblocks hidestars noindent entitiespretty -#+PROPERTY: header-args :exports code -#+PROPERTY: header-args+ :noweb yes -#+PROPERTY: header-args+ :results no -#+PROPERTY: header-args+ :cache no -#+PROPERTY: header-args+ :padline no +#+STARTUP: content hideblocks hidestars noindent entitiespretty +#+PROPERTY: header-args+ :eval never-export :exports code +#+PROPERTY: header-args+ :noweb yes :padline no +#+PROPERTY: header-args+ :results silent :cache no #+PROPERTY: header-args+ :mkdirp yes #+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t +- magic single double-quote → " ← FIX changes hilighting behavior (occuring + after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX * nix :nix: ** flake.nix :flake: @@ -24,35 +24,28 @@ { description = "a sisu like document parser search form"; inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; - #inputs.flake-utils.url = "github:numtide/flake-utils"; inputs.spine-d-overlay = { url = "<<sisudoc_dlang_nix_overlay_src>>"; inputs.nixpkgs.follows = "nixpkgs"; - flake = true; }; outputs = { self, nixpkgs, - #flake-utils, spine-d-overlay, ... }@inputs: let pname = "spine_search"; version = "<<spine_version>>"; - shell = ./shell.nix; - devEnv = ./nixDevEnv.sh; + #shell = ./shell.nix; + #devEnv = ./nixDevEnv.sh; supportedSystems = ["x86_64-linux"]; forAllSystems = nixpkgs.lib.genAttrs supportedSystems; nixpkgsFor = forAllSystems (system: import nixpkgs {inherit system;}); # nixpkgs instantiated for supported system types - checkPhase = '' - runHook preCheck - dub test --combined --skip-registry=all - runHook postCheck - ''; localOverlay = (final: prev: { - ldc = prev.callPackage ./nix-overlays/ldc { }; - dmd = prev.callPackage ./nix-overlays/dmd { }; - dub = prev.callPackage ./nix-overlays/dub { }; + ldc = prev.callPackage ./nix-overlays/ldc { }; # -> ok <<ldc_version>> + # dmd overlay: revert to GCC14; (current) GCC 15 introduced nullptr in its headers, and DMD's ImportC parser needs update to handle it, monitor and update: + dmd = prev.callPackage ./nix-overlays/dmd { stdenv = prev.gcc14Stdenv; }; # -> ok <<dmd_version>> + dub = prev.callPackage ./nix-overlays/dub { }; # -> ? <<dub_version>> #gdc = prev.callPackage ./nix-overlays/gdc { }; }); pkgsForSystem = system: import nixpkgs { @@ -64,165 +57,79 @@ preBuild = '' export HOME=$(pwd) ''; + checkPhase = '' + runHook preCheck + dub test --combined --skip-registry=all + runHook postCheck + ''; installPhase = '' runHook preInstall - mkdir -p $out/share/cgi-bin - install -m755 -D ./cgi-bin/spine_search $out/share/cgi-bin/spine_search + mkdir -p $out/cgi-bin + install -m755 -D ./cgi-bin/spine_search $out/cgi-bin/spine_search runHook postInstall ''; postInstall = '' - echo `ls -la $out/share/cgi-bin/spine_search` + echo `ls -la $out/cgi-bin/spine_search` ''; + # Helper to reduce duplication across build variants + mkSpineSearch = {pkgs, compiler, compilerCmd, dubConfig}: pkgs.stdenv.mkDerivation { + inherit pname version; + meta.mainProgram = "spine_search"; + src = self; + buildInputs = [pkgs.sqlite]; + nativeBuildInputs = [pkgs.dub compiler pkgs.gnumake]; + buildPhase = '' + runHook preBuild + buildCMD="dub run --cache=local --compiler=$(type -P ${compilerCmd}) --build=${dubConfig} --combined --skip-registry=all" + echo $buildCMD + $buildCMD + echo $buildCMD + runHook postBuild + ''; + inherit preBuild; + inherit checkPhase; + inherit installPhase; + inherit postInstall; + }; in { packages = forAllSystems (system: let pkgs-ovl = pkgsForSystem system; pkgs-nix = nixpkgsFor.${system}; - in - with pkgs-nix; { - default = stdenv.mkDerivation { - inherit pname; - inherit version; - meta.mainProgram = "spine-search"; - executable = true; - src = self; - inherit shell; - inherit devEnv; - buildInputs = [sqlite]; - nativeBuildInputs = [dub ldc gnumake]; - buildPhase = '' - runHook preBuild - for DC_ in dmd ldmd2 ldc2 gdc gdmd; do - echo "- check for D compiler $DC_" - DC=$(type -P $DC_ || echo "") - if [ ! "$DC" == "" ]; then break; fi - done - if [ "$DC" == "" ]; then exit "Error: could not find D compiler"; fi - echo "$DC_ used as D compiler to build $pname" - buildCMD="dub run --cache=local --compiler=$DC --build=$DC_ --combined --skip-registry=all" - echo $buildCMD - $buildCMD - echo $buildCMD - runHook postBuild - ''; - inherit preBuild; - inherit checkPhase; - inherit installPhase; - inherit postInstall; - }; - spine-search-nixpkgs-dmd = stdenv.mkDerivation { - inherit pname; - inherit version; - meta.mainProgram = "spine_search-dmd"; - executable = true; - src = self; - inherit shell; - inherit devEnv; - buildInputs = with pkgs-nix; [sqlite]; - nativeBuildInputs = with pkgs-nix; [dub dmd gnumake]; - buildPhase = '' - runHook preBuild - buildCMD="dub run --cache=local --compiler=$(type -P dmd) --build=dmd --combined --skip-registry=all" - echo $buildCMD - $buildCMD - echo $buildCMD - runHook postBuild - ''; - inherit preBuild; - inherit checkPhase; - inherit installPhase; - inherit postInstall; - }; - spine-search-nixpkgs-ldc = stdenv.mkDerivation { - inherit pname; - inherit version; - meta.mainProgram = "spine_search-ldc"; - executable = true; - src = self; - inherit shell; - inherit devEnv; - buildInputs = with pkgs-nix; [sqlite]; - nativeBuildInputs = with pkgs-nix; [dub ldc gnumake]; - buildPhase = '' - runHook preBuild - buildCMD="dub run --cache=local --compiler=$(type -P ldmd2) --build=ldmd2 --combined --skip-registry=all" - echo $buildCMD - $buildCMD - echo $buildCMD - runHook postBuild - ''; - inherit preBuild; - inherit checkPhase; - inherit installPhase; - inherit postInstall; - }; - spine-search-overlay-dmd = stdenv.mkDerivation { - inherit pname; - inherit version; - meta.mainProgram = "spine_search-dmd"; - executable = true; - src = self; - inherit shell; - inherit devEnv; - buildInputs = with pkgs-ovl; [sqlite]; - nativeBuildInputs = with pkgs-ovl; [dub dmd gnumake]; - buildPhase = '' - runHook preBuild - buildCMD="dub run --cache=local --compiler=$(type -P dmd) --build=dmd --combined --skip-registry=all" - echo $buildCMD - $buildCMD - echo $buildCMD - runHook postBuild - ''; - inherit preBuild; - inherit checkPhase; - inherit installPhase; - inherit postInstall; - }; - spine-search-overlay-ldc = stdenv.mkDerivation { - inherit pname; - inherit version; - meta.mainProgram = "spine_search-ldc"; - executable = true; - src = self; - inherit shell; - inherit devEnv; - buildInputs = with pkgs-ovl; [sqlite]; - nativeBuildInputs = with pkgs-ovl; [dub ldc gnumake]; - buildPhase = '' - runHook preBuild - buildCMD="dub run --cache=local --compiler=$(type -P ldmd2) --build=ldmd2 --combined --skip-registry=all" - echo $buildCMD - $buildCMD - echo $buildCMD - runHook postBuild - ''; - inherit preBuild; - inherit checkPhase; - inherit installPhase; - inherit postInstall; - }; - #spine-search-overlay-gdc = stdenv.mkDerivation { - # inherit pname; - # inherit version; - # meta.mainProgram = "spine_search-gdc"; - # executable = true; - # src = self; - # inherit shell; - # inherit devEnv; - # buildInputs = with pkgs-ovl; [ sqlite ]; - # nativeBuildInputs = with pkgs-ovl; [ dub gdc gnumake ]; - # buildPhase = '' - # runHook preBuild - # dub run --cache=local --compiler=$(type -P gdc) --build=gdc --combined --skip-registry=all - # runHook postBuild - # ''; - # inherit preBuild; - # inherit checkPhase; - # inherit installPhase; - # inherit postInstall; - #}; - #vendorSha256 = "sha256-0Q00000000000000000000000000000000000000000="; - }); + in { + default = self.packages.${system}.spine-search-nixpkgs-ldc; + spine-search-nixpkgs-dmd = mkSpineSearch { + pkgs = pkgs-nix; + # nixpkgs dmd: revert to GCC14 (same nullptr/ImportC fix as overlay dmd) + compiler = pkgs-nix.dmd.override { stdenv = pkgs-nix.gcc14Stdenv; }; + #compiler = pkgs-nix.dmd; + compilerCmd = "dmd"; + dubConfig = "dmd"; + }; + spine-search-nixpkgs-ldc = mkSpineSearch { + pkgs = pkgs-nix; + compiler = pkgs-nix.ldc; + compilerCmd = "ldmd2"; + dubConfig = "ldmd2"; + }; + spine-search-overlay-dmd = mkSpineSearch { + pkgs = pkgs-ovl; + compiler = pkgs-ovl.dmd; + compilerCmd = "dmd"; + dubConfig = "dmd"; + }; + spine-search-overlay-ldc = mkSpineSearch { + pkgs = pkgs-ovl; + compiler = pkgs-ovl.ldc; + compilerCmd = "ldmd2"; + dubConfig = "ldmd2"; + }; + #spine-search-overlay-gdc = mkSpineSearch { + # pkgs = pkgs-ovl; + # compiler = pkgs-ovl.gdc; + # compilerCmd = "gdc"; + # dubConfig = "gdc"; + #}; + }); devShells = forAllSystems (system: let pkgs-ovl = pkgsForSystem system; pkgs-nix = nixpkgsFor.${system}; @@ -234,9 +141,7 @@ with pkgs-nix; { dsh-d-overlay = spine-d-overlay.devShells.${system}.default; dsh-overlay = mkShell { - name = "spine-<<spine_version>> base dev shell, ldc-<<ldc_version>>, dub-<<dub_version>>"; - inherit shell; - inherit devEnv; + name = "${pname}-${version} dev shell (overlay)"; packages = with pkgs-ovl; [ ldc #dmd @@ -246,10 +151,8 @@ ]; inherit shellHook; }; - dsh-nixpkgs-dmd-dub = mkShell { - name = "spine-<<spine_version>> base dev shell"; - inherit shell; - inherit devEnv; + dsh-nixpkgs-dmd = mkShell { + name = "${pname}-${version} dev shell (nixpkgs dmd)"; packages = [ dmd dub @@ -258,10 +161,8 @@ ]; inherit shellHook; }; - dsh-nixpkgs-ldc-dub = mkShell { - name = "spine-<<spine_version>> base dev shell"; - inherit shell; - inherit devEnv; + dsh-nixpkgs-ldc = mkShell { + name = "${pname}-${version} dev shell (nixpkgs ldc)"; packages = [ ldc dub @@ -270,10 +171,9 @@ ]; inherit shellHook; }; - dsh-overlay-dmd-dub = mkShell { - name = "spine-<<spine_version>> base dev shell, dmd-<<dmd_version>>, dub-<<dub_version>>"; - inherit shell; - inherit devEnv; + dsh-overlay-dmd = mkShell { + name = "${pname}-${version} dev shell (overlay dmd)"; + # "spine-<<spine_version>> base dev shell, dmd-<<dmd_version>>, dub-<<dub_version>>"; packages = with pkgs-ovl; [ dmd dub @@ -282,10 +182,9 @@ ]; inherit shellHook; }; - dsh-overlay-ldc-dub = mkShell { - name = "spine-<<spine_version>> base dev shell, ldc-<<ldc_version>>, dub-<<dub_version>>"; - inherit shell; - inherit devEnv; + dsh-overlay-ldc = mkShell { + name = "${pname}-${version} dev shell (overlay ldc)"; + # "spine-<<spine_version>> base dev shell, ldc-<<ldc_version>>, dub-<<dub_version>>"; packages = with pkgs-ovl; [ ldc dub @@ -295,9 +194,7 @@ inherit shellHook; }; dsh-sqlite = mkShell { - name = "spine dev shell for latex & pdf output"; - inherit shell; - inherit devEnv; + name = "${pname}-${version} dev shell (sqlite)"; packages = [ ldc #dmd @@ -307,7 +204,8 @@ ]; inherit shellHook; }; - default = import ./shell.nix {inherit pkgs-nix;}; + default = self.devShells.${system}.dsh-nixpkgs-ldc; + #default = import ./shell.nix {pkgs = pkgs-nix;}; }); }; } @@ -316,10 +214,75 @@ ** shell.nix :shell: #+HEADER: :tangle ../shell.nix -#+HEADER: :tangle-mode (identity #o755) -#+HEADER: :shebang "#!/usr/bin/env -S nix-shell --pure\n#!nix-shell -i bash" #+HEADER: :noweb yes #+BEGIN_SRC nix +#!/usr/bin/env -S nix-shell --pure +#!nix-shell -i bash +# Development environment for sisudoc-spine-search-cgi. +# Builds the spine_search CGI binary via ./package.nix. +# +# Build logic for the spine_search derivation lives in ./package.nix - +# this file only describes the dev shell. +# +# Usage: +# nix-shell # enters shell, builds spine_search +# nix-shell --run 'echo $SPINE_SEARCH_BIN' +{ + pkgs ? import <nixpkgs> {}, + spine-search ? pkgs.callPackage ./package.nix {}, + # captured at eval time - nix-shell rewrites $SHELL at runtime, + # so we must remember the user's real login shell here. + userShell ? builtins.getEnv "SHELL", +}: +with pkgs; + mkShell { + name = "spine_search base dev shell"; + packages = [ + # ❯❯❯ spine_search CGI binary built from ./package.nix + spine-search + # ❯❯❯ d_build_related + ldc + #dmd + dub + # ❯❯❯ dev + gnumake + git + # ❯❯❯ nix workflow + direnv + nix-direnv + nix-prefetch-git + nix-output-monitor + nix-tree + jq + # ❯❯❯ sqlite + sqlite + ]; + shellHook = '' + export Date=$(date "+%Y%m%d") + ## set local values in .envrc-local (or here if you must) + ## spine_search lives under cgi-bin, not bin - expose it via env + ## var and prepend cgi-bin to PATH for convenience. + export SPINE_SEARCH_BIN="${spine-search}/cgi-bin/spine_search" + export PATH="${spine-search}/cgi-bin:$PATH" + echo "spine_search: $SPINE_SEARCH_BIN" + ## hand off to the user's login shell (e.g. zsh) only when this + ## is an interactive nix-shell entry - not under `nix-shell --run` + ## or `--command`, where exec would swallow the command. Guard env + ## var prevents re-entry loops. + __spine_user_shell=${if userShell == "" then "" else "\"" + userShell + "\""} + if [ -z "$__SPINE_SEARCH_SHELL_HANDOFF" ] \ + && [ -n "$__spine_user_shell" ] \ + && [ "$(basename "$__spine_user_shell")" != "bash" ] \ + && [[ $- == *i* ]]; then + export __SPINE_SEARCH_SHELL_HANDOFF=1 + exec "$__spine_user_shell" + fi + unset __spine_user_shell + ''; + } +#+END_SRC + +#+BEGIN_SRC nix {pkgs-nix ? import <nixpkgs> {}}: with pkgs-nix; mkShell { @@ -327,6 +290,7 @@ with pkgs-nix; # ❯❯❯ nix_related #nix direnv + nix-direnv nixVersions.latest #nixVersions.latest #nixVersions.git nix-prefetch-git validatePkgConfig @@ -363,80 +327,90 @@ with pkgs-nix; echo "ldc `${pkgs.ldc}/bin/ldc2 -v`" -** default.nix :default: - -#+HEADER: :tangle ../default.nix -#+HEADER: :tangle-mode (identity #o755) -#+HEADER: :shebang "#!/usr/bin/env -S nix-build" -#+BEGIN_SRC nix -{pkgs ? import <nixpkgs> {}}: -pkgs.callPackage ./devenv.nix {} -#+END_SRC - -#+HEADER: :tangle-mode (identity #o755) -#+HEADER: :shebang "#!/usr/bin/env -S nix-build" -#+BEGIN_SRC nix -{ pkgs ? import <nixpkgs> {} }: -pkgs.callPackage ./derivation.nix {} -#+END_SRC - -*** devenv.nix :default:devenv: +** package.nix :default: -#+HEADER: :tangle ../devenv.nix +#+HEADER: :tangle ../package.nix #+HEADER: :tangle-mode (identity #o644) +#+HEADER: :noweb yes #+BEGIN_SRC nix +# package.nix - spine_search CGI derivation (build logic) +# +# Standalone, callPackage-style derivation for the spine_search CGI +# binary. Used by shell.nix to put a freshly-built spine_search into +# the dev shell. May also be consumed via: +# nix-build ./package.nix +# pkgs.callPackage ./package.nix {} +# +# Compiler defaults to ldc/ldmd2 (matching the flake's default +# package `spine-search-nixpkgs-ldc`). Override to build with dmd: +# pkgs.callPackage ./package.nix { +# compilerPkg = pkgs.dmd; +# compilerBin = "dmd"; +# buildType = "dmd"; +# } +# +# Note: spine_search is a CGI binary and is installed to +# $out/cgi-bin/spine_search (not $out/bin/), since it is intended to +# be served by a web server, not invoked directly from PATH. { - pkgs ? import <nixpkgs> {}, - name ? "user-env", + lib, + stdenv, + dub, + ldc, + gnumake, + sqlite, + compilerPkg ? ldc, + compilerBin ? "ldmd2", + buildType ? "ldmd2", }: -with pkgs; - buildEnv { - inherit name; - extraOutputsToInstall = ["out" "man" "lib"]; # to get all needed symlinks - paths = [ - # ❯❯❯ nix_related - #nix # if not on NixOS, this is needed - direnv - nixVersions.latest #nixVersions.latest #nixVersions.git - nix-prefetch-git - validatePkgConfig - nix-output-monitor - nix-tree - jq #gx - #nixfmt-rfc-style - git - # ❯❯❯ dev - gnumake - ps - # ❯❯❯ d_build_related - # ❯❯ package manager - dub - # ❯❯ compiler - #dmd - ldc - # ❯❯❯ sqlite search related - sqlite - # this will create a script that will rebuild and upgrade your setup; using shell script syntax - (writeScriptBin "nix-rebuild" '' - #!${stdenv.shell} - #cd <path-to-flake> || exit 1 - nix flake update - nix profile upgrade '.*' - '') - # puts in your root the nixpkgs version - (writeTextFile { - name = "nixpkgs-version"; - destination = "/nixpkgs-version"; - text = lib.version; - }) - ]; - } +stdenv.mkDerivation { + pname = "spine_search"; + version = "0.18.0"; + src = lib.cleanSource ./.; + buildInputs = [ sqlite ]; + nativeBuildInputs = [ dub compilerPkg gnumake ]; + preBuild = '' + export HOME=$(pwd) + ''; + buildPhase = '' + runHook preBuild + buildCMD="dub run --cache=local \ + --compiler=$(type -P ${compilerBin}) \ + --build=${buildType} \ + --combined --skip-registry=all" + echo $buildCMD + $buildCMD + runHook postBuild + ''; + checkPhase = '' + runHook preCheck + dub test --combined --skip-registry=all + runHook postCheck + ''; + installPhase = '' + runHook preInstall + mkdir -p $out/cgi-bin + install -m755 -D ./cgi-bin/spine_search $out/cgi-bin/spine_search + runHook postInstall + ''; + postInstall = '' + echo `ls -la $out/cgi-bin/spine_search` + ''; + meta = { + description = "CGI search interface for spine document collections"; + homepage = "https://sisudoc.org"; + license = lib.licenses.agpl3Plus; + platforms = lib.platforms.linux; + mainProgram = "spine_search"; + }; +} #+END_SRC ** derivation.nix :default: #+HEADER: :tangle ../derivation.nix -#+HEADER: :tangle-mode (identity #o644) +#+HEADER: :tangle-mode (identity #o755) +#+HEADER: :shebang "#!/usr/bin/env -S nix-build derivation.nix" #+HEADER: :noweb yes #+BEGIN_SRC nix { @@ -458,7 +432,7 @@ with ( ); targetOf = package: "${package.targetPath or "."}/${package.targetName or package.name}"; # remove reference to build tools and library sources - disallowedReferences = deps: [dcompiler dub]; + disallowedRefs = [dcompiler dub]; removeExpr = refs: ''remove-references-to ${lib.concatMapStrings (ref: " -t ${ref}") refs}''; in { mkDubDerivation = lib.makeOverridable ({ @@ -471,9 +445,9 @@ with ( } @ attrs: stdenv.mkDerivation (attrs // { - pname = package.name; + pname = attrs.pname or package.name; nativeBuildInputs = [dcompiler dub pkgs.removeReferencesTo] ++ nativeBuildInputs; - disallowedReferences = disallowedReferences deps; + disallowedReferences = disallowedRefs; passthru = passthru // { @@ -484,7 +458,7 @@ with ( src = lib.cleanSource src; }; preFixup = '' - find $out/share/cgi-bin -type f -exec ${removeExpr (disallowedReferences deps)} '{}' + || true + find $out/cgi-bin -type f -exec ${removeExpr disallowedRefs} '{}' + || true ''; buildPhase = '' runHook preBuild @@ -498,7 +472,8 @@ with ( fi done if [ "$DC" == "" ]; then - exit "Error: could not find D compiler" + echo "Error: could not find D compiler" >&2 + exit 1 fi echo "$DC_ used as D compiler to build $pname" dub run --compiler=$DC --build=release --combined --skip-registry=all @@ -512,14 +487,12 @@ with ( ''; installPhase = '' runHook preInstall - mkdir -p $out/share/cgi-bin - cp -r "${targetOf package}" $out/share/cgi-bin - install -m755 -D $out/share/cgi-bin/spine_search spine_search + mkdir -p $out/cgi-bin + install -m755 -D "${targetOf package}" "$out/cgi-bin/spine_search" runHook postInstall ''; postInstall = '' - echo "HERE ${targetOf package} $out/share/cgi-bin" - echo `ls -la $out/share/cgi-bin/spine_search` + echo `ls -la $out/cgi-bin/spine_search` ''; meta = lib.optionalAttrs (package ? description) { @@ -533,35 +506,33 @@ with ( } ); mkDubDerivation rec { - name = "spine-search-<<spine_version>>"; - #name = "spine-search-${version}"; + pname = "spine-search"; + version = "<<spine_version>>"; src = ./.; buildInputs = [ pkgs.sqlite ( - with pkgs-nix; [ + with pkgs; [ nixVersions.latest #nixVersions.latest #nixVersions.git ## package manager dub ## compiler ldc - rund + #rund sqlite ] ) ]; - meta = with pkgs-nix.lib; { - pname = "spine-search"; - version = "<<spine_version>>"; + meta = with pkgs.lib; { homepage = "https://sisudoc.org"; description = "cgi sqlite search form for document object search"; longDescription = '' a sisu like parser & document generator ''; - homepage = "https://sisudoc.org"; license = licenses.agpl3Plus; platforms = platforms.linux; maintainers = ["RalphAmissah"]; + mainProgram = "spine_search"; }; } #+END_SRC diff --git a/org/nix-develop-dlang-shared.org b/org/nix-develop-dlang-shared.org index 671fa8d..e971677 100644 --- a/org/nix-develop-dlang-shared.org +++ b/org/nix-develop-dlang-shared.org @@ -4,16 +4,16 @@ #+FILETAGS: :dlang:develop:shared: #+AUTHOR: Ralph Amissah #+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] -#+COPYRIGHT: Copyright (C) 2015 - 2025 Ralph Amissah +#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah #+LANGUAGE: en #+STARTUP: content hideblocks hidestars noindent entitiespretty -#+PROPERTY: header-args :noweb yes -#+PROPERTY: header-args+ :exports code -#+PROPERTY: header-args+ :results no -#+PROPERTY: header-args+ :cache no -#+PROPERTY: header-args+ :padline no +#+PROPERTY: header-args+ :eval never-export :exports code +#+PROPERTY: header-args+ :noweb yes :padline no +#+PROPERTY: header-args+ :results silent :cache no #+PROPERTY: header-args+ :mkdirp yes #+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t +- magic single double-quote → " ← FIX changes hilighting behavior (occuring + after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX * nix ** direnv @@ -27,6 +27,7 @@ check for latest version: update direnv version and sha hash: - emacs org/nix-develop-dlang-shared.org +- direnv fetchurl https://raw.githubusercontent.com/nix-community/nix-direnv/3.1.0/direnvrc re-tangle file: @@ -40,74 +41,26 @@ update flake.lock: #+NAME: direnv-version #+BEGIN_SRC org -<<direnv_version_3_0_7>> +<<direnv_version_3_1_0>> #+END_SRC #+NAME: direnv-hash #+BEGIN_SRC org -<<direnv_hash_3_0_7>> +<<direnv_hash_3_1_0>> #+END_SRC *** versions -**** 3.0.7 +**** 3.1.0 -#+NAME: direnv_version_3_0_7 +#+NAME: direnv_version_3_1_0 #+BEGIN_SRC org -3.0.7 +3.1.0 #+END_SRC -#+NAME: direnv_hash_3_0_7 +#+NAME: direnv_hash_3_1_0 #+BEGIN_SRC org -sha256-bn8WANE5a91RusFmRI7kS751ApelG02nMcwRekC/qzc= -#+END_SRC - -**** 3.0.6 - -#+NAME: direnv_version_3_0_6 -#+BEGIN_SRC org -3.0.6 -#+END_SRC - -#+NAME: direnv_hash_3_0_6 -#+BEGIN_SRC org -sha256-RYcUJaRMf8oF5LznDrlCXbkOQrywm0HDv1VjYGaJGdM= -#+END_SRC - -**** 3.0.4 - -#+NAME: direnv_version_3_0_4 -#+BEGIN_SRC org -3.0.4 -#+END_SRC - -#+NAME: direnv_hash_3_0_4 -#+BEGIN_SRC org -sha256-DzlYZ33mWF/Gs8DDeyjr8mnVmQGx7ASYqA5WlxwvBG4= -#+END_SRC - -**** 3.0.0 - -#+NAME: direnv_version_3_0_0 -#+BEGIN_SRC org -3.0.0 -#+END_SRC - -#+NAME: direnv_hash_3_0_0 -#+BEGIN_SRC org -sha256-21TMnI2xWX7HkSTjFFri2UaohXVj854mgvWapWrxRXg= -#+END_SRC - -**** 2.4.0 - -#+NAME: direnv_version_2_4_0 -#+BEGIN_SRC org -2.4.0 -#+END_SRC - -#+NAME: direnv_hash_2_4_0 -#+BEGIN_SRC org -sha256-XQzUAvL6pysIJnRJyR7uVpmUSZfc7LSgWQwq/4mBr1U= +sha256-yMJ2OVMzrFaDPn7q8nCBZFRYpL/f0RcHzhmw/i6btJM= #+END_SRC * dlang @@ -159,20 +112,41 @@ sha256 = "<<ldc-hash>>"; #+NAME: ldc-version #+BEGIN_SRC org -<<ldc_version_1_41_0>> +<<ldc_version_1_42_0>> #+END_SRC #+NAME: ldc-hash #+BEGIN_SRC org -<<ldc_hash_1_41_0>> +<<ldc_hash_1_42_0>> #+END_SRC #+NAME: ldc-llvm-set #+BEGIN_SRC org -<<ldc_llvm_set_1_41_0>> +<<ldc_llvm_set_1_42_0>> #+END_SRC *** versions +**** 1.42 OK ✓ +***** 1.42.0 OK ✓ + + - nix-prefetch-url --unpack https://github.com/ldc-developers/ldc/archive/refs/tags/v1.42.0.tar.gz + - nix-prefetch-url --unpack https://github.com/ldc-developers/ldc/releases/download/v${version}/ldc-${version}-src.tar.gz + +#+NAME: ldc_version_1_42_0 +#+BEGIN_SRC nix +1.42.0 +#+END_SRC + +#+NAME: ldc_hash_1_42_0 +#+BEGIN_SRC nix +sha256-adA14tx/bruGvHVoODz13f8h/U2ol1lK0ytxnypsLv8= +#+END_SRC + +#+NAME: ldc_llvm_set_1_42_0 +#+BEGIN_SRC nix +llvm_21 +#+END_SRC + **** 1.41 OK ✓ ***** 1.41.0 OK ✓ diff --git a/org/nixpkgs_overlays_d_related.org b/org/nixpkgs_overlays_d_related.org index f01c362..727e136 100644 --- a/org/nixpkgs_overlays_d_related.org +++ b/org/nixpkgs_overlays_d_related.org @@ -7,14 +7,14 @@ #+COPYRIGHT: Copyright (C) 2020 Ralph Amissah #+LICENSE: Boost Software License 1.0 #+LANGUAGE: en -#+STARTUP: show3levels hideblocks hidestars noindent entitiespretty -#+PROPERTY: header-args :exports code -#+PROPERTY: header-args+ :noweb yes -#+PROPERTY: header-args+ :results no -#+PROPERTY: header-args+ :cache no -#+PROPERTY: header-args+ :padline no +#+STARTUP: content hideblocks hidestars noindent entitiespretty +#+PROPERTY: header-args+ :eval never-export :exports code +#+PROPERTY: header-args+ :noweb yes :padline no +#+PROPERTY: header-args+ :results silent :cache no #+PROPERTY: header-args+ :mkdirp yes #+OPTIONS: H:3 num:nil toc:t \n:nil @:t ::t |:t ^:nil _:nil -:t f:t *:t <:t +- magic single double-quote → " ← FIX changes hilighting behavior (occuring + after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX * nix ( written to ./dlang-nix-flakes/ ) ** check updates CHECK @@ -39,7 +39,7 @@ ,#+FILETAGS: :dlang:build:tools: ,#+AUTHOR: Ralph Amissah ,#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] -,#+COPYRIGHT: Copyright (C) 2015 - 2025 Ralph Amissah +,#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah ,#+LANGUAGE: en ,#+STARTUP: content hideblocks hidestars noindent entitiespretty @@ -351,7 +351,7 @@ use flake . ]; inherit shellHook; }; - dsh-overlay-ldc-dub = mkShell { + dsh-overlay-ldc = mkShell { name = "overlay - ldc-<<ldc_version>> - dub-<<dub_version>> - dtools-<<dtools_version>>"; inherit shell; inherit devEnv; @@ -363,7 +363,7 @@ use flake . ]; inherit shellHook; }; - dsh-overlay-dmd-dub = mkShell { + dsh-overlay-dmd = mkShell { name = "overlay - dmd-<<dmd_version>> - dub-<<dub_version>> - dtools-<<dtools_version>>"; inherit shell; inherit devEnv; @@ -401,7 +401,7 @@ use flake . # ]; # inherit shellHook; #}; - dsh-nixpkgs-ldc-dub = mkShell { + dsh-nixpkgs-ldc = mkShell { name = "nixpkgs - ldc - dub"; inherit shell; inherit devEnv; @@ -413,7 +413,7 @@ use flake . ]; inherit shellHook; }; - dsh-nixpkgs-dmd-dub = mkShell { + dsh-nixpkgs-dmd = mkShell { name = "nixpkgs - ldc - dub"; inherit shell; inherit devEnv; @@ -649,6 +649,8 @@ stdenv.mkDerivation (finalAttrs: { find $out/bin -type f -exec ${removeReferencesTo}/bin/remove-references-to -t ${ldcBootstrap} '{}' + ''; + # doCheck = false; + disallowedReferences = [ ldcBootstrap ]; meta = with lib; { @@ -692,8 +694,8 @@ stdenv.mkDerivation (finalAttrs: { no-references-to-compiler = helloWorld.overrideAttrs { disallowedReferences = [ ldc ]; dFlags = ["-g"]; + }; }; - }; }) #+END_SRC @@ -1399,7 +1401,7 @@ stdenv.mkDerivation (finalAttrs: { @@ -616,7 +616,7 @@ void runTests(string rdmdApp, string compiler, string model) enforce(res.status == 1, res.output); } - + - version (Posix) + version (none) { @@ -1493,22 +1495,22 @@ Adapted from https://github.com/dlang/tools/commit/6c6a042d1b08e3ec1790bd07a7f69 @@ -9,9 +9,8 @@ DUB=dub WITH_DOC = no DOC = ../dlang.org - + -# Load operating system $(OS) (e.g. linux, osx, ...) and $(MODEL) (e.g. 32, 64) detection Makefile from dmd -$(shell [ ! -d $(DMD_DIR) ] && git clone --depth=1 https://github.com/dlang/dmd $(DMD_DIR)) -include $(DMD_DIR)/compiler/src/osmodel.mak +# Load operating system $(OS) (e.g. linux, osx, ...) and $(MODEL) (e.g. 32, 64) detection Makefile +include osmodel.mak - + ifeq (windows,$(OS)) DOTEXE:=.exe @@ -30,7 +29,7 @@ DFLAGS = $(MODEL_FLAG) $(if $(findstring windows,$(OS)),,-fPIC) -preview=dip1000 DFLAGS += $(WARNINGS) - + # Default DUB flags (DUB uses a different architecture format) -DUBFLAGS = --arch=$(subst 32,x86,$(subst 64,x86_64,$(MODEL))) +DUBFLAGS = --arch=$(ARCH) - + TOOLS = \ $(ROOT)/catdoc$(DOTEXE) \ #+END_SRC diff --git a/org/sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org b/org/sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org index b536386..b2a6fec 100644 --- a/org/sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org +++ b/org/sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org @@ -4,16 +4,16 @@ #+FILETAGS: :spine:abstraction: #+AUTHOR: Ralph Amissah #+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] -#+COPYRIGHT: Copyright (C) 2015 - 2025 Ralph Amissah +#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah #+LANGUAGE: en #+STARTUP: content hideblocks hidestars noindent entitiespretty -#+PROPERTY: header-args :noweb yes -#+PROPERTY: header-args+ :exports code -#+PROPERTY: header-args+ :results no -#+PROPERTY: header-args+ :cache no -#+PROPERTY: header-args+ :padline no +#+PROPERTY: header-args+ :eval never-export :exports code +#+PROPERTY: header-args+ :noweb yes :padline no +#+PROPERTY: header-args+ :results silent :cache no #+PROPERTY: header-args+ :mkdirp yes #+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t +- magic single double-quote → " ← FIX changes hilighting behavior (occuring + after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX - [[./doc-reform.org][doc-reform.org]] [[./][org/]] @@ -29,7 +29,7 @@ - Author: Ralph Amissah [ralph.amissah@gmail.com] - - Copyright: (C) 2015 - 2025 Ralph Amissah, All Rights Reserved. + - Copyright: (C) 2015 (continuously updated, current 2026) Ralph Amissah, All Rights Reserved. - License: AGPL 3 or later: @@ -108,6 +108,13 @@ 0 #+END_SRC +* year + +#+NAME: year +#+BEGIN_SRC org +2026 +#+END_SRC + * sisudoc spine git source SET ** sisudoc-spine src diff --git a/org/spine_info.org b/org/spine_info.org index 9a61095..fbff7bb 100644 --- a/org/spine_info.org +++ b/org/spine_info.org @@ -4,16 +4,16 @@ #+FILETAGS: :spine:info: #+AUTHOR: Ralph Amissah #+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] -#+COPYRIGHT: Copyright (C) 2015 - 2025 Ralph Amissah +#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah #+LANGUAGE: en #+STARTUP: content hideblocks hidestars noindent entitiespretty -#+PROPERTY: header-args :exports code -#+PROPERTY: header-args+ :noweb yes -#+PROPERTY: header-args+ :results no -#+PROPERTY: header-args+ :cache no -#+PROPERTY: header-args+ :padline no +#+PROPERTY: header-args+ :eval never-export :exports code +#+PROPERTY: header-args+ :noweb yes :padline no +#+PROPERTY: header-args+ :results silent :cache no #+PROPERTY: header-args+ :mkdirp yes #+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t +- magic single double-quote → " ← FIX changes hilighting behavior (occuring + after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX - [[./doc-reform.org][doc-reform.org]] [[./][org/]] - [[./spine_build_scaffold.org][spine_build_scaffold.org]] @@ -74,14 +74,14 @@ ,#+FILETAGS: :spine:build:tools: ,#+AUTHOR: Ralph Amissah ,#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] -,#+COPYRIGHT: Copyright (C) 2015 - 2025 Ralph Amissah +,#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah ,#+LANGUAGE: en ,#+STARTUP: content hideblocks hidestars noindent entitiespretty ,#+OPTIONS: H:3 num:nil toc:t \n:nil @:t ::t |:t ^:nil _:nil -:t f:t *:t <:t ,#+PROPERTY: header-args :exports code ,#+PROPERTY: header-args+ :noweb yes ,#+PROPERTY: header-args+ :eval no -,#+PROPERTY: header-args+ :results no +,#+PROPERTY: header-args+ :results silent ,#+PROPERTY: header-args+ :cache no ,#+PROPERTY: header-args+ :padline no #+END_SRC @@ -144,7 +144,7 @@ author: name: "Ralph Amissah" email: ralph.amissah@gmail.com -copyright: "(C) 2015 - 2025 Ralph Amissah, All Rights Reserved." +copyright: "(C) 2015 - 2026 Ralph Amissah, All Rights Reserved." license: - "project code: AGPL 3 or later" @@ -206,9 +206,9 @@ Make a directory and clone the sisudoc-spine project mkdir ~/git.sisudoc cd ~/git.sisudoc - git clone --depth=1 git://git.sisudoc.org/software/sisudoc-spine && \ - git clone --depth=1 git://git.sisudoc.org/software/sisudoc-spine-search-cgi && \ - git clone --depth=1 git://git.sisudoc.org/markup/sisudoc-spine-samples + git clone --depth 1 git://git.sisudoc.org/software/sisudoc-spine && \ + git clone --depth 1 git://git.sisudoc.org/software/sisudoc-spine-search-cgi && \ + git clone --depth 1 git://git.sisudoc.org/markup/sisudoc-spine-samples such a relative directory layout will be assumed in the examples that provided @@ -269,7 +269,7 @@ mkdir -p ~/.config/nix && echo "experimental-features = flakes nix-command" >> ~ #+BEGIN_SRC text - to use the D compiler ldc2: # on nix (get dependencies by setting your development environment): - nix develop ".#dsh-nixpkgs-ldc-dub" --print-build-logs -c zsh + nix develop ".#dsh-nixpkgs-ldc" --print-build-logs -c zsh # assuming you have ldc2 & dub installed on your system: dub run --compiler=ldmd2 --config=ldmd2 --combined --skip-registry=all @@ -278,7 +278,7 @@ mkdir -p ~/.config/nix && echo "experimental-features = flakes nix-command" >> ~ - to use the D compiler dmd: # on nix (get dependencies by setting your development environment): - nix develop ".#dsh-nixpkgs-dmd-dub" --print-build-logs -c zsh + nix develop ".#dsh-nixpkgs-dmd" --print-build-logs -c zsh # assuming you have dmd & dub installed on your system: dub run --compiler=dmd --config=dmd --combined --skip-registry=all @@ -325,7 +325,7 @@ web-server is configured to find it. ** notices *** project (project root) ./ -#+HEADER: :tangle "../COPYRIGHT" +#+HEADER: :tangle-NO "../COPYRIGHT" #+HEADER: :noweb yes #+BEGIN_SRC text - Name: spine - SiSU Spine, Doc Reform @@ -342,7 +342,7 @@ web-server is configured to find it. *** code source ./src -#+HEADER: :tangle "../src/COPYRIGHT" +#+HEADER: :tangle-NO "../src/COPYRIGHT" #+HEADER: :noweb yes #+BEGIN_SRC text - Name: spine - SiSU Spine, Doc Reform @@ -359,7 +359,7 @@ web-server is configured to find it. *** code source ./src/sisudoc -#+HEADER: :tangle "../src/sisudoc/COPYRIGHT" +#+HEADER: :tangle-NO "../src/sisudoc/COPYRIGHT" #+HEADER: :noweb yes #+BEGIN_SRC text - Name: spine - SiSU Spine, Doc Reform @@ -382,7 +382,7 @@ web-server is configured to find it. - license - dependency information -#+HEADER: :tangle "../org/COPYRIGHT" +#+HEADER: :tangle-NO "../org/COPYRIGHT" #+HEADER: :noweb yes #+BEGIN_SRC text - Name: spine - SiSU Spine, Doc Reform @@ -412,7 +412,7 @@ web-server is configured to find it. - Author: Ralph Amissah [ralph.amissah@gmail.com] -- Copyright: (C) 2015 - 2025 Ralph Amissah +- Copyright: (C) 2015 - 2026 Ralph Amissah #+END_SRC *** license diff --git a/org/spine_search_cgi.org b/org/spine_search_cgi.org index 59cc5ca..b298da2 100644 --- a/org/spine_search_cgi.org +++ b/org/spine_search_cgi.org @@ -4,23 +4,23 @@ #+FILETAGS: :spine:hub: #+AUTHOR: Ralph Amissah #+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] -#+COPYRIGHT: Copyright (C) 2015 - 2025 Ralph Amissah +#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah #+LANGUAGE: en #+STARTUP: content hideblocks hidestars noindent entitiespretty -#+PROPERTY: header-args :exports code -#+PROPERTY: header-args+ :noweb yes -#+PROPERTY: header-args+ :results no -#+PROPERTY: header-args+ :cache no -#+PROPERTY: header-args+ :padline no +#+PROPERTY: header-args+ :eval never-export :exports code +#+PROPERTY: header-args+ :noweb yes :padline no +#+PROPERTY: header-args+ :results silent :cache no #+PROPERTY: header-args+ :mkdirp yes #+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t +- magic single double-quote → " ← FIX changes hilighting behavior (occuring + after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX - [[./doc-reform.org][doc-reform.org]] [[./][org/]] - [[./spine_build_scaffold.org][make/build]] VERSION #+NAME: year -#+BEGIN_SRC text -2025 +#+BEGIN_SRC org +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>> #+END_SRC * 0. version.txt configuration.txt (set version & configuration) diff --git a/package.nix b/package.nix new file mode 100644 index 0000000..bbe23ca --- /dev/null +++ b/package.nix @@ -0,0 +1,71 @@ +# package.nix - spine_search CGI derivation (build logic) +# +# Standalone, callPackage-style derivation for the spine_search CGI +# binary. Used by shell.nix to put a freshly-built spine_search into +# the dev shell. May also be consumed via: +# nix-build ./package.nix +# pkgs.callPackage ./package.nix {} +# +# Compiler defaults to ldc/ldmd2 (matching the flake's default +# package `spine-search-nixpkgs-ldc`). Override to build with dmd: +# pkgs.callPackage ./package.nix { +# compilerPkg = pkgs.dmd; +# compilerBin = "dmd"; +# buildType = "dmd"; +# } +# +# Note: spine_search is a CGI binary and is installed to +# $out/cgi-bin/spine_search (not $out/bin/), since it is intended to +# be served by a web server, not invoked directly from PATH. +{ + lib, + stdenv, + dub, + ldc, + gnumake, + sqlite, + compilerPkg ? ldc, + compilerBin ? "ldmd2", + buildType ? "ldmd2", +}: +stdenv.mkDerivation { + pname = "spine_search"; + version = "0.18.0"; + src = lib.cleanSource ./.; + buildInputs = [ sqlite ]; + nativeBuildInputs = [ dub compilerPkg gnumake ]; + preBuild = '' + export HOME=$(pwd) + ''; + buildPhase = '' + runHook preBuild + buildCMD="dub run --cache=local \ + --compiler=$(type -P ${compilerBin}) \ + --build=${buildType} \ + --combined --skip-registry=all" + echo $buildCMD + $buildCMD + runHook postBuild + ''; + checkPhase = '' + runHook preCheck + dub test --combined --skip-registry=all + runHook postCheck + ''; + installPhase = '' + runHook preInstall + mkdir -p $out/cgi-bin + install -m755 -D ./cgi-bin/spine_search $out/cgi-bin/spine_search + runHook postInstall + ''; + postInstall = '' + echo `ls -la $out/cgi-bin/spine_search` + ''; + meta = { + description = "CGI search interface for spine document collections"; + homepage = "https://sisudoc.org"; + license = lib.licenses.agpl3Plus; + platforms = lib.platforms.linux; + mainProgram = "spine_search"; + }; +} @@ -1,41 +1,64 @@ #!/usr/bin/env -S nix-shell --pure #!nix-shell -i bash -{pkgs-nix ? import <nixpkgs> {}}: -with pkgs-nix; +# Development environment for sisudoc-spine-search-cgi. +# Builds the spine_search CGI binary via ./package.nix. +# +# Build logic for the spine_search derivation lives in ./package.nix - +# this file only describes the dev shell. +# +# Usage: +# nix-shell # enters shell, builds spine_search +# nix-shell --run 'echo $SPINE_SEARCH_BIN' +{ + pkgs ? import <nixpkgs> {}, + spine-search ? pkgs.callPackage ./package.nix {}, + # captured at eval time - nix-shell rewrites $SHELL at runtime, + # so we must remember the user's real login shell here. + userShell ? builtins.getEnv "SHELL", +}: +with pkgs; mkShell { - buildInputs = [ - # ❯❯❯ nix_related - #nix + name = "spine_search base dev shell"; + packages = [ + # ❯❯❯ spine_search CGI binary built from ./package.nix + spine-search + # ❯❯❯ d_build_related + ldc + #dmd + dub + # ❯❯❯ dev + gnumake + git + # ❯❯❯ nix workflow direnv - nixVersions.latest #nixVersions.latest #nixVersions.git + nix-direnv nix-prefetch-git - validatePkgConfig nix-output-monitor nix-tree - jq #gx - #nixfmt-rfc-style - git - # ❯❯❯ dev - gnumake - ps - # ❯❯❯ d_build_related - # ❯❯ package manager - #dub - # ❯❯ compiler - #dmd - #ldc - #rund - # ❯❯ linker - #lld - #mold - # ❯❯ builder - #ninja - #meson - # ❯❯❯ sqlite search related + jq + # ❯❯❯ sqlite sqlite - # ❯❯❯ tools - #aria #wget #curl ]; shellHook = '' + export Date=$(date "+%Y%m%d") + ## set local values in .envrc-local (or here if you must) + ## spine_search lives under cgi-bin, not bin - expose it via env + ## var and prepend cgi-bin to PATH for convenience. + export SPINE_SEARCH_BIN="${spine-search}/cgi-bin/spine_search" + export PATH="${spine-search}/cgi-bin:$PATH" + echo "spine_search: $SPINE_SEARCH_BIN" + ## hand off to the user's login shell (e.g. zsh) only when this + ## is an interactive nix-shell entry - not under `nix-shell --run` + ## or `--command`, where exec would swallow the command. Guard env + ## var prevents re-entry loops. + __spine_user_shell=${if userShell == "" then "" else "\"" + userShell + "\""} + if [ -z "$__SPINE_SEARCH_SHELL_HANDOFF" ] \ + && [ -n "$__spine_user_shell" ] \ + && [ "$(basename "$__spine_user_shell")" != "bash" ] \ + && [[ $- == *i* ]]; then + export __SPINE_SEARCH_SHELL_HANDOFF=1 + exec "$__spine_user_shell" + fi + unset __spine_user_shell ''; } diff --git a/src/COPYRIGHT b/src/COPYRIGHT index fa0f004..6d7ee50 100644 --- a/src/COPYRIGHT +++ b/src/COPYRIGHT @@ -5,7 +5,7 @@ - Author: Ralph Amissah [ralph.amissah@gmail.com] - - Copyright: (C) 2015 - 2025 Ralph Amissah + - Copyright: (C) 2015 - 2026 Ralph Amissah - code under src/* src/sisudoc/* - License: AGPL 3 or later: diff --git a/src/ext_depends/arsd/cgi.d b/src/ext_depends/arsd/cgi.d index 0af9f25..a74babd 100644 --- a/src/ext_depends/arsd/cgi.d +++ b/src/ext_depends/arsd/cgi.d @@ -489,6 +489,8 @@ void main() { An import of `arsd.core` was added on March 21, 2023 (dub v11.0). Prior to this, the module's default configuration was completely stand-alone. You must now include the `core.d` file in your builds with `cgi.d`. This change is primarily to integrate the event loops across the library, allowing you to more easily use cgi.d along with my other libraries like simpledisplay and http2.d. Previously, you'd have to run separate helper threads. Now, they can all automatically work together. + + The `struct Uri` was removed on November 2, 2025. You can find that now in [arsd.http2]. Other functions, including `rawurlencode`, `makeDataUrl`, `decodeVariablesSingle`, and `encodeVariables` were moved to [arsd.uri]. +/ module arsd.cgi; @@ -606,6 +608,8 @@ unittest { static import std.file; static import arsd.core; +import arsd.core : EnableSynchronization; // polyfill for opend with removed monitor + version(Posix) import arsd.core : makeNonBlocking; @@ -1883,7 +1887,7 @@ class Cgi { // not using maxContentLength because that might be cranked up to allow // large file uploads. We can handle them, but a huge post[] isn't any good. - if(pps.buffer.length + chunk.length > 8 * 1024 * 1024) // surely this is plenty big enough + if(pps.buffer.length + chunk.length > 24 * 1024 * 1024) // surely this is plenty big enough throw new Exception("wtf is up with such a gigantic form submission????"); pps.buffer ~= chunk; @@ -2289,6 +2293,9 @@ class Cgi { ``` To ensure the necessary data is available to cgi.d. + + History: + The overload with the `checker` callback was added July 29, 2025. +/ void requireBasicAuth(string user, string pass, string message = null, string file = __FILE__, size_t line = __LINE__) { if(authorization != "Basic " ~ Base64.encode(cast(immutable(ubyte)[]) (user ~ ":" ~ pass))) { @@ -2296,6 +2303,17 @@ class Cgi { } } + /// ditto + void requireBasicAuth(scope bool delegate(string user, string pass) checker, string message = null, string file = __FILE__, size_t line = __LINE__) { + // FIXME + /+ + if(authorization != "Basic " ~ Base64.encode(cast(immutable(ubyte)[]) (user ~ ":" ~ pass))) { + throw new AuthorizationRequiredException("Basic", message, file, line); + } + +/ + } + + /// Very simple caching controls - setCache(false) means it will never be cached. Good for rapidly updated or sensitive sites. /// setCache(true) means it will always be cached for as long as possible. Best for static content. /// Use setResponseExpires and updateResponseExpires for more control @@ -2555,6 +2573,8 @@ class Cgi { buffer.add("HTTP/1.0 200 OK", terminator); else buffer.add("HTTP/1.1 200 OK", terminator); + } else { + buffer.add("Status: ", "200 OK", terminator); } if(websocketMode) @@ -2735,7 +2755,7 @@ class Cgi { } // closing the last chunk... - if(nph && rawDataOutput !is null && responseChunked) + if(requestMethod != RequestMethod.HEAD && nph && rawDataOutput !is null && responseChunked) rawDataOutput(cast(const(ubyte)[]) "0\r\n\r\n"); if(flushDelegate) @@ -3117,404 +3137,6 @@ class CgiTester { // should this be a separate module? Probably, but that's a hassle. -/// Makes a data:// uri that can be used as links in most newer browsers (IE8+). -string makeDataUrl(string mimeType, in void[] data) { - auto data64 = Base64.encode(cast(const(ubyte[])) data); - return "data:" ~ mimeType ~ ";base64," ~ assumeUnique(data64); -} - -// FIXME: I don't think this class correctly decodes/encodes the individual parts -/// Represents a url that can be broken down or built up through properties -struct Uri { - alias toString this; // blargh idk a url really is a string, but should it be implicit? - - // scheme//userinfo@host:port/path?query#fragment - - string scheme; /// e.g. "http" in "http://example.com/" - string userinfo; /// the username (and possibly a password) in the uri - string host; /// the domain name - int port; /// port number, if given. Will be zero if a port was not explicitly given - string path; /// e.g. "/folder/file.html" in "http://example.com/folder/file.html" - string query; /// the stuff after the ? in a uri - string fragment; /// the stuff after the # in a uri. - - // idk if i want to keep these, since the functions they wrap are used many, many, many times in existing code, so this is either an unnecessary alias or a gratuitous break of compatibility - // the decode ones need to keep different names anyway because we can't overload on return values... - static string encode(string s) { return encodeUriComponent(s); } - static string encode(string[string] s) { return encodeVariables(s); } - static string encode(string[][string] s) { return encodeVariables(s); } - - /// Breaks down a uri string to its components - this(string uri) { - reparse(uri); - } - - private void reparse(string uri) { - // from RFC 3986 - // the ctRegex triples the compile time and makes ugly errors for no real benefit - // it was a nice experiment but just not worth it. - // enum ctr = ctRegex!r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?"; - /* - Captures: - 0 = whole url - 1 = scheme, with : - 2 = scheme, no : - 3 = authority, with // - 4 = authority, no // - 5 = path - 6 = query string, with ? - 7 = query string, no ? - 8 = anchor, with # - 9 = anchor, no # - */ - // Yikes, even regular, non-CT regex is also unacceptably slow to compile. 1.9s on my computer! - // instead, I will DIY and cut that down to 0.6s on the same computer. - /* - - Note that authority is - user:password@domain:port - where the user:password@ part is optional, and the :port is optional. - - Regex translation: - - Scheme cannot have :, /, ?, or # in it, and must have one or more chars and end in a :. It is optional, but must be first. - Authority must start with //, but cannot have any other /, ?, or # in it. It is optional. - Path cannot have any ? or # in it. It is optional. - Query must start with ? and must not have # in it. It is optional. - Anchor must start with # and can have anything else in it to end of string. It is optional. - */ - - this = Uri.init; // reset all state - - // empty uri = nothing special - if(uri.length == 0) { - return; - } - - size_t idx; - - scheme_loop: foreach(char c; uri[idx .. $]) { - switch(c) { - case ':': - case '/': - case '?': - case '#': - break scheme_loop; - default: - } - idx++; - } - - if(idx == 0 && uri[idx] == ':') { - // this is actually a path! we skip way ahead - goto path_loop; - } - - if(idx == uri.length) { - // the whole thing is a path, apparently - path = uri; - return; - } - - if(idx > 0 && uri[idx] == ':') { - scheme = uri[0 .. idx]; - idx++; - } else { - // we need to rewind; it found a / but no :, so the whole thing is prolly a path... - idx = 0; - } - - if(idx + 2 < uri.length && uri[idx .. idx + 2] == "//") { - // we have an authority.... - idx += 2; - - auto authority_start = idx; - authority_loop: foreach(char c; uri[idx .. $]) { - switch(c) { - case '/': - case '?': - case '#': - break authority_loop; - default: - } - idx++; - } - - auto authority = uri[authority_start .. idx]; - - auto idx2 = authority.indexOf("@"); - if(idx2 != -1) { - userinfo = authority[0 .. idx2]; - authority = authority[idx2 + 1 .. $]; - } - - if(authority.length && authority[0] == '[') { - // ipv6 address special casing - idx2 = authority.indexOf(']'); - if(idx2 != -1) { - auto end = authority[idx2 + 1 .. $]; - if(end.length && end[0] == ':') - idx2 = idx2 + 1; - else - idx2 = -1; - } - } else { - idx2 = authority.indexOf(":"); - } - - if(idx2 == -1) { - port = 0; // 0 means not specified; we should use the default for the scheme - host = authority; - } else { - host = authority[0 .. idx2]; - if(idx2 + 1 < authority.length) - port = to!int(authority[idx2 + 1 .. $]); - else - port = 0; - } - } - - path_loop: - auto path_start = idx; - - foreach(char c; uri[idx .. $]) { - if(c == '?' || c == '#') - break; - idx++; - } - - path = uri[path_start .. idx]; - - if(idx == uri.length) - return; // nothing more to examine... - - if(uri[idx] == '?') { - idx++; - auto query_start = idx; - foreach(char c; uri[idx .. $]) { - if(c == '#') - break; - idx++; - } - query = uri[query_start .. idx]; - } - - if(idx < uri.length && uri[idx] == '#') { - idx++; - fragment = uri[idx .. $]; - } - - // uriInvalidated = false; - } - - private string rebuildUri() const { - string ret; - if(scheme.length) - ret ~= scheme ~ ":"; - if(userinfo.length || host.length) - ret ~= "//"; - if(userinfo.length) - ret ~= userinfo ~ "@"; - if(host.length) - ret ~= host; - if(port) - ret ~= ":" ~ to!string(port); - - ret ~= path; - - if(query.length) - ret ~= "?" ~ query; - - if(fragment.length) - ret ~= "#" ~ fragment; - - // uri = ret; - // uriInvalidated = false; - return ret; - } - - /// Converts the broken down parts back into a complete string - string toString() const { - // if(uriInvalidated) - return rebuildUri(); - } - - /// Returns a new absolute Uri given a base. It treats this one as - /// relative where possible, but absolute if not. (If protocol, domain, or - /// other info is not set, the new one inherits it from the base.) - /// - /// Browsers use a function like this to figure out links in html. - Uri basedOn(in Uri baseUrl) const { - Uri n = this; // copies - if(n.scheme == "data") - return n; - // n.uriInvalidated = true; // make sure we regenerate... - - // userinfo is not inherited... is this wrong? - - // if anything is given in the existing url, we don't use the base anymore. - if(n.scheme.empty) { - n.scheme = baseUrl.scheme; - if(n.host.empty) { - n.host = baseUrl.host; - if(n.port == 0) { - n.port = baseUrl.port; - if(n.path.length > 0 && n.path[0] != '/') { - auto b = baseUrl.path[0 .. baseUrl.path.lastIndexOf("/") + 1]; - if(b.length == 0) - b = "/"; - n.path = b ~ n.path; - } else if(n.path.length == 0) { - n.path = baseUrl.path; - } - } - } - } - - n.removeDots(); - - return n; - } - - void removeDots() { - auto parts = this.path.split("/"); - string[] toKeep; - foreach(part; parts) { - if(part == ".") { - continue; - } else if(part == "..") { - //if(toKeep.length > 1) - toKeep = toKeep[0 .. $-1]; - //else - //toKeep = [""]; - continue; - } else { - //if(toKeep.length && toKeep[$-1].length == 0 && part.length == 0) - //continue; // skip a `//` situation - toKeep ~= part; - } - } - - auto path = toKeep.join("/"); - if(path.length && path[0] != '/') - path = "/" ~ path; - - this.path = path; - } - - unittest { - auto uri = Uri("test.html"); - assert(uri.path == "test.html"); - uri = Uri("path/1/lol"); - assert(uri.path == "path/1/lol"); - uri = Uri("http://me@example.com"); - assert(uri.scheme == "http"); - assert(uri.userinfo == "me"); - assert(uri.host == "example.com"); - uri = Uri("http://example.com/#a"); - assert(uri.scheme == "http"); - assert(uri.host == "example.com"); - assert(uri.fragment == "a"); - uri = Uri("#foo"); - assert(uri.fragment == "foo"); - uri = Uri("?lol"); - assert(uri.query == "lol"); - uri = Uri("#foo?lol"); - assert(uri.fragment == "foo?lol"); - uri = Uri("?lol#foo"); - assert(uri.fragment == "foo"); - assert(uri.query == "lol"); - - uri = Uri("http://127.0.0.1/"); - assert(uri.host == "127.0.0.1"); - assert(uri.port == 0); - - uri = Uri("http://127.0.0.1:123/"); - assert(uri.host == "127.0.0.1"); - assert(uri.port == 123); - - uri = Uri("http://[ff:ff::0]/"); - assert(uri.host == "[ff:ff::0]"); - - uri = Uri("http://[ff:ff::0]:123/"); - assert(uri.host == "[ff:ff::0]"); - assert(uri.port == 123); - } - - // This can sometimes be a big pain in the butt for me, so lots of copy/paste here to cover - // the possibilities. - unittest { - auto url = Uri("cool.html"); // checking relative links - - assert(url.basedOn(Uri("http://test.com/what/test.html")) == "http://test.com/what/cool.html"); - assert(url.basedOn(Uri("https://test.com/what/test.html")) == "https://test.com/what/cool.html"); - assert(url.basedOn(Uri("http://test.com/what/")) == "http://test.com/what/cool.html"); - assert(url.basedOn(Uri("http://test.com/")) == "http://test.com/cool.html"); - assert(url.basedOn(Uri("http://test.com")) == "http://test.com/cool.html"); - assert(url.basedOn(Uri("http://test.com/what/test.html?a=b")) == "http://test.com/what/cool.html"); - assert(url.basedOn(Uri("http://test.com/what/test.html?a=b&c=d")) == "http://test.com/what/cool.html"); - assert(url.basedOn(Uri("http://test.com/what/test.html?a=b&c=d#what")) == "http://test.com/what/cool.html"); - assert(url.basedOn(Uri("http://test.com")) == "http://test.com/cool.html"); - - url = Uri("/something/cool.html"); // same server, different path - assert(url.basedOn(Uri("http://test.com/what/test.html")) == "http://test.com/something/cool.html"); - assert(url.basedOn(Uri("https://test.com/what/test.html")) == "https://test.com/something/cool.html"); - assert(url.basedOn(Uri("http://test.com/what/")) == "http://test.com/something/cool.html"); - assert(url.basedOn(Uri("http://test.com/")) == "http://test.com/something/cool.html"); - assert(url.basedOn(Uri("http://test.com")) == "http://test.com/something/cool.html"); - assert(url.basedOn(Uri("http://test.com/what/test.html?a=b")) == "http://test.com/something/cool.html"); - assert(url.basedOn(Uri("http://test.com/what/test.html?a=b&c=d")) == "http://test.com/something/cool.html"); - assert(url.basedOn(Uri("http://test.com/what/test.html?a=b&c=d#what")) == "http://test.com/something/cool.html"); - assert(url.basedOn(Uri("http://test.com")) == "http://test.com/something/cool.html"); - - url = Uri("?query=answer"); // same path. server, protocol, and port, just different query string and fragment - assert(url.basedOn(Uri("http://test.com/what/test.html")) == "http://test.com/what/test.html?query=answer"); - assert(url.basedOn(Uri("https://test.com/what/test.html")) == "https://test.com/what/test.html?query=answer"); - assert(url.basedOn(Uri("http://test.com/what/")) == "http://test.com/what/?query=answer"); - assert(url.basedOn(Uri("http://test.com/")) == "http://test.com/?query=answer"); - assert(url.basedOn(Uri("http://test.com")) == "http://test.com?query=answer"); - assert(url.basedOn(Uri("http://test.com/what/test.html?a=b")) == "http://test.com/what/test.html?query=answer"); - assert(url.basedOn(Uri("http://test.com/what/test.html?a=b&c=d")) == "http://test.com/what/test.html?query=answer"); - assert(url.basedOn(Uri("http://test.com/what/test.html?a=b&c=d#what")) == "http://test.com/what/test.html?query=answer"); - assert(url.basedOn(Uri("http://test.com")) == "http://test.com?query=answer"); - - url = Uri("/test/bar"); - assert(Uri("./").basedOn(url) == "/test/", Uri("./").basedOn(url)); - assert(Uri("../").basedOn(url) == "/"); - - url = Uri("http://example.com/"); - assert(Uri("../foo").basedOn(url) == "http://example.com/foo"); - - //auto uriBefore = url; - url = Uri("#anchor"); // everything should remain the same except the anchor - //uriBefore.anchor = "anchor"); - //assert(url == uriBefore); - - url = Uri("//example.com"); // same protocol, but different server. the path here should be blank. - - url = Uri("//example.com/example.html"); // same protocol, but different server and path - - url = Uri("http://example.com/test.html"); // completely absolute link should never be modified - - url = Uri("http://example.com"); // completely absolute link should never be modified, even if it has no path - - // FIXME: add something for port too - } - - // these are like javascript's location.search and location.hash - string search() const { - return query.length ? ("?" ~ query) : ""; - } - string hash() const { - return fragment.length ? ("#" ~ fragment) : ""; - } -} - - -/* - for session, see web.d -*/ - /// breaks down a url encoded string string[][string] decodeVariables(string data, string separator = "&", string[]* namesInOrder = null, string[]* valuesInOrder = null) { auto vars = data.split(separator); @@ -3542,76 +3164,6 @@ string[][string] decodeVariables(string data, string separator = "&", string[]* return _get; } -/// breaks down a url encoded string, but only returns the last value of any array -string[string] decodeVariablesSingle(string data) { - string[string] va; - auto varArray = decodeVariables(data); - foreach(k, v; varArray) - va[k] = v[$-1]; - - return va; -} - -/// url encodes the whole string -string encodeVariables(in string[string] data) { - string ret; - - bool outputted = false; - foreach(k, v; data) { - if(outputted) - ret ~= "&"; - else - outputted = true; - - ret ~= encodeUriComponent(k) ~ "=" ~ encodeUriComponent(v); - } - - return ret; -} - -/// url encodes a whole string -string encodeVariables(in string[][string] data) { - string ret; - - bool outputted = false; - foreach(k, arr; data) { - foreach(v; arr) { - if(outputted) - ret ~= "&"; - else - outputted = true; - ret ~= encodeUriComponent(k) ~ "=" ~ encodeUriComponent(v); - } - } - - return ret; -} - -/// Encodes all but the explicitly unreserved characters per rfc 3986 -/// Alphanumeric and -_.~ are the only ones left unencoded -/// name is borrowed from php -string rawurlencode(in char[] data) { - string ret; - ret.reserve(data.length * 2); - foreach(char c; data) { - if( - (c >= 'a' && c <= 'z') || - (c >= 'A' && c <= 'Z') || - (c >= '0' && c <= '9') || - c == '-' || c == '_' || c == '.' || c == '~') - { - ret ~= c; - } else { - ret ~= '%'; - // since we iterate on char, this should give us the octets of the full utf8 string - ret ~= toHexUpper(c); - } - } - - return ret; -} - - // http helper functions // for chunked responses (which embedded http does whenever possible) @@ -3639,22 +3191,6 @@ string toHex(long num) { return to!string(array(ret.retro)); } -string toHexUpper(long num) { - string ret; - while(num) { - int v = num % 16; - num /= 16; - char d = cast(char) ((v < 10) ? v + '0' : (v-10) + 'A'); - ret ~= d; - } - - if(ret.length == 1) - ret ~= "0"; // url encoding requires two digits and that's what this function is used for... - - return to!string(array(ret.retro)); -} - - // the generic mixins /++ @@ -5481,6 +5017,8 @@ void doThreadScgiConnection(CustomCgi, alias fun, long maxContentLength)(Socket } else if (range.sourceClosed) range.source.close(); + range.consume(data.length); + return data; } @@ -5507,7 +5045,6 @@ void doThreadScgiConnection(CustomCgi, alias fun, long maxContentLength)(Socket fun(cgi); cgi.close(); connection.close(); - } catch(AuthorizationRequiredException are) { cgi.setResponseStatus("401 Authorization Required"); cgi.header ("WWW-Authenticate: "~are.type~" realm=\""~are.realm~"\""); @@ -5959,6 +5496,8 @@ import core.atomic; FIXME: should I offer an event based async thing like netman did too? Yeah, probably. */ class ListeningConnectionManager { + version(D_OpenD) mixin EnableSynchronization; + Semaphore semaphore; Socket[256] queue; shared(ubyte) nextIndexFront; @@ -6855,6 +6394,8 @@ ByChunkRange byChunk(BufferedInputRange ir, size_t atMost) { version(cgi_with_websocket) { // http://tools.ietf.org/html/rfc6455 + public import arsd.core : WebSocketOpcode, WebSocketFrame; + /++ WEBSOCKET SUPPORT: @@ -6892,11 +6433,9 @@ version(cgi_with_websocket) { --- +/ - class WebSocket { + class WebSocket : arsd.core.WebSocketBase { Cgi cgi; - private bool isClient = false; - private this(Cgi cgi) { this.cgi = cgi; @@ -6914,7 +6453,7 @@ version(cgi_with_websocket) { return false; } - public bool lowLevelReceive() { + public override bool lowLevelReceive() { auto bfr = cgi.idlol; top: auto got = bfr.front; @@ -6941,7 +6480,7 @@ version(cgi_with_websocket) { } - bool isDataPending(Duration timeout = 0.seconds) { + override bool isDataPending(Duration timeout = 0.seconds) { Socket socket = cgi.idlol.source; auto check = new SocketSet(); @@ -6961,427 +6500,24 @@ version(cgi_with_websocket) { - private void llclose() { + protected override void llshutdown() { cgi.close(); } - private void llsend(ubyte[] data) { + protected override void llclose() {} + + protected override void llsend(ubyte[] data) { cgi.write(data); cgi.flush(); } - void unregisterActiveSocket(WebSocket) {} - - /* copy/paste section { */ - - private int readyState_; - private ubyte[] receiveBuffer; - private size_t receiveBufferUsedLength; - - private Config config; - - enum CONNECTING = 0; /// Socket has been created. The connection is not yet open. - enum OPEN = 1; /// The connection is open and ready to communicate. - enum CLOSING = 2; /// The connection is in the process of closing. - enum CLOSED = 3; /// The connection is closed or couldn't be opened. - - /++ - - +/ - /// Group: foundational - static struct Config { - /++ - These control the size of the receive buffer. - - It starts at the initial size, will temporarily - balloon up to the maximum size, and will reuse - a buffer up to the likely size. - - Anything larger than the maximum size will cause - the connection to be aborted and an exception thrown. - This is to protect you against a peer trying to - exhaust your memory, while keeping the user-level - processing simple. - +/ - size_t initialReceiveBufferSize = 4096; - size_t likelyReceiveBufferSize = 4096; /// ditto - size_t maximumReceiveBufferSize = 10 * 1024 * 1024; /// ditto - - /++ - Maximum combined size of a message. - +/ - size_t maximumMessageSize = 10 * 1024 * 1024; - - string[string] cookies; /// Cookies to send with the initial request. cookies[name] = value; - string origin; /// Origin URL to send with the handshake, if desired. - string protocol; /// the protocol header, if desired. - - /++ - Additional headers to put in the HTTP request. These should be formatted `Name: value`, like for example: - - --- - Config config; - config.additionalHeaders ~= "Authorization: Bearer your_auth_token_here"; - --- - - History: - Added February 19, 2021 (included in dub version 9.2) - +/ - string[] additionalHeaders; - - /++ - Amount of time (in msecs) of idleness after which to send an automatic ping - - Please note how this interacts with [timeoutFromInactivity] - a ping counts as activity that - keeps the socket alive. - +/ - int pingFrequency = 5000; - - /++ - Amount of time to disconnect when there's no activity. Note that automatic pings will keep the connection alive; this timeout only occurs if there's absolutely nothing, including no responses to websocket ping frames. Since the default [pingFrequency] is only seconds, this one minute should never elapse unless the connection is actually dead. - - The one thing to keep in mind is if your program is busy and doesn't check input, it might consider this a time out since there's no activity. The reason is that your program was busy rather than a connection failure, but it doesn't care. You should avoid long processing periods anyway though! - - History: - Added March 31, 2021 (included in dub version 9.4) - +/ - Duration timeoutFromInactivity = 1.minutes; - - /++ - For https connections, if this is `true`, it will fail to connect if the TLS certificate can not be - verified. Setting this to `false` will skip this check and allow the connection to continue anyway. - - History: - Added April 5, 2022 (dub v10.8) - - Prior to this, it always used the global (but undocumented) `defaultVerifyPeer` setting, and sometimes - even if it was true, it would skip the verification. Now, it always respects this local setting. - +/ - bool verifyPeer = true; - } - - /++ - Returns one of [CONNECTING], [OPEN], [CLOSING], or [CLOSED]. - +/ - int readyState() { - return readyState_; - } - - /++ - Closes the connection, sending a graceful teardown message to the other side. - - Code 1000 is the normal closure code. - - History: - The default `code` was changed to 1000 on January 9, 2023. Previously it was 0, - but also ignored anyway. - +/ - /// Group: foundational - void close(int code = 1000, string reason = null) - //in (reason.length < 123) - in { assert(reason.length < 123); } do - { - if(readyState_ != OPEN) - return; // it cool, we done - WebSocketFrame wss; - wss.fin = true; - wss.masked = this.isClient; - wss.opcode = WebSocketOpcode.close; - wss.data = [ubyte((code >> 8) & 0xff), ubyte(code & 0xff)] ~ cast(ubyte[]) reason.dup; - wss.send(&llsend); - - readyState_ = CLOSING; - - closeCalled = true; - - llclose(); - } - - private bool closeCalled; - - /++ - Sends a ping message to the server. This is done automatically by the library if you set a non-zero [Config.pingFrequency], but you can also send extra pings explicitly as well with this function. - +/ - /// Group: foundational - void ping(in ubyte[] data = null) { - WebSocketFrame wss; - wss.fin = true; - wss.masked = this.isClient; - wss.opcode = WebSocketOpcode.ping; - if(data !is null) wss.data = data.dup; - wss.send(&llsend); - } - - /++ - Sends a pong message to the server. This is normally done automatically in response to pings. - +/ - /// Group: foundational - void pong(in ubyte[] data = null) { - WebSocketFrame wss; - wss.fin = true; - wss.masked = this.isClient; - wss.opcode = WebSocketOpcode.pong; - if(data !is null) wss.data = data.dup; - wss.send(&llsend); - } - - /++ - Sends a text message through the websocket. - +/ - /// Group: foundational - void send(in char[] textData) { - WebSocketFrame wss; - wss.fin = true; - wss.masked = this.isClient; - wss.opcode = WebSocketOpcode.text; - wss.data = cast(ubyte[]) textData.dup; - wss.send(&llsend); - } - - /++ - Sends a binary message through the websocket. - +/ - /// Group: foundational - void send(in ubyte[] binaryData) { - WebSocketFrame wss; - wss.masked = this.isClient; - wss.fin = true; - wss.opcode = WebSocketOpcode.binary; - wss.data = cast(ubyte[]) binaryData.dup; - wss.send(&llsend); - } - - /++ - Waits for and returns the next complete message on the socket. - - Note that the onmessage function is still called, right before - this returns. - +/ - /// Group: blocking_api - public WebSocketFrame waitForNextMessage() { - do { - auto m = processOnce(); - if(m.populated) - return m; - } while(lowLevelReceive()); - + override void unregisterAsActiveSocket() {} + override WebSocketFrame waitGotNothing() { throw new ConnectionClosedException("Websocket receive timed out"); - //return WebSocketFrame.init; // FIXME? maybe. } - - /++ - Tells if [waitForNextMessage] would block. - +/ - /// Group: blocking_api - public bool waitForNextMessageWouldBlock() { - checkAgain: - if(isMessageBuffered()) - return false; - if(!isDataPending()) - return true; - - while(isDataPending()) { - if(lowLevelReceive() == false) - throw new ConnectionClosedException("Connection closed in middle of message"); - } - - goto checkAgain; + override bool connectionClosedInMiddleOfMessage() { + throw new ConnectionClosedException("Connection closed in middle of message"); } - - /++ - Is there a message in the buffer already? - If `true`, [waitForNextMessage] is guaranteed to return immediately. - If `false`, check [isDataPending] as the next step. - +/ - /// Group: blocking_api - public bool isMessageBuffered() { - ubyte[] d = receiveBuffer[0 .. receiveBufferUsedLength]; - auto s = d; - if(d.length) { - auto orig = d; - auto m = WebSocketFrame.read(d); - // that's how it indicates that it needs more data - if(d !is orig) - return true; - } - - return false; - } - - private ubyte continuingType; - private ubyte[] continuingData; - //private size_t continuingDataLength; - - private WebSocketFrame processOnce() { - ubyte[] d = receiveBuffer[0 .. receiveBufferUsedLength]; - auto s = d; - // FIXME: handle continuation frames more efficiently. it should really just reuse the receive buffer. - WebSocketFrame m; - if(d.length) { - auto orig = d; - m = WebSocketFrame.read(d); - // that's how it indicates that it needs more data - if(d is orig) - return WebSocketFrame.init; - m.unmaskInPlace(); - switch(m.opcode) { - case WebSocketOpcode.continuation: - if(continuingData.length + m.data.length > config.maximumMessageSize) - throw new Exception("message size exceeded"); - - continuingData ~= m.data; - if(m.fin) { - if(ontextmessage) - ontextmessage(cast(char[]) continuingData); - if(onbinarymessage) - onbinarymessage(continuingData); - - continuingData = null; - } - break; - case WebSocketOpcode.text: - if(m.fin) { - if(ontextmessage) - ontextmessage(m.textData); - } else { - continuingType = m.opcode; - //continuingDataLength = 0; - continuingData = null; - continuingData ~= m.data; - } - break; - case WebSocketOpcode.binary: - if(m.fin) { - if(onbinarymessage) - onbinarymessage(m.data); - } else { - continuingType = m.opcode; - //continuingDataLength = 0; - continuingData = null; - continuingData ~= m.data; - } - break; - case WebSocketOpcode.close: - - //import std.stdio; writeln("closed ", cast(string) m.data); - - ushort code = CloseEvent.StandardCloseCodes.noStatusCodePresent; - const(char)[] reason; - - if(m.data.length >= 2) { - code = (m.data[0] << 8) | m.data[1]; - reason = (cast(char[]) m.data[2 .. $]); - } - - if(onclose) - onclose(CloseEvent(code, reason, true)); - - // if we receive one and haven't sent one back we're supposed to echo it back and close. - if(!closeCalled) - close(code, reason.idup); - - readyState_ = CLOSED; - - unregisterActiveSocket(this); - break; - case WebSocketOpcode.ping: - // import std.stdio; writeln("ping received ", m.data); - pong(m.data); - break; - case WebSocketOpcode.pong: - // import std.stdio; writeln("pong received ", m.data); - // just really references it is still alive, nbd. - break; - default: // ignore though i could and perhaps should throw too - } - } - - if(d.length) { - m.data = m.data.dup(); - } - - import core.stdc.string; - memmove(receiveBuffer.ptr, d.ptr, d.length); - receiveBufferUsedLength = d.length; - - return m; - } - - private void autoprocess() { - // FIXME - do { - processOnce(); - } while(lowLevelReceive()); - } - - /++ - Arguments for the close event. The `code` and `reason` are provided from the close message on the websocket, if they are present. The spec says code 1000 indicates a normal, default reason close, but reserves the code range from 3000-5000 for future definition; the 3000s can be registered with IANA and the 4000's are application private use. The `reason` should be user readable, but not displayed to the end user. `wasClean` is true if the server actually sent a close event, false if it just disconnected. - - $(PITFALL - The `reason` argument references a temporary buffer and there's no guarantee it will remain valid once your callback returns. It may be freed and will very likely be overwritten. If you want to keep the reason beyond the callback, make sure you `.idup` it. - ) - - History: - Added March 19, 2023 (dub v11.0). - +/ - static struct CloseEvent { - ushort code; - const(char)[] reason; - bool wasClean; - - string extendedErrorInformationUnstable; - - /++ - See https://www.rfc-editor.org/rfc/rfc6455#section-7.4.1 for details. - +/ - enum StandardCloseCodes { - purposeFulfilled = 1000, - goingAway = 1001, - protocolError = 1002, - unacceptableData = 1003, // e.g. got text message when you can only handle binary - Reserved = 1004, - noStatusCodePresent = 1005, // not set by endpoint. - abnormalClosure = 1006, // not set by endpoint. closed without a Close control. FIXME: maybe keep a copy of errno around for these - inconsistentData = 1007, // e.g. utf8 validation failed - genericPolicyViolation = 1008, - messageTooBig = 1009, - clientRequiredExtensionMissing = 1010, // only the client should send this - unnexpectedCondition = 1011, - unverifiedCertificate = 1015, // not set by client - } - } - - /++ - The `CloseEvent` you get references a temporary buffer that may be overwritten after your handler returns. If you want to keep it or the `event.reason` member, remember to `.idup` it. - - History: - The `CloseEvent` was changed to a [arsd.core.FlexibleDelegate] on March 19, 2023 (dub v11.0). Before that, `onclose` was a public member of type `void delegate()`. This change means setters still work with or without the [CloseEvent] argument. - - Your onclose method is now also called on abnormal terminations. Check the `wasClean` member of the `CloseEvent` to know if it came from a close frame or other cause. - +/ - arsd.core.FlexibleDelegate!(void delegate(CloseEvent event)) onclose; - void delegate() onerror; /// - void delegate(in char[]) ontextmessage; /// - void delegate(in ubyte[]) onbinarymessage; /// - void delegate() onopen; /// - - /++ - - +/ - /// Group: browser_api - void onmessage(void delegate(in char[]) dg) { - ontextmessage = dg; - } - - /// ditto - void onmessage(void delegate(in ubyte[]) dg) { - onbinarymessage = dg; - } - - /* } end copy/paste */ - - - } /++ @@ -7431,213 +6567,6 @@ version(cgi_with_websocket) { } // FIXME get websocket to work on other modes, not just embedded_httpd - - /* copy/paste in http2.d { */ - enum WebSocketOpcode : ubyte { - continuation = 0, - text = 1, - binary = 2, - // 3, 4, 5, 6, 7 RESERVED - close = 8, - ping = 9, - pong = 10, - // 11,12,13,14,15 RESERVED - } - - public struct WebSocketFrame { - private bool populated; - bool fin; - bool rsv1; - bool rsv2; - bool rsv3; - WebSocketOpcode opcode; // 4 bits - bool masked; - ubyte lengthIndicator; // don't set this when building one to send - ulong realLength; // don't use when sending - ubyte[4] maskingKey; // don't set this when sending - ubyte[] data; - - static WebSocketFrame simpleMessage(WebSocketOpcode opcode, void[] data) { - WebSocketFrame msg; - msg.fin = true; - msg.opcode = opcode; - msg.data = cast(ubyte[]) data.dup; - - return msg; - } - - private void send(scope void delegate(ubyte[]) llsend) { - ubyte[64] headerScratch; - int headerScratchPos = 0; - - realLength = data.length; - - { - ubyte b1; - b1 |= cast(ubyte) opcode; - b1 |= rsv3 ? (1 << 4) : 0; - b1 |= rsv2 ? (1 << 5) : 0; - b1 |= rsv1 ? (1 << 6) : 0; - b1 |= fin ? (1 << 7) : 0; - - headerScratch[0] = b1; - headerScratchPos++; - } - - { - headerScratchPos++; // we'll set header[1] at the end of this - auto rlc = realLength; - ubyte b2; - b2 |= masked ? (1 << 7) : 0; - - assert(headerScratchPos == 2); - - if(realLength > 65535) { - // use 64 bit length - b2 |= 0x7f; - - // FIXME: double check endinaness - foreach(i; 0 .. 8) { - headerScratch[2 + 7 - i] = rlc & 0x0ff; - rlc >>>= 8; - } - - headerScratchPos += 8; - } else if(realLength > 125) { - // use 16 bit length - b2 |= 0x7e; - - // FIXME: double check endinaness - foreach(i; 0 .. 2) { - headerScratch[2 + 1 - i] = rlc & 0x0ff; - rlc >>>= 8; - } - - headerScratchPos += 2; - } else { - // use 7 bit length - b2 |= realLength & 0b_0111_1111; - } - - headerScratch[1] = b2; - } - - //assert(!masked, "masking key not properly implemented"); - if(masked) { - // FIXME: randomize this - headerScratch[headerScratchPos .. headerScratchPos + 4] = maskingKey[]; - headerScratchPos += 4; - - // we'll just mask it in place... - int keyIdx = 0; - foreach(i; 0 .. data.length) { - data[i] = data[i] ^ maskingKey[keyIdx]; - if(keyIdx == 3) - keyIdx = 0; - else - keyIdx++; - } - } - - //writeln("SENDING ", headerScratch[0 .. headerScratchPos], data); - llsend(headerScratch[0 .. headerScratchPos]); - llsend(data); - } - - static WebSocketFrame read(ref ubyte[] d) { - WebSocketFrame msg; - - auto orig = d; - - WebSocketFrame needsMoreData() { - d = orig; - return WebSocketFrame.init; - } - - if(d.length < 2) - return needsMoreData(); - - ubyte b = d[0]; - - msg.populated = true; - - msg.opcode = cast(WebSocketOpcode) (b & 0x0f); - b >>= 4; - msg.rsv3 = b & 0x01; - b >>= 1; - msg.rsv2 = b & 0x01; - b >>= 1; - msg.rsv1 = b & 0x01; - b >>= 1; - msg.fin = b & 0x01; - - b = d[1]; - msg.masked = (b & 0b1000_0000) ? true : false; - msg.lengthIndicator = b & 0b0111_1111; - - d = d[2 .. $]; - - if(msg.lengthIndicator == 0x7e) { - // 16 bit length - msg.realLength = 0; - - if(d.length < 2) return needsMoreData(); - - foreach(i; 0 .. 2) { - msg.realLength |= d[0] << ((1-i) * 8); - d = d[1 .. $]; - } - } else if(msg.lengthIndicator == 0x7f) { - // 64 bit length - msg.realLength = 0; - - if(d.length < 8) return needsMoreData(); - - foreach(i; 0 .. 8) { - msg.realLength |= ulong(d[0]) << ((7-i) * 8); - d = d[1 .. $]; - } - } else { - // 7 bit length - msg.realLength = msg.lengthIndicator; - } - - if(msg.masked) { - - if(d.length < 4) return needsMoreData(); - - msg.maskingKey = d[0 .. 4]; - d = d[4 .. $]; - } - - if(msg.realLength > d.length) { - return needsMoreData(); - } - - msg.data = d[0 .. cast(size_t) msg.realLength]; - d = d[cast(size_t) msg.realLength .. $]; - - return msg; - } - - void unmaskInPlace() { - if(this.masked) { - int keyIdx = 0; - foreach(i; 0 .. this.data.length) { - this.data[i] = this.data[i] ^ this.maskingKey[keyIdx]; - if(keyIdx == 3) - keyIdx = 0; - else - keyIdx++; - } - } - } - - char[] textData() { - return cast(char[]) data; - } - } - /* } */ } @@ -11035,7 +9964,7 @@ private auto serveApiInternal(T)(string urlPrefix) { return internalHandlerWithObject(obj, remainingUrl, cgi, presenter); } catch(Throwable t) { - switch(cgi.request("format", "html")) { + switch(cgi.request("format", cgi.isCalledWithCommandLineArguments ? "json" : "html")) { case "html": static void dummy() {} presenter.presentExceptionAsHtml(cgi, t, null); @@ -11246,7 +10175,7 @@ private auto serveApiInternal(T)(string urlPrefix) { if(callFunction) +/ - auto format = cgi.request("format", defaultFormat!overload()); + auto format = cgi.request("format", cgi.isCalledWithCommandLineArguments ? "json" : defaultFormat!overload()); auto wantsFormFormat = format.startsWith("form-"); if(wantsFormFormat || (automaticForm && cgi.requestMethod == Cgi.RequestMethod.GET)) { @@ -12130,8 +11059,10 @@ auto handleWith(alias handler)(string urlPrefix) { // cuz I'm too lazy to do it better right now static class Hack : WebObject { static import std.traits; + static if(is(typeof(handler) Params == __parameters)) + @(__traits(getAttributes, handler)) @UrlName("") - auto handle(std.traits.Parameters!handler args) { + auto handle(Params args) { return handler(args); } } diff --git a/src/ext_depends/arsd/core.d b/src/ext_depends/arsd/core.d index 52f71ba..e603870 100644 --- a/src/ext_depends/arsd/core.d +++ b/src/ext_depends/arsd/core.d @@ -26,6 +26,20 @@ +/ module arsd.core; +/+ + Intended to be Supported OSes: + * Windows (at least Vista, MAYBE XP) + * Linux + * FreeBSD 14 (maybe 13 too) + * Mac OS + + Eventually also: + * ios + * OpenBSD + * Android + * maybe apple watch os? ++/ + static if(__traits(compiles, () { import core.interpolation; })) { import core.interpolation; @@ -42,6 +56,26 @@ static if(__traits(compiles, () { import core.interpolation; })) { struct InterpolatedExpression(string code) {} } +static if(!__traits(hasMember, object, "SynchronizableObject")) { + alias SynchronizableObject = Object; + mixin template EnableSynchronization() {} +} else { + alias SynchronizableObject = object.SynchronizableObject; + alias EnableSynchronization = Object.EnableSynchronization; +} + +// the old char.inits if you need them +enum char char_invalid = '\xFF'; +enum wchar wchar_invalid = '\uFFFF'; +enum dchar dchar_invalid = '\U0000FFFF'; + +// arsd core is now default but you can opt out for a lil while +version(no_arsd_core) { + +} else { + version=use_arsd_core; +} + version(use_arsd_core) enum use_arsd_core = true; else @@ -85,11 +119,24 @@ version(ArsdUseCustomRuntime) } else { - version(D_OpenD) { - version(OSX) - version=OSXCocoa; - version(iOS) - version=OSXCocoa; + version(ArsdNoCocoa) { + } else { + version(D_OpenD) { + version(OSX) + version=OSXCocoa; + version(iOS) + version=OSXCocoa; + } else version(DigitalMars) { + version(OSX) + version=OSXCocoa; + version(iOS) + version=OSXCocoa; + } else version(LDC) { + version(OSX) + version=OSXCocoa; + version(iOS) + version=OSXCocoa; + } } version = HasFile; @@ -101,7 +148,7 @@ else version = HasTimer; version(linux) version = HasTimer; - version(OSXCocoa) + version(OSX) version = HasTimer; } @@ -127,9 +174,12 @@ import core.time; version(OSXCocoa) { version(ArsdNoCocoa) enum bool UseCocoa = false; - else + else { + version=UseCocoa; enum bool UseCocoa = true; -} + } +} else + enum bool UseCocoa = false; import core.attribute; static if(!__traits(hasMember, core.attribute, "mustuse")) @@ -189,28 +239,36 @@ version(Emscripten) { // THIS FILE DOESN'T ACTUALLY EXIST, WE NEED TO MAKE IT import core.sys.openbsd.sys.event; } else version(OSX) { - version=Arsd_core_kqueue; + version(ArsdNoCocoa) { + version=Arsd_core_kqueue; + } else { + version=Arsd_core_dispatch; + } import core.sys.darwin.sys.event; } else version(iOS) { - version=Arsd_core_kqueue; + version=Arsd_core_dispatch; import core.sys.darwin.sys.event; } // FIXME: pragma(linkerDirective, "-framework", "Cocoa") works in ldc -version(OSXCocoa) +static if(UseCocoa) enum CocoaAvailable = true; else enum CocoaAvailable = false; version(D_OpenD) { - version(OSXCocoa) + static if(UseCocoa) { pragma(linkerDirective, "-framework", "Cocoa"); + pragma(linkerDirective, "-framework", "QuartzCore"); + } } else { - version(OSXCocoa) - version(LDC) + static if(UseCocoa) + version(LDC) { pragma(linkerDirective, "-framework", "Cocoa"); + pragma(linkerDirective, "-framework", "QuartzCore"); + } } version(Posix) { @@ -282,6 +340,55 @@ ref T reinterpretCast(T, V)(return ref V value) @system { } /++ + Determines whether `needle` is a slice of `haystack`. + + History: + Added on February 11, 2025. + +/ +bool isSliceOf(T1, T2)(scope const(T1)[] needle, scope const(T2)[] haystack) @trusted pure nothrow @nogc { + return ( + needle.ptr >= haystack.ptr + && ((needle.ptr + needle.length) <= (haystack.ptr + haystack.length)) + ); +} + +/// +@safe unittest { + string s0 = "01234"; + const(char)[] s1 = s0[1 .. $]; + const(void)[] s2 = s1.castTo!(const(void)[]); + string s3 = s1.idup; + + assert( s0.isSliceOf(s0)); + assert( s1.isSliceOf(s0)); + assert( s2.isSliceOf(s0)); + assert(!s3.isSliceOf(s0)); + + assert(!s0.isSliceOf(s1)); + assert( s1.isSliceOf(s1)); + assert( s2.isSliceOf(s1)); + assert(!s3.isSliceOf(s1)); + + assert(!s0.isSliceOf(s2)); + assert( s1.isSliceOf(s2)); + assert( s2.isSliceOf(s2)); + assert(!s3.isSliceOf(s2)); + + assert(!s0.isSliceOf(s3)); + assert(!s1.isSliceOf(s3)); + assert(!s2.isSliceOf(s3)); + assert( s3.isSliceOf(s3)); + + assert(s1.length == 4); + assert(s1[0 .. 0].isSliceOf(s1)); + assert(s1[0 .. 1].isSliceOf(s1)); + assert(s1[1 .. 2].isSliceOf(s1)); + assert(s1[1 .. 3].isSliceOf(s1)); + assert(s1[1 .. $].isSliceOf(s1)); + assert(s1[$ .. $].isSliceOf(s1)); +} + +/++ Does math as a 64 bit number, but saturates at int.min and int.max when converting back to a 32 bit int. History: @@ -374,6 +481,46 @@ struct stringz { } /+ +/++ + A runtime tagged union, aka a sumtype. + + History: + Added February 15, 2025 ++/ +struct Union(T...) { + private uint contains_; + private union { + private T payload; + } + + static foreach(index, type; T) + @implicit public this(type t) { + contains_ = index; + payload[index] = t; + } + + bool contains(Part)() const { + static assert(indexFor!Part != -1); + return contains_ == indexFor!Part; + } + + inout(Part) get(Part)() inout { + if(!contains!Part) { + throw new ArsdException!"Dynamic type mismatch"(indexFor!Part, contains_); + } + return payload[indexFor!Part]; + } + + private int indexFor(Part)() { + foreach(idx, thing; T) + static if(is(T == Part)) + return idx; + return -1; + } +} ++/ + +/+ DateTime year: 16 bits (-32k to +32k) month: 4 bits @@ -385,13 +532,15 @@ struct stringz { total: 25 bits + 17 bits = 42 bits - fractional seconds: 10 bits + fractional seconds: 10 bits (about milliseconds) accuracy flags: date_valid | time_valid = 2 bits 54 bits used, 8 bits remain. reserve 1 for signed. - would need 11 bits for minute-precise dt offset but meh. + tz offset in 15 minute intervals = 96 slots... can fit in 7 remaining bits... + + would need 11 bits for minute-precise dt offset but meh. would need 10 bits for referring back to tz database (and that's iffy to key, better to use a string tbh) +/ /++ @@ -433,6 +582,112 @@ struct PackedDateTime { } /++ + Construction helpers + +/ + static PackedDateTime withDate(int year, int month, int day) { + PackedDateTime p; + p.setDate(year, month, day); + return p; + } + /// ditto + static PackedDateTime withTime(int hours, int minutes, int seconds, int fractionalSeconds = 0) { + PackedDateTime p; + p.setTime(hours, minutes, seconds, fractionalSeconds); + return p; + } + /// ditto + static PackedDateTime withDateAndTime(int year, int month, int day = 1, int hours = 0, int minutes = 0, int seconds = 0, int fractionalSeconds = 0) { + PackedDateTime p; + p.setDate(year, month, day); + p.setTime(hours, minutes, seconds, fractionalSeconds); + return p; + } + /// ditto + static PackedDateTime lastDayOfMonth(int year, int month) { + PackedDateTime p; + p.setDate(year, month, daysInMonth(year, month)); + return p; + } + /++ +/ + static bool isLeapYear(int year) { + return + (year % 4) == 0 + && + ( + ((year % 100) != 0) + || + ((year % 400) == 0) + ) + ; + } + unittest { + assert(isLeapYear(2024)); + assert(!isLeapYear(2023)); + assert(!isLeapYear(2025)); + assert(isLeapYear(2000)); + assert(!isLeapYear(1900)); + } + static immutable ubyte[12] daysInMonthTable = [ + 31, 28, 31, 30, 31, 30, + 31, 31, 30, 31, 30, 31 + ]; + + static int daysInMonth(int year, int month) { + assert(month >= 1 && month <= 12); + if(month == 2) + return isLeapYear(year) ? 29 : 28; + else + return daysInMonthTable[month - 1]; + } + unittest { + assert(daysInMonth(2025, 12) == 31); + assert(daysInMonth(2025, 2) == 28); + assert(daysInMonth(2024, 2) == 29); + } + static int daysInYear(int year) { + return isLeapYear(year) ? 366 : 365; + } + + /++ + Sets the whole date and time portions in one function call. + + History: + Added December 13, 2025 + +/ + void setTime(int hours, int minutes, int seconds, int fractionalSeconds = 0) { + this.hours = hours; + this.minutes = minutes; + this.seconds = seconds; + this.fractionalSeconds = fractionalSeconds; + this.hasTime = true; + } + + /// ditto + void setDate(int year, int month, int day) { + this.year = year; + this.month = month; + this.day = day; + this.hasDate = true; + } + + /// ditto + void clearTime() { + this.hours = 0; + this.minutes = 0; + this.seconds = 0; + this.fractionalSeconds = 0; + this.hasTime = false; + } + + /// ditto + void clearDate() { + this.year = 0; + this.month = 0; + this.day = 0; + this.hasDate = false; + } + + /++ +/ int fractionalSeconds() const { return getFromMask(00, 10); } /// ditto @@ -488,6 +743,49 @@ struct PackedDateTime { return cast(int) (packedData & mask); } + + /++ + Returns the day of week for the date portion. + + Throws AssertError if used when [hasDate] is false. + + Returns: + 0 == Sunday, 6 == Saturday + + History: + Added December 13, 2025 + +/ + int dayOfWeek() const { + assert(hasDate); + auto y = year; + auto m = month; + if(m == 1 || m == 2) { + y--; + m += 12; + } + return ( + day + + (13 * (m+1) / 5) + + (y % 100) + + (y % 100) / 4 + + (y / 100) / 4 - + 2 * (y / 100) + ) % 7; + } + + long opCmp(PackedDateTime rhs) const { + if(this.hasDate == rhs.hasDate && this.hasTime == rhs.hasTime) + return cast(long) this.packedData - cast(long) rhs.packedData; + if(this.hasDate && rhs.hasDate) { + PackedDateTime c1 = this; + c1.clearTime(); + rhs.clearTime(); + return c1.opCmp(rhs); + } + // if one of them is just time, no date, we can't compare + // but as long as there's two date components we can compare them. + assert(0, "invalid comparison, one is a date, other is a time"); + } } unittest { @@ -508,6 +806,89 @@ unittest { assert(dt.toString() == "2024-05-31", dt.toString()); dt.hasTime = true; assert(dt.toString() == "2024-05-31T14:30:25", dt.toString()); + + assert(dt.dayOfWeek == 6); +} + +unittest { + PackedDateTime a; + PackedDateTime b; + a.setDate(2025, 01, 01); + b.setDate(2024, 12, 31); + assert(a > b); +} + +/++ + A `PackedInterval` can be thought of as the difference between [PackedDateTime]s, similarly to how a [Duration] is a difference between [MonoTime]s or [SimplifiedUtcTimestamp]s. + + + The key speciality is in how it treats months and days separately. Months are not a consistent length, and neither are days when you consider daylight saving time. This thing assumes that if you add those, the month/day number will always increase, just the exact details since then might be truncated. (E.g., January 31st + 1 month = February 28/29 depending on leap year). If you multiply, the parts are done individually, so January 31st + 1 month * 2 = March 31st, despite + 1 month truncating to the shorter day in February. + + Internally, this stores months and days as 16 bit signed `short`s each, then the milliseconds is stored as a 32 bit signed `int`. It applies by first adding months, truncating days as needed, then adding days, then adding milliseconds. + + If you iterate over intervals, be careful not to allow month truncation to change the result. (Jan 31st + 1 month) + 1 month will not actually give the same result as Jan 31st + 2 months. You want to add to the interval, then apply to the original date again, not to some accumulated date. + + History: + Added December 13, 2025 ++/ +struct PackedInterval { + private ulong packedData; + + this(int months, int days = 0, int milliseconds = 0) { + this.months = months; + this.days = days; + this.milliseconds = milliseconds; + } + + /++ + Getters and setters for the components + +/ + short months() const { + return cast(short)((packedData >> 48) & 0xffff); + } + + /// ditto + short days() const { + return cast(short)((packedData >> 32) & 0xffff); + } + + /// ditto + int milliseconds() const { + return cast(int)(packedData & 0xffff_ffff); + } + + /// ditto + void months(int v) { + short d = cast(short) v; + ulong s = d; + packedData &= ~(0xffffUL << 48); + packedData |= s << 48; + } + + /// ditto + void days(int v) { + short d = cast(short) v; + ulong s = d; + packedData &= ~(0xffffUL << 32); + packedData |= s << 32; + } + + /// ditto + void milliseconds(int v) { + packedData &= 0xffffffff_00000000UL; + packedData |= cast(ulong) v; + } + + PackedInterval opBinary(string op : "*")(int iterations) const { + return PackedInterval(this.months * iterations, this.days * iterations, this.milliseconds * iterations); + } +} + +unittest { + PackedInterval pi = PackedInterval(1); + assert(pi.months == 1); + assert(pi.days == 0); + assert(pi.milliseconds == 0); } /++ @@ -516,6 +897,12 @@ unittest { struct SimplifiedUtcTimestamp { long timestamp; + this(long hnsecTimestamp) { + this.timestamp = hnsecTimestamp; + } + + // this(PackedDateTime pdt) + string toString() const { import core.stdc.time; char[128] buffer; @@ -534,9 +921,26 @@ struct SimplifiedUtcTimestamp { return SimplifiedUtcTimestamp(621_355_968_000_000_000L + t * 1_000_000_000L / 100); } + /++ + History: + Added November 22, 2025 + +/ + static SimplifiedUtcTimestamp now() { + import core.stdc.time; + return SimplifiedUtcTimestamp.fromUnixTime(time(null)); + } + time_t toUnixTime() const { return cast(time_t) ((timestamp - 621_355_968_000_000_000L) / 1_000_000_0); // hnsec = 7 digits } + + long stdTime() const { + return timestamp; + } + + SimplifiedUtcTimestamp opBinary(string op : "+")(Duration d) const { + return SimplifiedUtcTimestamp(this.timestamp + d.total!"hnsecs"); + } } unittest { @@ -545,7 +949,76 @@ unittest { } /++ + A little builder pattern helper that is meant for use by other library code. + + History: + Added October 31, 2025 ++/ +struct AdHocBuiltStruct(string tag, string[] names = [], T...) { + static assert(names.length == T.length); + + T values; + + auto opDispatch(string name, Arg)(Arg value) { + return AdHocBuiltStruct!(tag, names ~ name, T, Arg)(values, value); + } +} + +unittest { + AdHocBuiltStruct!"tag"() + .id(5) + .name("five") + ; +} + +/++ + Represents a generic raw element to be embedded in an interpolated sequence. + + Use with caution, its exact meaning is dependent on the specific function being called, but it generally is meant to disable encoding protections the function normally provides. + + History: + Added October 31, 2025 ++/ +struct iraw { + string s; + + @system this(string s) { + this.s = s; + } +} + +/++ + Counts the number of bits set to `1` in a value, using intrinsics when available. + + History: + Added December 15, 2025 ++/ +int countOfBitsSet(ulong v) { + version(LDC) { + import ldc.intrinsics; + return cast(int) llvm_ctpop(v); + } else { + // kerninghan's algorithm + int count = 0; + while(v) { + v &= v - 1; + count++; + } + return count; + } +} + +unittest { + assert(countOfBitsSet(0) == 0); + assert(countOfBitsSet(ulong.max) == 64); + assert(countOfBitsSet(0x0f0f) == 8); + assert(countOfBitsSet(0x0f0f2) == 9); +} + +/++ A limited variant to hold just a few types. It is made for the use of packing a small amount of extra data into error messages and some transit across virtual function boundaries. + + Note that empty strings and null values are indistinguishable unless you explicitly slice the end of some other existing string! +/ /+ ALL OF THESE ARE SUBJECT TO CHANGE @@ -562,10 +1035,10 @@ unittest { * if ptr == 8, length is a utc timestamp (hnsecs) * if ptr == 9, length is a duration (signed hnsecs) * if ptr == 10, length is a date or date time (bit packed, see flags in data to determine if it is a Date, Time, or DateTime) - * if ptr == 11, length is a dchar - * if ptr == 12, length is a bool (redundant to int?) + * if ptr == 11, length is a decimal - 13, 14 reserved. prolly decimals. (4, 8 digits after decimal) + * if ptr == 12, length is a bool (redundant to int?) + 13, 14 reserved. maybe char? * if ptr == 15, length must be 0. this holds an empty, non-null, SSO string. * if ptr >= 16 && < 24, length is reinterpret-casted a small string of length of (ptr & 0x7) + 1 @@ -629,8 +1102,11 @@ struct LimitedVariant { utcTimestamp, duration, dateTime, + decimal, + + // FIXME interval like postgres? e.g. 30 days, 2 months. distinct from Duration, which is a difference of monoTimes or utcTimestamps, interval is more like a difference of PackedDateTime. + // FIXME boolean? char? specializations of float for various precisions... - // FIXME boolean? char? decimal? // could do enums by way of a pointer but kinda iffy // maybe some kind of prefixed string too for stuff like xml and json or enums etc. @@ -666,6 +1142,7 @@ struct LimitedVariant { case 8: return Contains.utcTimestamp; case 9: return Contains.duration; case 10: return Contains.dateTime; + case 11: return Contains.decimal; case 15: return length is null ? Contains.emptySso : Contains.invalid; default: @@ -677,7 +1154,7 @@ struct LimitedVariant { else return isHighBitSet ? Contains.bytes : Contains.string; } else { - return Contains.invalid; + return isHighBitSet ? Contains.bytes : Contains.invalid; } } } @@ -688,6 +1165,11 @@ struct LimitedVariant { } /// ditto + bool containsDecimal() const { + return contains() == Contains.decimal; + } + + /// ditto bool containsInt() const { with(Contains) switch(contains) { @@ -765,7 +1247,7 @@ struct LimitedVariant { } /++ - getString gets a reference to the string stored internally, see [toString] to get a string representation or whatever is inside. + getString gets a reference to the string stored internally, which may be a temporary. See [toString] to get a normal string representation or whatever is inside. +/ const(char)[] getString() const return { @@ -868,6 +1350,15 @@ struct LimitedVariant { assert(0); } + /// ditto + DynamicDecimal getDecimal() const { + if(containsDecimal) + return DynamicDecimal(cast(long) length); + else + Throw(); + assert(0); + } + /++ @@ -913,11 +1404,13 @@ struct LimitedVariant { return getDuration().toString(); case dateTime: return getDateTime().toString(); + case decimal: + return getDecimal().toString(); case double_: auto d = getDouble(); import core.stdc.stdio; - char[128] buffer; + char[64] buffer; auto count = snprintf(buffer.ptr, buffer.length, "%.17lf", d); return buffer[0 .. count].idup; case invalid: @@ -1015,6 +1508,12 @@ struct LimitedVariant { this.ptr = cast(ubyte*) 10; this.length = cast(void*) a.packedData; } + + /// ditto + this(DynamicDecimal a) { + this.ptr = cast(ubyte*) 11; + this.length = cast(void*) a.storage; + } } unittest { @@ -1044,6 +1543,166 @@ private union floathack { void* e; } +/+ + 64 bit signed goes up to 9.22x10^18 + + 3 bit precision = 0-7 + 60 bits remain for the value = 1.15x10^18. + + so you can use up to 10 digits decimal 7 digits. + + 9,999,999,999.9999999 + + math between decimals must always have the same precision on both sides. + + decimal and 32 bit int is always allowed assuming the int is a whole number. + + FIXME add this to LimitedVariant ++/ +/++ + A DynamicDecimal is a fixed-point object whose precision is dynamically typed. + + + It packs everything into a 64 bit value. It uses one bit for sign, three bits + for precision, then the rest of them for the value. This means the precision + (that is, the number of digits after the decimal) can be from 0 to 7, and there + can be a total of 18 digits. + + Numbers can be added and subtracted only if they have matching precision. They can + be multiplied and divided only by whole numbers. + + History: + Added December 12, 2025. ++/ +struct DynamicDecimal { + private ulong storage; + + private this(ulong storage) { + this.storage = storage; + } + + this(long value, int precision) { + assert(precision >= 0 && precision <= 7); + bool isNeg = value < 0; + if(isNeg) + value = -value; + assert((value & 0xf000_0000_0000_0000) == 0); + + storage = + (isNeg ? 0x8000_0000_0000_0000 : 0) + | + (cast(ulong) precision << 60) + | + (value) + ; + } + + private bool isNegative() { + return (storage >> 63) ? true : false; + } + + /++ + +/ + int precision() { + return (storage >> 60) & 7; + } + + /++ + +/ + long value() { + long omg = storage & 0x0fff_ffff_ffff_ffff; + if(isNegative) + omg = -omg; + return omg; + } + + /++ + Some basic arithmetic operators are defined on this: +, -, *, and /, but only between + numbers of the same precision. Note that division always returns the quotient and remainder + together in one return and any overflowing operations will also throw. + +/ + typeof(this) opBinary(string op)(typeof(this) rhs) if(op == "+" || op == "-") { + assert(this.precision == rhs.precision); + return typeof(this)(mixin("this.value" ~ op ~ "rhs.value"), this.precision); + } + + /// ditto + typeof(this) opBinary(string op)(int rhs) if(op == "*") { + // what if we overflow on the multiplication? FIXME + return typeof(this)(this.value * rhs, this.precision); + } + + /// ditto + static struct DivisionResult { + DynamicDecimal quotient; + DynamicDecimal remainder; + } + + /// ditto + DivisionResult opBinary(string op)(int rhs) if(op == "/") { + return DivisionResult(typeof(this)(this.value / rhs, this.precision), typeof(this)(this.value % rhs, this.precision)); + } + + /// ditto + typeof(this) opUnary(string op : "-")() { + return typeof(this)(-this.value, this.precision); + } + + /// ditto + long opCmp(typeof(this) rhs) { + assert(this.precision == rhs.precision); + return this.value - rhs.value; + } + + /++ + Converts to a floating point type. There's potentially a loss of precision here. + +/ + double toFloatingPoint() { + long divisor = 1; + foreach(i; 0 .. this.precision) + divisor *= 10; + return cast(double) this.value / divisor; + } + + /++ + +/ + string toString(int minimumNumberOfDigitsLeftOfDecimal = 1) @system { + char[64] buffer = void; + // FIXME: what about a group separator arg? + IntToStringArgs args = IntToStringArgs(). + withPadding(minimumNumberOfDigitsLeftOfDecimal + this.precision); + auto got = intToString(this.value, buffer[], args); + assert(got.length >= this.precision); + int digitsLeftOfDecimal = cast(int) got.length - this.precision; + auto toShift = buffer[got.length - this.precision .. got.length]; + import core.stdc.string; + memmove(toShift.ptr + 1, toShift.ptr, toShift.length); + toShift[0] = '.'; + return buffer[0 .. got.length + 1].idup; + } +} + +unittest { + DynamicDecimal a = DynamicDecimal(100, 2); + auto res = a / 3; + assert(res.quotient.value == 33); + assert(res.remainder.value == 1); + res = a / 2; + assert(res.quotient.value == 50); + assert(res.remainder.value == 0); + + assert(res.quotient.toFloatingPoint == 0.50); + assert(res.quotient.toString() == "0.50"); + + assert((a * 2).value == 200); + + DynamicDecimal b = DynamicDecimal(1, 4); + assert(b.toFloatingPoint() == 0.0001); + assert(b.toString() == "0.0001"); + + assert(a > (a / 2).quotient); +} + /++ This is a dummy type to indicate the end of normal arguments and the beginning of the file/line inferred args. It is meant to ensure you don't accidentally send a string that is interpreted as a filename when it was meant to be a normal argument to the function and trigger the wrong overload. +/ @@ -1261,6 +1920,7 @@ struct CharzBuffer { buffer[0 .. data.length] = data[]; buffer[data.length] = 0; + buffer = buffer[0 .. data.length + 1]; } } @@ -1434,6 +2094,14 @@ char[] intToString(long value, char[] buffer, IntToStringArgs args = IntToString int pos; + bool needsOverflowFixup = false; + + if(value == long.min) { + // -long.min will overflow so we're gonna cheat + value += 1; + needsOverflowFixup = true; + } + if(value < 0) { buffer[pos++] = '-'; value = -value; @@ -1441,19 +2109,46 @@ char[] intToString(long value, char[] buffer, IntToStringArgs args = IntToString int start = pos; int digitCount; + int groupCount; + + void outputDigit(char c) { + if(groupSize && groupCount == groupSize) { + buffer[pos++] = args.separator; + groupCount = 0; + } + + buffer[pos++] = c; + groupCount++; + digitCount++; + + } do { auto remainder = value % radix; value = value / radix; + if(needsOverflowFixup) { + if(remainder + 1 == radix) { + outputDigit('0'); + remainder = 0; + value += 1; + } else { + remainder += 1; + } + needsOverflowFixup = false; + } - buffer[pos++] = cast(char) (remainder < 10 ? (remainder + '0') : (remainder - 10 + args.ten)); - digitCount++; + outputDigit(cast(char) (remainder < 10 ? (remainder + '0') : (remainder - 10 + args.ten))); } while(value); if(digitsPad > 0) { while(digitCount < digitsPad) { + if(groupSize && groupCount == groupSize) { + buffer[pos++] = args.separator; + groupCount = 0; + } buffer[pos++] = args.padWith; digitCount++; + groupCount++; } } @@ -1504,6 +2199,167 @@ struct IntToStringArgs { } } +struct FloatToStringArgs { + private { + // whole number component + ubyte padTo; + char padWith; + ubyte groupSize; + char separator; + + // for the fractional component + ubyte minimumPrecision = 0; // will always show at least this many digits after the decimal (if it is 0 there may be no decimal) + ubyte maximumPrecision = 32; // will round to this many after the decimal + + bool useScientificNotation; // if this is true, note the whole number component will always be exactly one digit, so the pad stuff applies to the exponent only and it assumes pad with zero's to two digits + } + + FloatToStringArgs withPadding(int padTo, char padWith = '0') { + FloatToStringArgs args = this; + args.padTo = cast(ubyte) padTo; + args.padWith = padWith; + return args; + } + + FloatToStringArgs withGroupSeparator(int groupSize, char separator = '_') { + FloatToStringArgs args = this; + args.groupSize = cast(ubyte) groupSize; + args.separator = separator; + return args; + } + + FloatToStringArgs withPrecision(int minDigits, int maxDigits = 0) { + FloatToStringArgs args = this; + args.minimumPrecision = cast(ubyte) minDigits; + if(maxDigits < minDigits) + maxDigits = minDigits; + args.maximumPrecision = cast(ubyte) maxDigits; + return args; + } + + FloatToStringArgs withScientificNotation(bool enabled) { + FloatToStringArgs args = this; + args.useScientificNotation = enabled; + return args; + } +} + +// the buffer should be at least 32 bytes long, maybe more with other args +char[] floatToString(double value, char[] buffer, FloatToStringArgs args = FloatToStringArgs.init) { + // actually doing this is pretty painful, so gonna pawn it off on the C lib + import core.stdc.stdio; + // FIXME: what if there's a locale in place that changes the decimal point? + auto ret = snprintf(buffer.ptr, buffer.length, args.useScientificNotation ? "%.*e" : "%.*f", args.maximumPrecision, value); + if(!args.useScientificNotation && (args.padTo || args.groupSize)) { + char[32] scratch = void; + auto idx = buffer[0 .. ret].indexOf("."); + + int digitsOutput = 0; + int digitsGrouped = 0; + if(idx > 0) { + // there is a whole number component + int pos = cast(int) scratch.length; + + auto splitPoint = idx; + + while(idx) { + if(args.groupSize && digitsGrouped == args.groupSize) { + scratch[--pos] = args.separator; + digitsGrouped = 0; + } + scratch[--pos] = buffer[--idx]; + + digitsOutput++; + digitsGrouped++; + } + + if(args.padTo) + while(digitsOutput < args.padTo) { + if(args.groupSize && digitsGrouped == args.groupSize) { + scratch[--pos] = args.separator; + digitsGrouped = 0; + } + + scratch[--pos] = args.padWith; + + digitsOutput++; + digitsGrouped++; + } + + char[32] remainingBuffer; + remainingBuffer[0 .. ret - splitPoint]= buffer[splitPoint .. ret]; + + buffer[0 .. scratch.length - pos] = scratch[pos .. $]; + buffer[scratch.length - pos .. scratch.length - pos + ret - splitPoint] = remainingBuffer[0 .. ret - splitPoint]; + + ret = cast(int) scratch.length - pos + ret - splitPoint; + } + } + + // sprintf will always put zeroes on to the maximum precision, but if it is a bunch of trailing zeroes, we can trim them + // if scientific notation, don't forget to bring the e back down though. + int trailingZeroesStart = -1; + int dot = -1; + int trailingZeroesEnd; + bool inZone; + foreach(idx, ch; buffer[0 .. ret]) { + if(inZone) { + if(ch == '0') { + if(trailingZeroesStart == -1) { + trailingZeroesStart = cast(int) idx; + } + } else if(ch == 'e') { + trailingZeroesEnd = cast(int) idx; + break; + } else { + trailingZeroesStart = -1; + } + } else { + if(ch == '.') { + inZone = true; + dot = cast(int) idx; + } + } + } + if(trailingZeroesEnd == 0) + trailingZeroesEnd = ret; + + // 0.430000 + // end = $ + // dot = 1 + // start = 4 + // precision is thus 3-1 = 2 + // if min precision = 0 + if(dot != -1 && trailingZeroesStart > dot) { + auto currentPrecision = trailingZeroesStart - dot - 1; + auto precWanted = (args.minimumPrecision > currentPrecision) ? args.minimumPrecision : currentPrecision; + auto sliceOffset = dot + precWanted + 1; + if(precWanted == 0) + sliceOffset -= 1; // remove the dot + char[] keep = buffer[trailingZeroesEnd .. ret]; + + // slice copy doesn't allow overlapping and since it can, we need to memmove + //buffer[sliceOffset .. sliceOffset + keep.length] = keep[]; + import core.stdc.string; + memmove(buffer[sliceOffset .. ret].ptr, keep.ptr, keep.length); + + ret = cast(int) (sliceOffset + keep.length); + } + /+ + if(minimumPrecision > 0) { + auto idx = buffer[0 .. ret].indexOf("."); + if(idx == -1) { + buffer[ret++] = '.'; + idx = ret; + } + + while(ret - idx < minimumPrecision) + buffer[ret++] = '0'; + } + +/ + return buffer[0 .. ret]; +} + unittest { char[32] buffer; assert(intToString(0, buffer[]) == "0"); @@ -1521,6 +2377,40 @@ unittest { assert(intToString(0xef1, buffer[], IntToStringArgs().withRadix(16).withPadding(8)) == "00000ef1"); assert(intToString(-0xef1, buffer[], IntToStringArgs().withRadix(16).withPadding(8)) == "-00000ef1"); assert(intToString(-0xef1, buffer[], IntToStringArgs().withRadix(16, 'A').withPadding(8, ' ')) == "- EF1"); + + assert(intToString(4000, buffer[], IntToStringArgs().withPadding(4).withGroupSeparator(3, ',')) == "4,000"); + assert(intToString(400, buffer[], IntToStringArgs().withPadding(4).withGroupSeparator(3, ',')) == "0,400"); + + const pi = 3.14159256358979; + assert(floatToString(pi, buffer[], FloatToStringArgs().withPrecision(3)) == "3.142"); + assert(floatToString(pi, buffer[], FloatToStringArgs().withPrecision(2)) == "3.14"); + assert(floatToString(pi, buffer[], FloatToStringArgs().withPrecision(0)) == "3"); + + assert(floatToString(4.0, buffer[], FloatToStringArgs().withPrecision(0)) == "4"); + assert(floatToString(4.0, buffer[], FloatToStringArgs().withPrecision(3)) == "4.000"); + + assert(floatToString(4.0, buffer[], FloatToStringArgs().withPadding(3).withPrecision(3)) == "004.000"); + assert(floatToString(4.0, buffer[], FloatToStringArgs().withPadding(3).withGroupSeparator(3, ',').withPrecision(3)) == "004.000"); + assert(floatToString(4.0, buffer[], FloatToStringArgs().withPadding(4).withGroupSeparator(3, ',').withPrecision(3)) == "0,004.000"); + assert(floatToString(4000.0, buffer[], FloatToStringArgs().withPadding(4).withGroupSeparator(3, ',').withPrecision(3)) == "4,000.000"); + + assert(floatToString(4.25, buffer[], FloatToStringArgs().withPrecision(3, 5)) == "4.250"); + assert(floatToString(4.25, buffer[], FloatToStringArgs().withPrecision(2, 5)) == "4.25"); + assert(floatToString(4.25, buffer[], FloatToStringArgs().withPrecision(0, 5)) == "4.25"); + assert(floatToString(4.25, buffer[], FloatToStringArgs().withPrecision(0)) == "4"); + assert(floatToString(4.251, buffer[], FloatToStringArgs().withPrecision(1)) == "4.3"); // 2.25 would be rounded to even and thus be 2.2... sometimes. this less ambiguous + + //assert(floatToString(4.25, buffer[], FloatToStringArgs().withPrecision(1)) == "4.2"); + //assert(floatToString(4.35, buffer[], FloatToStringArgs().withPrecision(1)) == "4.3"); + /+ + import core.stdc.stdio; + printf("%.1f\n", 4.25); // 4.2 + printf("%.1f\n", 4.35); // 4.3 + +/ + + assert(floatToString(pi*10, buffer[], FloatToStringArgs().withPrecision(2).withScientificNotation(true)) == "3.14e+01"); + + assert(floatToString(500, buffer[], FloatToStringArgs().withPrecision(0, 2).withScientificNotation(true)) == "5e+02"); } /++ @@ -1532,18 +2422,16 @@ inout(char)[] stripInternal(return inout(char)[] s) { bool isAllWhitespace = true; foreach(i, char c; s) if(c != ' ' && c != '\t' && c != '\n' && c != '\r') { - s = s[i .. $]; isAllWhitespace = false; + s = s[i .. $]; break; } - if(isAllWhitespace) - return s[$..$]; + return s[0 .. 0]; - for(int a = cast(int)(s.length - 1); a > 0; a--) { - char c = s[a]; + foreach_reverse(i, char c; s) { if(c != ' ' && c != '\t' && c != '\n' && c != '\r') { - s = s[0 .. a + 1]; + s = s[0 .. i + 1]; break; } } @@ -1576,8 +2464,13 @@ inout(char)[] stripRightInternal(return inout(char)[] s) { Moved from color.d to core.d in March 2023 (dub v11.0). +/ string toStringInternal(T)(T t) { - char[32] buffer; - static if(is(T : string)) + char[256] bufferBacking; + return writeGuts(bufferBacking[], null, null, null, false, false, &makeString, t); + /+ + char[64] buffer; + static if(is(typeof(t.toString) : string)) + return t.toString(); + else static if(is(T : string)) return t; else static if(is(T == enum)) { switch(t) { @@ -1599,10 +2492,24 @@ string toStringInternal(T)(T t) { } ret ~= "]"; return ret; + } else static if(is(T : double)) { + import core.stdc.stdio; + auto ret = snprintf(buffer.ptr, buffer.length, "%f", t); + return buffer[0 .. ret].idup; } else { static assert(0, T.stringof ~ " makes compile too slow"); // import std.conv; return to!string(t); } + +/ +} + +unittest { + assert(toStringInternal(-43) == "-43"); + assert(toStringInternal(4.5) == "4.5"); +} + +char[] toTextBuffer(T...)(char[] bufferBacking, T t) { + return cast(char[]) writeGuts(bufferBacking[], null, null, null, false, false, &makeStringCasting, t); } /++ @@ -1690,7 +2597,55 @@ package size_t encodeUtf8(out char[4] buf, dchar c) @safe pure { goto L3; } +/++ + If it fits in the provided buffer, it will use it, otherwise, it will reallocate as-needed with the append operator. + + Returns: + the slice of `buffer` actually used, or the newly allocated array, if it was necessary. + History: + Added November 14, 2025 ++/ +char[] transcodeUtf(scope const wchar[] input, char[] buffer) { + size_t pos; + char[4] temp; + foreach(dchar ch; input) { + auto stride = encodeUtf8(temp, ch); + if(pos + stride < buffer.length) { + buffer[pos .. pos + stride] = temp[0 .. stride]; + pos += stride; + } else { + char[] t = buffer[0 .. pos]; + t ~= temp[0 .. stride]; + buffer = t; + } + } + return buffer[0 .. pos]; +} + +/// ditto +char[] transcodeUtf(scope const dchar[] input, char[] buffer) { + // yes, this function body is char-for-char identical to the + // previous overload. i just don't want to use a template here. + size_t pos; + char[4] temp; + foreach(dchar ch; input) { + auto stride = encodeUtf8(temp, ch); + if(pos + stride < buffer.length) { + buffer[pos .. pos + stride] = temp[0 .. stride]; + pos += stride; + } else { + char[] t = buffer[0 .. pos]; + t ~= temp[0 .. stride]; + buffer = t; + } + } + return buffer[0 .. pos]; +} +inout(char)[] transcodeUtf(inout(char)[] input) { + // no change needed + return input; +} private bool isValidDchar(dchar c) pure nothrow @safe @nogc { @@ -1776,7 +2731,7 @@ package string decodeUriComponent(string s, bool translatePlusToSpace = false) { if(idx + 2 >= s.length) throw ArsdException!"Invalid percent-encoding"("End of string reached", idx, s); - n ~= (hexDecode(s[idx + 1]) << 4) | hexDecode(s[idx + 2]); + n ~= cast(char) ((hexDecode(s[idx + 1]) << 4) | hexDecode(s[idx + 2])); previous = idx + 3; } else if(translatePlusToSpace && ch == '+') { @@ -1806,9 +2761,11 @@ unittest { assert(decodeUriComponent("+") == "+"); assert(decodeUriComponent("+", true) == " "); + + assert(decodeUriComponent("%C3%A4") == "ä"); } -private auto toDelegate(T)(T t) { +public auto toDelegate(T)(T t) { // static assert(is(T == function)); // lol idk how to do what i actually want here static if(is(T Return == return)) @@ -1819,8 +2776,8 @@ private auto toDelegate(T)(T t) { } } return &((cast(Wrapper*) t).call); - } else static assert(0, "could not get params"); - else static assert(0, "could not get return value"); + } else static assert(0, "could not get params; is it already a delegate you can pass directly?"); + else static assert(0, "could not get return value, if it is a functor maybe try getting a delegate with `&yourobj.opCall` instead of toDelegate(yourobj)"); } @system unittest { @@ -2160,11 +3117,13 @@ class ArsdExceptionBase : object.Exception { sink(value); }); - // full stack trace - sink("\n----------------\n"); - foreach(str; info) { - sink(str); - sink("\n"); + // full stack trace, if available + if(info) { + sink("\n----------------\n"); + foreach(str; info) { + sink(str); + sink("\n"); + } } } /// ditto @@ -2765,11 +3724,13 @@ interface ICoreEventLoop { Runs the event loop for this thread until the `until` delegate returns `true`. +/ final void run(scope bool delegate() until) { + exitApplicationRequested = false; while(!exitApplicationRequested && !until()) { runOnce(); } } + package static int function() getTimeout; private __gshared bool exitApplicationRequested; final static void exitApplication() { @@ -2845,14 +3806,17 @@ interface ICoreEventLoop { 1: run before each loop OS wait call 2: run after each loop OS wait call 3: run both before and after each OS wait call - 4: single shot? - 8: no-coalesce? (if after was just run, it will skip the before loops unless this flag is set) + 4: single shot? NOT IMPLEMENTED + 8: no-coalesce? NOT IMPLEMENTED (if after was just run, it will skip the before loops unless this flag is set) + FIXME: it should return a handle you can use to unregister it +/ - void addDelegateOnLoopIteration(void delegate() dg, uint timingFlags); + UnregisterToken addDelegateOnLoopIteration(void delegate() dg, uint timingFlags); - final void addDelegateOnLoopIteration(void function() dg, uint timingFlags) { - addDelegateOnLoopIteration(toDelegate(dg), timingFlags); + final UnregisterToken addDelegateOnLoopIteration(void function() dg, uint timingFlags) { + if(timingFlags == 0) + assert(0, "would never run"); + return addDelegateOnLoopIteration(toDelegate(dg), timingFlags); } // to send messages between threads, i'll queue up a function that just call dispatchMessage. can embed the arg inside the callback helper prolly. @@ -2862,8 +3826,9 @@ interface ICoreEventLoop { @mustuse static struct UnregisterToken { private CoreEventLoopImplementation impl; - private int fd; + private int fd = -1; private CallbackHelper cb; + private void delegate() dg; /++ Unregisters the file descriptor from the event loop and releases the reference to the callback held by the event loop (which will probably free it). @@ -2873,8 +3838,14 @@ interface ICoreEventLoop { void unregister() { assert(impl !is null, "Cannot reuse unregister token"); + if(dg !is null) + impl.unregisterDg(dg); + version(Arsd_core_epoll) { - impl.unregisterFd(fd); + if(fd != -1) + impl.unregisterFd(fd); + } else version(Arsd_core_dispatch) { + throw new NotYetImplementedException(); } else version(Arsd_core_kqueue) { // intentionally blank - all registrations are one-shot there // FIXME: actually it might not have gone off yet, in that case we do need to delete the filter @@ -2883,7 +3854,8 @@ interface ICoreEventLoop { } else static assert(0); - cb.release(); + if(cb) + cb.release(); this = typeof(this).init; } } @@ -2904,6 +3876,8 @@ interface ICoreEventLoop { version(Arsd_core_epoll) { impl.unregisterFd(fd); + } else version(Arsd_core_dispatch) { + throw new NotYetImplementedException(); } else version(Arsd_core_kqueue) { // intentionally blank - all registrations are one-shot there // FIXME: actually it might not have gone off yet, in that case we do need to delete the filter @@ -2935,6 +3909,7 @@ interface ICoreEventLoop { private CoreEventLoopImplementation impl; private HANDLE handle; private CallbackHelper cb; + private void delegate() dg; /++ Unregisters the handle from the event loop and releases the reference to the callback held by the event loop (which will probably free it). @@ -2944,9 +3919,14 @@ interface ICoreEventLoop { void unregister() { assert(impl !is null, "Cannot reuse unregister token"); - impl.unregisterHandle(handle, cb); + if(dg !is null) + impl.unregisterDg(dg); - cb.release(); + if(handle) + impl.unregisterHandle(handle, cb); + + if(cb) + cb.release(); this = typeof(this).init; } } @@ -3157,6 +4137,19 @@ struct FilePath { } } + /+ + FilePath makeRelative(FilePath base, TreatAsWindowsPath treatAsWindowsPath = TreatAsWindowsPath.guess) const { + if(this.path.startsWith(base.path)) { + auto p = this.path[base.path .. $]; + if(p.length && p[0] == '/') + p = p[1 .. $]; + if(p.length) + return FilePath(p); + } + throw new Exception("idk how to make " ~ this.path ~ " relative to " ~ base.path); + } + +/ + // dg returns true to continue, false to break void foreachPathComponent(scope bool delegate(size_t index, in char[] component) dg) const { size_t start; @@ -3421,6 +4414,693 @@ FilePath getCurrentWorkingDirectory() { assert(0, "Not implemented"); } +/++ + Specialization of `string` to indicate it is a URI. You should generally use [arsd.uri.Uri] instead of this in user code. + + History: + Added November 2, 2025 ++/ +struct UriString { + string uri; + + alias toString this; + + string toString() { + return uri; + } +} + +/++ + Shared base code for web socket client in [arsd.http2] and server in [arsd.cgi]. + + History: + Moved to arsd.core on November 2, 2025 ++/ +class WebSocketBase { + /* copy/paste section { */ + + package int readyState_; + protected ubyte[] receiveBuffer; + protected size_t receiveBufferUsedLength; + + protected Config config; + + enum CONNECTING = 0; /// Socket has been created. The connection is not yet open. + enum OPEN = 1; /// The connection is open and ready to communicate. + enum CLOSING = 2; /// The connection is in the process of closing. + enum CLOSED = 3; /// The connection is closed or couldn't be opened. + + /++ + + +/ + /// Group: foundational + static struct Config { + /++ + These control the size of the receive buffer. + + It starts at the initial size, will temporarily + balloon up to the maximum size, and will reuse + a buffer up to the likely size. + + Anything larger than the maximum size will cause + the connection to be aborted and an exception thrown. + This is to protect you against a peer trying to + exhaust your memory, while keeping the user-level + processing simple. + +/ + size_t initialReceiveBufferSize = 4096; + size_t likelyReceiveBufferSize = 4096; /// ditto + size_t maximumReceiveBufferSize = 10 * 1024 * 1024; /// ditto + + /++ + Maximum combined size of a message. + +/ + size_t maximumMessageSize = 10 * 1024 * 1024; + + string[string] cookies; /// Cookies to send with the initial request. cookies[name] = value; + string origin; /// Origin URL to send with the handshake, if desired. + string protocol; /// the protocol header, if desired. + + /++ + Additional headers to put in the HTTP request. These should be formatted `Name: value`, like for example: + + --- + Config config; + config.additionalHeaders ~= "Authorization: Bearer your_auth_token_here"; + --- + + History: + Added February 19, 2021 (included in dub version 9.2) + +/ + string[] additionalHeaders; + + /++ + Amount of time (in msecs) of idleness after which to send an automatic ping + + Please note how this interacts with [timeoutFromInactivity] - a ping counts as activity that + keeps the socket alive. + +/ + int pingFrequency = 5000; + + /++ + Amount of time to disconnect when there's no activity. Note that automatic pings will keep the connection alive; this timeout only occurs if there's absolutely nothing, including no responses to websocket ping frames. Since the default [pingFrequency] is only seconds, this one minute should never elapse unless the connection is actually dead. + + The one thing to keep in mind is if your program is busy and doesn't check input, it might consider this a time out since there's no activity. The reason is that your program was busy rather than a connection failure, but it doesn't care. You should avoid long processing periods anyway though! + + History: + Added March 31, 2021 (included in dub version 9.4) + +/ + Duration timeoutFromInactivity = 1.minutes; + + /++ + For https connections, if this is `true`, it will fail to connect if the TLS certificate can not be + verified. Setting this to `false` will skip this check and allow the connection to continue anyway. + + History: + Added April 5, 2022 (dub v10.8) + + Prior to this, it always used the global (but undocumented) `defaultVerifyPeer` setting, and sometimes + even if it was true, it would skip the verification. Now, it always respects this local setting. + +/ + bool verifyPeer = true; + } + + /++ + Returns one of [CONNECTING], [OPEN], [CLOSING], or [CLOSED]. + +/ + int readyState() { + return readyState_; + } + + /++ + Closes the connection, sending a graceful teardown message to the other side. + If you provide no arguments, it sends code 1000, normal closure. If you provide + a code, you should also provide a short reason string. + + Params: + code = reason code. + + 0-999 are invalid. + 1000-2999 are defined by the RFC. [https://www.rfc-editor.org/rfc/rfc6455.html#section-7.4.1] + 1000 - normal finish + 1001 - endpoint going away + 1002 - protocol error + 1003 - unacceptable data received (e.g. binary message when you can't handle it) + 1004 - reserved + 1005 - missing status code (should not be set except by implementations) + 1006 - abnormal connection closure (should only be set by implementations) + 1007 - inconsistent data received (i.e. utf-8 decode error in text message) + 1008 - policy violation + 1009 - received message too big + 1010 - client aborting due to required extension being unsupported by the server + 1011 - server had unexpected failure + 1015 - reserved for TLS handshake failure + 3000-3999 are to be registered with IANA. + 4000-4999 are private-use custom codes depending on the application. These are what you'd most commonly set here. + + reason = <= 123 bytes of human-readable reason text, used for logs and debugging + + History: + The default `code` was changed to 1000 on January 9, 2023. Previously it was 0, + but also ignored anyway. + + On May 11, 2024, the optional arguments were changed to overloads since if you provide a code, you should also provide a reason. + +/ + /// Group: foundational + void close() { + close(1000, null); + } + + /// ditto + void close(int code, string reason) + //in (reason.length < 123) + in { assert(reason.length <= 123); } do + { + if(readyState_ != OPEN) + return; // it cool, we done + WebSocketFrame wss; + wss.fin = true; + wss.masked = this.isClient; + wss.opcode = WebSocketOpcode.close; + wss.data = [ubyte((code >> 8) & 0xff), ubyte(code & 0xff)] ~ cast(ubyte[]) reason.dup; + wss.send(&llsend, &getRandomByte); + + readyState_ = CLOSING; + + closeCalled = true; + + llshutdown(); + } + + deprecated("If you provide a code, please also provide a reason string") void close(int code) { + close(code, null); + } + + + protected bool closeCalled; + + /++ + Sends a ping message to the server. This is done automatically by the library if you set a non-zero [Config.pingFrequency], but you can also send extra pings explicitly as well with this function. + +/ + /// Group: foundational + void ping(in ubyte[] data = null) { + WebSocketFrame wss; + wss.fin = true; + wss.masked = this.isClient; + wss.opcode = WebSocketOpcode.ping; + if(data !is null) wss.data = data.dup; + wss.send(&llsend, &getRandomByte); + } + + /++ + Sends a pong message to the server. This is normally done automatically in response to pings. + +/ + /// Group: foundational + void pong(in ubyte[] data = null) { + WebSocketFrame wss; + wss.fin = true; + wss.masked = this.isClient; + wss.opcode = WebSocketOpcode.pong; + if(data !is null) wss.data = data.dup; + wss.send(&llsend, &getRandomByte); + } + + /++ + Sends a text message through the websocket. + +/ + /// Group: foundational + void send(in char[] textData) { + WebSocketFrame wss; + wss.fin = true; + wss.masked = this.isClient; + wss.opcode = WebSocketOpcode.text; + wss.data = cast(ubyte[]) textData.dup; + wss.send(&llsend, &getRandomByte); + } + + /++ + Sends a binary message through the websocket. + +/ + /// Group: foundational + void send(in ubyte[] binaryData) { + WebSocketFrame wss; + wss.masked = this.isClient; + wss.fin = true; + wss.opcode = WebSocketOpcode.binary; + wss.data = cast(ubyte[]) binaryData.dup; + wss.send(&llsend, &getRandomByte); + } + + /++ + Waits for and returns the next complete message on the socket. + + Note that the onmessage function is still called, right before + this returns. + +/ + /// Group: blocking_api + public WebSocketFrame waitForNextMessage() { + do { + auto m = processOnce(); + if(m.populated) + return m; + } while(lowLevelReceive()); + + return waitGotNothing(); + } + + /++ + Tells if [waitForNextMessage] would block. + +/ + /// Group: blocking_api + public bool waitForNextMessageWouldBlock() { + checkAgain: + if(isMessageBuffered()) + return false; + if(!isDataPending()) + return true; + while(isDataPending()) + if(lowLevelReceive() == false) + return connectionClosedInMiddleOfMessage(); + goto checkAgain; + } + + /++ + Is there a message in the buffer already? + If `true`, [waitForNextMessage] is guaranteed to return immediately. + If `false`, check [isDataPending] as the next step. + +/ + /// Group: blocking_api + public bool isMessageBuffered() { + ubyte[] d = receiveBuffer[0 .. receiveBufferUsedLength]; + auto s = d; + if(d.length) { + auto orig = d; + auto m = WebSocketFrame.read(d); + // that's how it indicates that it needs more data + if(d !is orig) + return true; + } + + return false; + } + + protected ubyte continuingType; + protected ubyte[] continuingData; + //protected size_t continuingDataLength; + + protected WebSocketFrame processOnce() { + ubyte[] d = receiveBuffer[0 .. receiveBufferUsedLength]; + auto s = d; + // FIXME: handle continuation frames more efficiently. it should really just reuse the receive buffer. + WebSocketFrame m; + if(d.length) { + auto orig = d; + m = WebSocketFrame.read(d); + // that's how it indicates that it needs more data + if(d is orig) + return WebSocketFrame.init; + m.unmaskInPlace(); + switch(m.opcode) { + case WebSocketOpcode.continuation: + if(continuingData.length + m.data.length > config.maximumMessageSize) + throw new Exception("message size exceeded"); + + continuingData ~= m.data; + if(m.fin) { + if(ontextmessage) + ontextmessage(cast(char[]) continuingData); + if(onbinarymessage) + onbinarymessage(continuingData); + + continuingData = null; + } + break; + case WebSocketOpcode.text: + if(m.fin) { + if(ontextmessage) + ontextmessage(m.textData); + } else { + continuingType = m.opcode; + //continuingDataLength = 0; + continuingData = null; + continuingData ~= m.data; + } + break; + case WebSocketOpcode.binary: + if(m.fin) { + if(onbinarymessage) + onbinarymessage(m.data); + } else { + continuingType = m.opcode; + //continuingDataLength = 0; + continuingData = null; + continuingData ~= m.data; + } + break; + case WebSocketOpcode.close: + + //import std.stdio; writeln("closed ", cast(string) m.data); + + ushort code = CloseEvent.StandardCloseCodes.noStatusCodePresent; + const(char)[] reason; + + if(m.data.length >= 2) { + code = (m.data[0] << 8) | m.data[1]; + reason = (cast(char[]) m.data[2 .. $]); + } + + if(onclose) + onclose(CloseEvent(code, reason, true)); + + // if we receive one and haven't sent one back we're supposed to echo it back and close. + if(!closeCalled) + close(code, reason.idup); + + readyState_ = CLOSED; + + unregisterAsActiveSocket(); + llclose(); + break; + case WebSocketOpcode.ping: + // import std.stdio; writeln("ping received ", m.data); + pong(m.data); + break; + case WebSocketOpcode.pong: + // import std.stdio; writeln("pong received ", m.data); + // just really references it is still alive, nbd. + break; + default: // ignore though i could and perhaps should throw too + } + } + + if(d.length) { + m.data = m.data.dup(); + } + + import core.stdc.string; + memmove(receiveBuffer.ptr, d.ptr, d.length); + receiveBufferUsedLength = d.length; + + return m; + } + + /++ + Arguments for the close event. The `code` and `reason` are provided from the close message on the websocket, if they are present. The spec says code 1000 indicates a normal, default reason close, but reserves the code range from 3000-5000 for future definition; the 3000s can be registered with IANA and the 4000's are application private use. The `reason` should be user readable, but not displayed to the end user. `wasClean` is true if the server actually sent a close event, false if it just disconnected. + + $(PITFALL + The `reason` argument references a temporary buffer and there's no guarantee it will remain valid once your callback returns. It may be freed and will very likely be overwritten. If you want to keep the reason beyond the callback, make sure you `.idup` it. + ) + + History: + Added March 19, 2023 (dub v11.0). + +/ + static struct CloseEvent { + ushort code; + const(char)[] reason; + bool wasClean; + + string extendedErrorInformationUnstable; + + /++ + See https://www.rfc-editor.org/rfc/rfc6455#section-7.4.1 for details. + +/ + enum StandardCloseCodes { + purposeFulfilled = 1000, + goingAway = 1001, + protocolError = 1002, + unacceptableData = 1003, // e.g. got text message when you can only handle binary + Reserved = 1004, + noStatusCodePresent = 1005, // not set by endpoint. + abnormalClosure = 1006, // not set by endpoint. closed without a Close control. FIXME: maybe keep a copy of errno around for these + inconsistentData = 1007, // e.g. utf8 validation failed + genericPolicyViolation = 1008, + messageTooBig = 1009, + clientRequiredExtensionMissing = 1010, // only the client should send this + unnexpectedCondition = 1011, + unverifiedCertificate = 1015, // not set by client + } + + string toString() { + return cast(string) (arsd.core.toStringInternal(code) ~ ": " ~ reason); + } + } + + /++ + The `CloseEvent` you get references a temporary buffer that may be overwritten after your handler returns. If you want to keep it or the `event.reason` member, remember to `.idup` it. + + History: + The `CloseEvent` was changed to a [arsd.core.FlexibleDelegate] on March 19, 2023 (dub v11.0). Before that, `onclose` was a public member of type `void delegate()`. This change means setters still work with or without the [CloseEvent] argument. + + Your onclose method is now also called on abnormal terminations. Check the `wasClean` member of the `CloseEvent` to know if it came from a close frame or other cause. + +/ + arsd.core.FlexibleDelegate!(void delegate(CloseEvent event)) onclose; + void delegate() onerror; /// + void delegate(in char[]) ontextmessage; /// + void delegate(in ubyte[]) onbinarymessage; /// + void delegate() onopen; /// + + /++ + + +/ + /// Group: browser_api + void onmessage(void delegate(in char[]) dg) { + ontextmessage = dg; + } + + /// ditto + void onmessage(void delegate(in ubyte[]) dg) { + onbinarymessage = dg; + } + + /* } end copy/paste */ + + + // used to decide if we mask outgoing msgs + protected bool isClient; + protected abstract void llsend(ubyte[] d); + protected ubyte getRandomByte() @trusted { + // FIXME: it is just for masking but still should be less awful + __gshared ubyte seed = 0xe2; + return ++seed; + } + protected abstract void llclose(); + protected abstract void llshutdown(); + public abstract bool lowLevelReceive(); + protected abstract bool isDataPending(Duration timeout = 0.seconds); + protected abstract void unregisterAsActiveSocket(); + protected abstract WebSocketFrame waitGotNothing(); + protected abstract bool connectionClosedInMiddleOfMessage(); +} +/* copy/paste from cgi.d */ +public { + enum WebSocketOpcode : ubyte { + continuation = 0, + text = 1, + binary = 2, + // 3, 4, 5, 6, 7 RESERVED + close = 8, + ping = 9, + pong = 10, + // 11,12,13,14,15 RESERVED + } + + public struct WebSocketFrame { + package(arsd) bool populated; + bool fin; + bool rsv1; + bool rsv2; + bool rsv3; + WebSocketOpcode opcode; // 4 bits + bool masked; + ubyte lengthIndicator; // don't set this when building one to send + ulong realLength; // don't use when sending + ubyte[4] maskingKey; // don't set this when sending + ubyte[] data; + + static WebSocketFrame simpleMessage(WebSocketOpcode opcode, in void[] data) { + WebSocketFrame msg; + msg.fin = true; + msg.opcode = opcode; + msg.data = cast(ubyte[]) data.dup; // it is mutated below when masked, so need to be cautious and copy it, sigh + + return msg; + } + + private void send(scope void delegate(ubyte[]) llsend, scope ubyte delegate() getRandomByte) { + ubyte[64] headerScratch; + int headerScratchPos = 0; + + realLength = data.length; + + { + ubyte b1; + b1 |= cast(ubyte) opcode; + b1 |= rsv3 ? (1 << 4) : 0; + b1 |= rsv2 ? (1 << 5) : 0; + b1 |= rsv1 ? (1 << 6) : 0; + b1 |= fin ? (1 << 7) : 0; + + headerScratch[0] = b1; + headerScratchPos++; + } + + { + headerScratchPos++; // we'll set header[1] at the end of this + auto rlc = realLength; + ubyte b2; + b2 |= masked ? (1 << 7) : 0; + + assert(headerScratchPos == 2); + + if(realLength > 65535) { + // use 64 bit length + b2 |= 0x7f; + + // FIXME: double check endinaness + foreach(i; 0 .. 8) { + headerScratch[2 + 7 - i] = rlc & 0x0ff; + rlc >>>= 8; + } + + headerScratchPos += 8; + } else if(realLength > 125) { + // use 16 bit length + b2 |= 0x7e; + + // FIXME: double check endinaness + foreach(i; 0 .. 2) { + headerScratch[2 + 1 - i] = rlc & 0x0ff; + rlc >>>= 8; + } + + headerScratchPos += 2; + } else { + // use 7 bit length + b2 |= realLength & 0b_0111_1111; + } + + headerScratch[1] = b2; + } + + //assert(!masked, "masking key not properly implemented"); + if(masked) { + foreach(ref item; maskingKey) + item = getRandomByte(); + headerScratch[headerScratchPos .. headerScratchPos + 4] = maskingKey[]; + headerScratchPos += 4; + + // we'll just mask it in place... + int keyIdx = 0; + foreach(i; 0 .. data.length) { + data[i] = data[i] ^ maskingKey[keyIdx]; + if(keyIdx == 3) + keyIdx = 0; + else + keyIdx++; + } + } + + //writeln("SENDING ", headerScratch[0 .. headerScratchPos], data); + llsend(headerScratch[0 .. headerScratchPos]); + if(data.length) + llsend(data); + } + + static WebSocketFrame read(ref ubyte[] d) { + WebSocketFrame msg; + + auto orig = d; + + WebSocketFrame needsMoreData() { + d = orig; + return WebSocketFrame.init; + } + + if(d.length < 2) + return needsMoreData(); + + ubyte b = d[0]; + + msg.populated = true; + + msg.opcode = cast(WebSocketOpcode) (b & 0x0f); + b >>= 4; + msg.rsv3 = b & 0x01; + b >>= 1; + msg.rsv2 = b & 0x01; + b >>= 1; + msg.rsv1 = b & 0x01; + b >>= 1; + msg.fin = b & 0x01; + + b = d[1]; + msg.masked = (b & 0b1000_0000) ? true : false; + msg.lengthIndicator = b & 0b0111_1111; + + d = d[2 .. $]; + + if(msg.lengthIndicator == 0x7e) { + // 16 bit length + msg.realLength = 0; + + if(d.length < 2) return needsMoreData(); + + foreach(i; 0 .. 2) { + msg.realLength |= d[0] << ((1-i) * 8); + d = d[1 .. $]; + } + } else if(msg.lengthIndicator == 0x7f) { + // 64 bit length + msg.realLength = 0; + + if(d.length < 8) return needsMoreData(); + + foreach(i; 0 .. 8) { + msg.realLength |= ulong(d[0]) << ((7-i) * 8); + d = d[1 .. $]; + } + } else { + // 7 bit length + msg.realLength = msg.lengthIndicator; + } + + if(msg.masked) { + + if(d.length < 4) return needsMoreData(); + + msg.maskingKey = d[0 .. 4]; + d = d[4 .. $]; + } + + if(msg.realLength > d.length) { + return needsMoreData(); + } + + msg.data = d[0 .. cast(size_t) msg.realLength]; + d = d[cast(size_t) msg.realLength .. $]; + + return msg; + } + + void unmaskInPlace() { + if(this.masked) { + int keyIdx = 0; + foreach(i; 0 .. this.data.length) { + this.data[i] = this.data[i] ^ this.maskingKey[keyIdx]; + if(keyIdx == 3) + keyIdx = 0; + else + keyIdx++; + } + } + } + + char[] textData() { + return cast(char[]) data; + } + } +} + /+ struct FilePathGeneric { @@ -3667,7 +5347,7 @@ version(HasFile) class AbstractFile { final switch(require) { case RequirePreexisting.no: - creation = CREATE_ALWAYS; + creation = OPEN_ALWAYS; break; case RequirePreexisting.yes: creation = OPEN_EXISTING; @@ -3744,7 +5424,7 @@ version(HasFile) class AbstractFile { } break; case OpenMode.appendOnly: - flags |= O_APPEND; + flags |= O_WRONLY | O_APPEND; final switch(require) { case RequirePreexisting.no: @@ -3808,6 +5488,10 @@ version(HasFile) class AbstractFile { handle = -1; } } + + NativeFileHandle nativeHandle() { + return this.handle; + } } /++ @@ -3824,6 +5508,10 @@ version(HasFile) class File : AbstractFile { super(false, filename, mode, require, specialFlags); } + this(NativeFileHandle wrapWithoutOtherwiseChanging) { + super(wrapWithoutOtherwiseChanging); + } + /++ +/ @@ -3885,20 +5573,51 @@ else class AsyncFile { Tip: prefer the callback ones. If settings where async is possible, it will do async, and if not, it will sync. - NOT IMPLEMENTED + NOT FULLY IMPLEMENTED +/ void writeFile(string filename, const(void)[] contents) { - + // FIXME: stop using the C lib and start error checking + import core.stdc.stdio; + CharzBuffer fn = filename; + auto file = fopen(fn.ptr, "wb"); + if(file is null) + throw new ErrnoApiException("fopen", errno, [SavedArgument("filename", LimitedVariant(filename))]); + fwrite(contents.ptr, 1, contents.length, file); + fclose(file); } /// ditto -string readTextFile(string filename, string fileEncoding = null) { - return null; +const(ubyte[]) readBinaryFile(string filename) { + // FIXME: stop using the C lib and check for more errors + + import core.stdc.stdio; + CharzBuffer fn = filename; + auto file = fopen(fn.ptr, "rb"); + if(file is null) + throw new ErrnoApiException("fopen", errno, [SavedArgument("filename", LimitedVariant(filename))]); + ubyte[] buffer = new ubyte[](64 * 1024); + ubyte[] contents; + + while(true) { + auto ret = fread(buffer.ptr, 1, buffer.length, file); + if(ret < buffer.length) { + if(contents is null) + contents = buffer[0 .. ret]; + else + contents ~= buffer[0 .. ret]; + break; + } else { + contents ~= buffer[0 .. ret]; + } + } + fclose(file); + + return contents; } /// ditto -const(ubyte[]) readBinaryFile(string filename) { - return null; +string readTextFile(string filename, string fileEncoding = null) { + return cast(string) readBinaryFile(filename); } /+ @@ -4273,6 +5992,8 @@ class Timer { auto el = getThisThreadEventLoop(EventLoopType.Ui); unregisterToken = el.addCallbackOnFdReadable(fd, new CallbackHelper(&trigger)); + } else version(Arsd_core_kqueue) { + this.ident = ++identTicker; } else throw new NotYetImplementedException(); // FIXME: freebsd 12 has timer_fd and netbsd 10 too } @@ -4308,6 +6029,17 @@ class Timer { if(timerfd_settime(fd, 0, &value, null) == -1) { throw new ErrnoApiException("couldn't change pulse timer", errno); } + } else version(Arsd_core_kqueue) { + // FIXME + + auto el = cast(CoreEventLoopImplementation) getThisThreadEventLoop(); + + kevent_t ev; + + cbh = new CallbackHelper(&trigger); + + EV_SET(&ev, this.ident, EVFILT_TIMER, EV_ADD | EV_ENABLE | EV_CLEAR | (repeats ? 0 : EV_ONESHOT), NOTE_USECONDS, 1000 * intervalInMilliseconds, cast(void*) cbh); + ErrnoEnforce!kevent(el.kqueuefd, &ev, 1, null, 0, null); } else { throw new NotYetImplementedException(); } @@ -4370,7 +6102,7 @@ class Timer { } } - version(Windows) {} else { + version(Windows) {} else version(Arsd_core_kqueue) {} else { ICoreEventLoop.UnregisterToken unregisterToken; } @@ -4385,26 +6117,24 @@ class Timer { void destroy() { version(Windows) { cbh.release(); - } else { - unregisterToken.unregister(); - } - - version(Windows) { staticDestroy(handle); handle = null; } else version(linux) { + unregisterToken.unregister(); staticDestroy(fd); fd = -1; + } else version(Arsd_core_kqueue) { } else throw new NotYetImplementedException(); } ~this() { - version(Windows) {} else + version(Windows) { + if(handle) + cleanupQueue.queue!staticDestroy(handle); + } else version(linux) { cleanupQueue.queue!unregister(unregisterToken); - version(Windows) { if(handle) - cleanupQueue.queue!staticDestroy(handle); - } else version(linux) { if(fd != -1) - cleanupQueue.queue!staticDestroy(fd); + if(fd != -1) + cleanupQueue.queue!staticDestroy(fd); } } @@ -4430,7 +6160,8 @@ class Timer { version(Windows) {} else static void unregister(arsd.core.ICoreEventLoop.UnregisterToken urt) { - urt.unregister(); + if(urt.impl !is null) + urt.unregister(); } @@ -4466,6 +6197,7 @@ class Timer { if(this.lastEventLoopRoundTriggered == eventLoopRound) return; // never try to actually run faster than the event loop lastEventLoopRoundTriggered = eventLoopRound; + } else version(Arsd_core_kqueue) { } else throw new NotYetImplementedException(); if(onPulse) @@ -4487,7 +6219,11 @@ class Timer { CallbackHelper cbh; } else version(linux) { int fd = -1; - } else version(OSXCocoa) { + } else version(Arsd_core_kqueue) { + int ident; + static int identTicker; + CallbackHelper cbh; + } else static if(UseCocoa) { } else static assert(0, "timer not supported"); } @@ -5360,22 +7096,23 @@ enum GetFilesResult { More things may be added later to be more like what Phobos supports. +/ -bool matchesFilePattern(scope const(char)[] name, scope const(char)[] pattern) { +bool matchesFilePattern(scope const(char)[] name, scope const(char)[] pattern, char star = '*') { if(pattern.length == 0) return false; - if(pattern == "*") + if(pattern.length == 1 && pattern[0] == star) return true; - if(pattern.length > 2 && pattern[0] == '*' && pattern[$-1] == '*') { + if(pattern.length > 2 && pattern[0] == star && pattern[$-1] == star) { // if the rest of pattern appears in name, it is good return name.indexOf(pattern[1 .. $-1]) != -1; - } else if(pattern[0] == '*') { + } else if(pattern[0] == star) { // if the rest of pattern is at end of name, it is good return name.endsWith(pattern[1 .. $]); - } else if(pattern[$-1] == '*') { + } else if(pattern[$-1] == star) { // if the rest of pattern is at start of name, it is good return name.startsWith(pattern[0 .. $-1]); } else if(pattern.length >= 3) { - auto idx = pattern.indexOf("*"); + char[1] starString = star; + auto idx = pattern.indexOf(starString[]); if(idx != -1) { auto lhs = pattern[0 .. idx]; auto rhs = pattern[idx + 1 .. $]; @@ -6202,6 +7939,32 @@ class TaskCancelledException : object.Exception { } } +/+ +version(HasThread) private class ArsdThread : Thread { + this(void delegate() run) { + this.run = run; + + super(&runner); + } + + private void delegate() run; + final void runner() { + // FIXME: need to mask most signals so we don't handle things intended for the process as a whole + + try { + run(); + + // FIXME: post a thread complete notification to the supervisor + // supervisor should join it at that time + } catch(Throwable t) { + // FIXME: post a thread failed notification to the supervisor + // supervisor should join it at that time + throw t; + } + } +} ++/ + version(HasThread) private class CoreWorkerThread : Thread { this(EventLoopType type) { this.type = type; @@ -6372,14 +8135,112 @@ private class CoreEventLoopImplementation : ICoreEventLoop { } LoopIterationDelegate[] loopIterationDelegates; - void runLoopIterationDelegates() { + void runLoopIterationDelegates(bool isAfter) { foreach(lid; loopIterationDelegates) - lid.dg(); + if((!isAfter && (lid.flags & 1)) || (isAfter && (lid.flags & 2))) + lid.dg(); } } - void addDelegateOnLoopIteration(void delegate() dg, uint timingFlags) { + UnregisterToken addDelegateOnLoopIteration(void delegate() dg, uint timingFlags) { loopIterationDelegates ~= LoopIterationDelegate(dg, timingFlags); + UnregisterToken ut; + ut.impl = this; + ut.dg = dg; + return ut; + } + + void unregisterDg(void delegate() dg) { + LoopIterationDelegate[] toKeep; + foreach(lid; loopIterationDelegates) { + if(lid.dg !is dg) { + toKeep ~= lid; + } + } + loopIterationDelegates = toKeep; + } + + version(Arsd_core_dispatch) { + + private NSRunLoop ttrl; + + private this() { + ttrl = NSRunLoop.currentRunLoop; + } + + // FIXME: this lies!! it runs until completion + RunOnceResult runOnce(Duration timeout = Duration.max) { + scope(exit) eventLoopRound++; + + // FIXME: autorelease pool + + if(false /*isWorker*/) { + runLoopIterationDelegates(false); + + // FIXME: timeout is wrong + auto retValue = ttrl.runMode(NSDefaultRunLoopMode, /+beforeDate:+/ NSDate.distantFuture); + if(retValue == false) + throw new Exception("could not start run loop"); + + runLoopIterationDelegates(true); + + // NSApp.run(); + // exitApplication(); + //return RunOnceResult(RunOnceResult.Possibilities.GlobalExit); + return RunOnceResult(RunOnceResult.Possibilities.CarryOn); + } else { + // ui thread needs to pump nsapp events... + runLoopIterationDelegates(false); + + auto timeoutNs = NSDate.distantFuture; // FIXME timeout here, future means no timeout + + again: + NSEvent event = NSApp.nextEventMatchingMask( + NSEventMask.NSEventMaskAny, + timeoutNs, + NSDefaultRunLoopMode, + true + ); + if(event !is null) { + NSApp.sendEvent(event); + timeoutNs = NSDate.distantPast; // only keep going if it won't block; we just want to clear the queue + goto again; + } + + runLoopIterationDelegates(true); + return RunOnceResult(RunOnceResult.Possibilities.CarryOn); + } + } + + UnregisterToken addCallbackOnFdReadable(int fd, CallbackHelper cb) { + auto input_src = dispatch_source_create(DISPATCH_SOURCE_TYPE_READ, fd, 0, dispatch_get_main_queue()); + // FIXME: can the GC reap this prematurely? + auto b = block(() { + cb.call(); + }); + // FIXME: should prolly free it eventually idk + import core.memory; + GC.addRoot(b); + + dispatch_source_set_event_handler(input_src, b); + // dispatch_source_set_cancel_handler(input_src, ^{ close(my_file); }); + dispatch_resume(input_src); + + return UnregisterToken(this, fd, cb); + + } + RearmToken addCallbackOnFdReadableOneShot(int fd, CallbackHelper cb) { + throw new NotYetImplementedException(); + } + RearmToken addCallbackOnFdWritableOneShot(int fd, CallbackHelper cb) { + throw new NotYetImplementedException(); + } + private void rearmFd(RearmToken token) { + if(token.readable) + cast(void) addCallbackOnFdReadableOneShot(token.fd, token.cb); + else + cast(void) addCallbackOnFdWritableOneShot(token.fd, token.cb); + } } version(Arsd_core_kqueue) { @@ -6388,6 +8249,9 @@ private class CoreEventLoopImplementation : ICoreEventLoop { RunOnceResult runOnce(Duration timeout = Duration.max) { scope(exit) eventLoopRound++; + + runLoopIterationDelegates(false); + kevent_t[16] ev; //timespec tout = timespec(1, 0); auto nev = kevent(kqueuefd, null, 0, ev.ptr, ev.length, null/*&tout*/); @@ -6410,7 +8274,7 @@ private class CoreEventLoopImplementation : ICoreEventLoop { } } - runLoopIterationDelegates(); + runLoopIterationDelegates(true); return RunOnceResult(RunOnceResult.Possibilities.CarryOn); } @@ -6557,13 +8421,16 @@ private class CoreEventLoopImplementation : ICoreEventLoop { RunOnceResult runOnce(Duration timeout = Duration.max) { scope(exit) eventLoopRound++; + + runLoopIterationDelegates(false); + if(isWorker) { // this function is only supported on Windows Vista and up, so using this // means dropping support for XP. //GetQueuedCompletionStatusEx(); assert(0); // FIXME } else { - auto wto = 0; + auto wto = getTimeout(); auto waitResult = MsgWaitForMultipleObjectsEx( cast(int) handles.length, handles.ptr, @@ -6607,7 +8474,7 @@ private class CoreEventLoopImplementation : ICoreEventLoop { } } - runLoopIterationDelegates(); + runLoopIterationDelegates(true); return RunOnceResult(RunOnceResult.Possibilities.CarryOn); } @@ -6621,10 +8488,12 @@ private class CoreEventLoopImplementation : ICoreEventLoop { if(cas(&sigChildHappened, 1, 0)) { while(true) { // multiple children could have exited before we processed the notification + // Means child stopped, terminated, or continued. Not necessarily just terminated! + import core.sys.posix.sys.wait; int status; - auto pid = waitpid(-1, &status, WNOHANG); + auto pid = waitpid(-1, &status, WNOHANG | WUNTRACED | WCONTINUED); if(pid == -1) { import core.stdc.errno; auto errno = errno; @@ -6642,7 +8511,7 @@ private class CoreEventLoopImplementation : ICoreEventLoop { // to wake up so it can check its waitForCompletion, // trigger its callbacks, etc. - ExternalProcess.recordChildTerminated(pid, status); + ExternalProcess.recordChildChanged(pid, status); } } @@ -6757,7 +8626,7 @@ private class CoreEventLoopImplementation : ICoreEventLoop { } - private static final class CallbackQueue { + private static final class CallbackQueue : SynchronizableObject { int fd = -1; string name; CallbackHelper callback; @@ -6893,8 +8762,11 @@ private class CoreEventLoopImplementation : ICoreEventLoop { RunOnceResult runOnce(Duration timeout = Duration.max) { scope(exit) eventLoopRound++; + + runLoopIterationDelegates(false); + epoll_event[16] events; - auto ret = epoll_wait(epollfd, events.ptr, cast(int) events.length, -1); // FIXME: timeout + auto ret = epoll_wait(epollfd, events.ptr, cast(int) events.length, getTimeout ? getTimeout() : -1); // FIXME: timeout argument if(ret == -1) { import core.stdc.errno; if(errno == EINTR) { @@ -6916,7 +8788,7 @@ private class CoreEventLoopImplementation : ICoreEventLoop { } } - runLoopIterationDelegates(); + runLoopIterationDelegates(true); return RunOnceResult(RunOnceResult.Possibilities.CarryOn); } @@ -7091,14 +8963,14 @@ struct SynchronizedCircularBuffer(T, size_t maxSize = 128) { private int front; private int back; - private Object synchronizedOn; + private SynchronizableObject synchronizedOn; @disable this(); /++ The Object's monitor is used to synchronize the methods in here. +/ - this(Object synchronizedOn) { + this(SynchronizableObject synchronizedOn) { this.synchronizedOn = synchronizedOn; } @@ -7182,7 +9054,7 @@ struct SynchronizedCircularBuffer(T, size_t maxSize = 128) { } unittest { - Object object = new Object(); + SynchronizableObject object = new SynchronizableObject(); auto queue = SynchronizedCircularBuffer!CallbackHelper(object); assert(queue.isEmpty); foreach(i; 0 .. queue.ring.length - 1) @@ -7554,6 +9426,15 @@ unittest { // stream.feedData([1,2,3,4,1,2,3,4,1,2,3,4]); } +private char[] asciiToUpper(scope const(char)[] s) pure { + char[] copy = s.dup; + foreach(ref ch; copy) { + if(ch >= 'a' && ch <= 'z') + ch -= 32; + } + return copy; +} + /++ UNSTABLE, NOT FULLY IMPLEMENTED. DO NOT USE YET. @@ -7592,12 +9473,41 @@ class ExternalProcess /*: AsyncOperationRequest*/ { } } - void recordChildTerminated(pid_t pid, int status) { + void recordChildChanged(pid_t pid, int status) { synchronized(typeid(ExternalProcess)) { if(pid in activeChildren) { auto ac = activeChildren[pid]; - ac.markComplete(status); - activeChildren.remove(pid); + + // import unix = core.sys.posix.unistd; unix.write(1, "SIGCHLD\n".ptr, 8); + + import core.sys.posix.sys.wait; + if(WIFEXITED(status)) { + // exited normally + ac.markComplete(WEXITSTATUS(status)); + activeChildren.remove(pid); + } else if(WIFSIGNALED(status)) { + // terminated by signal + + // version(linux) import core.sys.linux.sys.wait : WCOREDUMP; + + bool coredumped; + static if(is(typeof(WCOREDUMP))) { + if(WCOREDUMP(status)) { + coredumped = true; + } + } + + ac.markTerminatedBySignal(WTERMSIG(status), coredumped); + activeChildren.remove(pid); + } else if(WIFSTOPPED(status)) { + // stopped by signal + ac.markStoppedBySignal(WSTOPSIG(status)); + } else if(WIFCONTINUED(status)) { + // continued by SIGCONT + ac.markContinued(); + } else { + // unknown condition...... + } } } } @@ -7649,6 +9559,127 @@ class ExternalProcess /*: AsyncOperationRequest*/ { } /++ + This allows you to record a process as existing to the core event loop, + so you get completion and other notifications on it, but without doing any + other processing. The process should already exist as a child of our main process + and you should not attempt to use any of the i/o files on it, as they will be null. + + History: + Added December 18, 2025 + +/ + version(Posix) + this(pid_t recordForMinimalWrapping) { + recordChildCreated(recordForMinimalWrapping, this); + } + + version(Posix) + package { + // if you use the override thing, it is YOUR responsibility to close them! + int overrideStdin = -2; + int overrideStdout = -2; + int overrideStderr = -2; + int pgid = -2; + + const(char*)* environment; + + // FIXME: change it to string[string] + // it will modify the passed AA + void setEnvironmentWithModifications(string[string] mods) @system { + const(char*)[] ret; + + const(char*)* head = environ; + while(*head) { + auto headz = stringz(*head); + // see if head and any of the mods are the same var + auto headd = headz.borrow; + auto equal = headd.indexOf("="); + if(equal == -1) + equal = cast(int) headd.length; + auto name = headd[0 .. equal]; + if(name in mods) { + ret ~= cast(char*) (name ~ "=" ~ mods[name] ~ "\0").ptr; + mods.remove(cast(string) name); + } else { + ret ~= *head; + } + + head++; + } + + // append the remainder of mods to the ret + foreach(name, value; mods) + ret ~= cast(char*) (name ~ "=" ~ value ~ "\0").ptr; + + ret ~= null; + + environment = ret.ptr; + } + } + + version(Windows) + package { + // if you use the override thing, it is YOUR responsibility to close them! + HANDLE overrideStdin = INVALID_HANDLE_VALUE; + HANDLE overrideStdout = INVALID_HANDLE_VALUE; + HANDLE overrideStderr = INVALID_HANDLE_VALUE; + + wchar* environment; + + void setEnvironmentWithModifications(string[string] mods) @system { + wchar[] ret; + + // FIXME: case sensitivity in name lookup,the passed mods should all be uppercase + + // FIXME: "All strings in the environment block must be sorted alphabetically by name. The sort is case-insensitive, Unicode order, without regard to locale." + + auto originalEnv = GetEnvironmentStringsW(); + if(originalEnv is null) + throw new WindowsApiException("GetEnvironmentStringsW",GetLastError()); + scope(exit) { + if(originalEnv) + FreeEnvironmentStringsW(originalEnv); + } + + // read null terminated strings until we hit one of zero length + // create a new block of memory with the same data, but all copied + auto env = originalEnv; + more: + wchar* start = env; + while(*env) { + env++; + } + wchar[] wv = start[0 .. env - start]; + if(wv.length) { + string v = makeUtf8StringFromWindowsString(wv); + auto equal = v.indexOf("="); + if(equal == -1) + equal = cast(int) v.length; + auto name = v[0 .. equal].asciiToUpper; + + if(name in mods) { + WCharzBuffer bfr = (name ~ "=" ~ mods[name]); + ret ~= bfr.ptr[0 .. bfr.length + 1]; // to include the zero terminator + mods.remove(cast(string) name); + } else { + ret ~= start[0 .. env - start + 1]; // include zero terminator + } + + env++; // move past the zero terminator + goto more; + } + + foreach(name, mod; mods) { + WCharzBuffer bfr = (name ~ "=" ~ mod); + ret ~= bfr.ptr[0 .. bfr.length + 1]; // to include the zero terminator + } + + ret ~= 0; + + this.environment = ret.ptr; + } + } + + /++ +/ void start() { @@ -7658,40 +9689,52 @@ class ExternalProcess /*: AsyncOperationRequest*/ { int ret; int[2] stdinPipes; - ret = pipe(stdinPipes); - if(ret == -1) - throw new ErrnoApiException("stdin pipe", errno); + if(overrideStdin == -2) { + ret = pipe(stdinPipes); + if(ret == -1) + throw new ErrnoApiException("stdin pipe", errno); + } scope(failure) { - close(stdinPipes[0]); - close(stdinPipes[1]); + if(overrideStdin == -2) { + close(stdinPipes[0]); + close(stdinPipes[1]); + } } - auto stdinFd = stdinPipes[1]; + auto stdinFd = overrideStdin == -2 ? stdinPipes[1] : -1; int[2] stdoutPipes; - ret = pipe(stdoutPipes); - if(ret == -1) - throw new ErrnoApiException("stdout pipe", errno); + if(overrideStdout == -2) { + ret = pipe(stdoutPipes); + if(ret == -1) + throw new ErrnoApiException("stdout pipe", errno); + } scope(failure) { - close(stdoutPipes[0]); - close(stdoutPipes[1]); + if(overrideStdout == -2) { + close(stdoutPipes[0]); + close(stdoutPipes[1]); + } } - auto stdoutFd = stdoutPipes[0]; + auto stdoutFd = overrideStdout == -2 ? stdoutPipes[0] : -1; int[2] stderrPipes; - ret = pipe(stderrPipes); - if(ret == -1) - throw new ErrnoApiException("stderr pipe", errno); + if(overrideStderr == -2) { + ret = pipe(stderrPipes); + if(ret == -1) + throw new ErrnoApiException("stderr pipe", errno); + } scope(failure) { - close(stderrPipes[0]); - close(stderrPipes[1]); + if(overrideStderr == -2) { + close(stderrPipes[0]); + close(stderrPipes[1]); + } } - auto stderrFd = stderrPipes[0]; + auto stderrFd = overrideStderr == -2 ? stderrPipes[0] : -1; int[2] errorReportPipes; @@ -7707,10 +9750,10 @@ class ExternalProcess /*: AsyncOperationRequest*/ { setCloExec(errorReportPipes[0]); setCloExec(errorReportPipes[1]); + // writeln(pgid); auto forkRet = fork(); if(forkRet == -1) throw new ErrnoApiException("fork", errno); - if(forkRet == 0) { // child side @@ -7734,18 +9777,39 @@ class ExternalProcess /*: AsyncOperationRequest*/ { exit(1); } + // both parent and child are supposed to try to set it + if(pgid != -2) { + setpgid(0, pgid == 0 ? getpid() : pgid); + } + // dup2 closes the fd it is replacing automatically - dup2(stdinPipes[0], 0); - dup2(stdoutPipes[1], 1); - dup2(stderrPipes[1], 2); - - // don't need either of the original pipe fds anymore - close(stdinPipes[0]); - close(stdinPipes[1]); - close(stdoutPipes[0]); - close(stdoutPipes[1]); - close(stderrPipes[0]); - close(stderrPipes[1]); + // then don't need either of the original pipe fds anymore + if(overrideStdin == -2) { + dup2(stdinPipes[0], 0); + close(stdinPipes[0]); + close(stdinPipes[1]); + } else if(overrideStdin != 0) { + dup2(overrideStdin, 0); + close(overrideStdin); + } + + if(overrideStdout == -2) { + dup2(stdoutPipes[1], 1); + close(stdoutPipes[0]); + close(stdoutPipes[1]); + } else if(overrideStdout != 1) { + dup2(overrideStdout, 1); + close(overrideStdout); + } + + if(overrideStderr == -2) { + dup2(stderrPipes[1], 2); + close(stderrPipes[0]); + close(stderrPipes[1]); + } else if(overrideStderr != 2) { + dup2(overrideStderr, 2); + close(overrideStderr); + } // the error reporting pipe will be closed upon exec since we set cloexec before fork // and everything else should have cloexec set too hopefully. @@ -7769,7 +9833,7 @@ class ExternalProcess /*: AsyncOperationRequest*/ { } argv[args.length] = null; - auto rete = execvp/*e*/(file, argv.ptr/*, envp*/); + auto rete = execve(file, argv.ptr, this.environment is null ? environ : this.environment); // FIXME: i used to use execvp, which searches path but i think i like this more if(rete == -1) { fail(4); } else { @@ -7779,6 +9843,11 @@ class ExternalProcess /*: AsyncOperationRequest*/ { } else { pid = forkRet; + // both parent and child are supposed to try to set it + if(pgid != -2) { + setpgid(pid, pgid == 0 ? pid : pgid); + } + recordChildCreated(pid, this); // close our copy of the write side of the error reporting pipe @@ -7787,10 +9856,14 @@ class ExternalProcess /*: AsyncOperationRequest*/ { int[2] msg; // this will block to wait for it to actually either start up or fail to exec (which should be near instant) + try_again: auto val = read(errorReportPipes[0], msg.ptr, msg.sizeof); - if(val == -1) + if(val == -1) { + if(errno == EINTR) + goto try_again; throw new ErrnoApiException("read error report", errno); + } if(val == msg.sizeof) { // error happened @@ -7803,25 +9876,29 @@ class ExternalProcess /*: AsyncOperationRequest*/ { } // set the ones we keep to close upon future execs - // FIXME should i set NOBLOCK at this time too? prolly should - setCloExec(stdinPipes[1]); - setCloExec(stdoutPipes[0]); - setCloExec(stderrPipes[0]); - // and close the others - ErrnoEnforce!close(stdinPipes[0]); - ErrnoEnforce!close(stdoutPipes[1]); - ErrnoEnforce!close(stderrPipes[1]); + if(overrideStdin == -2) { + setCloExec(stdinPipes[1]); + ErrnoEnforce!close(stdinPipes[0]); + makeNonBlocking(stdinFd); + _stdin = new AsyncFile(stdinFd); + } - ErrnoEnforce!close(errorReportPipes[0]); + if(overrideStdout == -2) { + setCloExec(stdoutPipes[0]); + ErrnoEnforce!close(stdoutPipes[1]); + makeNonBlocking(stdoutFd); + _stdout = new AsyncFile(stdoutFd); + } - makeNonBlocking(stdinFd); - makeNonBlocking(stdoutFd); - makeNonBlocking(stderrFd); + if(overrideStderr == -2) { + setCloExec(stderrPipes[0]); + ErrnoEnforce!close(stderrPipes[1]); + makeNonBlocking(stderrFd); + _stderr = new AsyncFile(stderrFd); + } - _stdin = new AsyncFile(stdinFd); - _stdout = new AsyncFile(stdoutFd); - _stderr = new AsyncFile(stderrFd); + ErrnoEnforce!close(errorReportPipes[0]); } } else version(Windows) { WCharzBuffer program = this.program.path; @@ -7837,30 +9914,60 @@ class ExternalProcess /*: AsyncOperationRequest*/ { HANDLE inreadPipe; HANDLE inwritePipe; - if(MyCreatePipeEx(&inreadPipe, &inwritePipe, &saAttr, 0, 0, FILE_FLAG_OVERLAPPED) == 0) - throw new WindowsApiException("CreatePipe", GetLastError()); - if(!SetHandleInformation(inwritePipe, 1/*HANDLE_FLAG_INHERIT*/, 0)) - throw new WindowsApiException("SetHandleInformation", GetLastError()); + + if(overrideStdin == INVALID_HANDLE_VALUE) { + if(MyCreatePipeEx(&inreadPipe, &inwritePipe, &saAttr, 0, 0, FILE_FLAG_OVERLAPPED) == 0) + throw new WindowsApiException("CreatePipe", GetLastError()); + if(!SetHandleInformation(inwritePipe, 1/*HANDLE_FLAG_INHERIT*/, 0)) + throw new WindowsApiException("SetHandleInformation", GetLastError()); + } + + scope(failure) { + if(overrideStdin == INVALID_HANDLE_VALUE) { + CloseHandle(inreadPipe); + CloseHandle(inwritePipe); + } + } HANDLE outreadPipe; HANDLE outwritePipe; - if(MyCreatePipeEx(&outreadPipe, &outwritePipe, &saAttr, 0, FILE_FLAG_OVERLAPPED, 0) == 0) - throw new WindowsApiException("CreatePipe", GetLastError()); - if(!SetHandleInformation(outreadPipe, 1/*HANDLE_FLAG_INHERIT*/, 0)) - throw new WindowsApiException("SetHandleInformation", GetLastError()); + if(overrideStdout == INVALID_HANDLE_VALUE) { + if(MyCreatePipeEx(&outreadPipe, &outwritePipe, &saAttr, 0, FILE_FLAG_OVERLAPPED, 0) == 0) + throw new WindowsApiException("CreatePipe", GetLastError()); + if(!SetHandleInformation(outreadPipe, 1/*HANDLE_FLAG_INHERIT*/, 0)) + throw new WindowsApiException("SetHandleInformation", GetLastError()); + } + + scope(failure) { + if(overrideStdout == INVALID_HANDLE_VALUE) { + CloseHandle(outreadPipe); + CloseHandle(outwritePipe); + } + } + HANDLE errreadPipe; HANDLE errwritePipe; - if(MyCreatePipeEx(&errreadPipe, &errwritePipe, &saAttr, 0, FILE_FLAG_OVERLAPPED, 0) == 0) - throw new WindowsApiException("CreatePipe", GetLastError()); - if(!SetHandleInformation(errreadPipe, 1/*HANDLE_FLAG_INHERIT*/, 0)) - throw new WindowsApiException("SetHandleInformation", GetLastError()); + if(overrideStderr == INVALID_HANDLE_VALUE) { + if(MyCreatePipeEx(&errreadPipe, &errwritePipe, &saAttr, 0, FILE_FLAG_OVERLAPPED, 0) == 0) + throw new WindowsApiException("CreatePipe", GetLastError()); + if(!SetHandleInformation(errreadPipe, 1/*HANDLE_FLAG_INHERIT*/, 0)) + throw new WindowsApiException("SetHandleInformation", GetLastError()); + } + + scope(failure) { + if(overrideStderr == INVALID_HANDLE_VALUE) { + CloseHandle(errreadPipe); + CloseHandle(errwritePipe); + } + } + startupInfo.cb = startupInfo.sizeof; startupInfo.dwFlags = STARTF_USESTDHANDLES; - startupInfo.hStdInput = inreadPipe; - startupInfo.hStdOutput = outwritePipe; - startupInfo.hStdError = errwritePipe; + startupInfo.hStdInput = (overrideStdin == INVALID_HANDLE_VALUE) ? inreadPipe : overrideStdin; + startupInfo.hStdOutput = (overrideStdout == INVALID_HANDLE_VALUE) ? outwritePipe : overrideStdout; + startupInfo.hStdError = (overrideStderr == INVALID_HANDLE_VALUE) ? errwritePipe : overrideStderr; auto result = CreateProcessW( program.ptr, @@ -7868,8 +9975,8 @@ class ExternalProcess /*: AsyncOperationRequest*/ { null, // process attributes null, // thread attributes true, // inherit handles; necessary for the std in/out/err ones to work - 0, // dwCreationFlags FIXME might be useful to change - null, // environment, might be worth changing + this.environment is null ? 0 : CREATE_UNICODE_ENVIRONMENT, // dwCreationFlags FIXME might be useful to change + this.environment, // environment, might be worth changing null, // current directory &startupInfo, &pi @@ -7878,13 +9985,18 @@ class ExternalProcess /*: AsyncOperationRequest*/ { if(!result) throw new WindowsApiException("CreateProcessW", GetLastError()); - _stdin = new AsyncFile(inwritePipe); - _stdout = new AsyncFile(outreadPipe); - _stderr = new AsyncFile(errreadPipe); - - Win32Enforce!CloseHandle(inreadPipe); - Win32Enforce!CloseHandle(outwritePipe); - Win32Enforce!CloseHandle(errwritePipe); + if(overrideStdin == INVALID_HANDLE_VALUE) { + _stdin = new AsyncFile(inwritePipe); + Win32Enforce!CloseHandle(inreadPipe); + } + if(overrideStdout == INVALID_HANDLE_VALUE) { + _stdout = new AsyncFile(outreadPipe); + Win32Enforce!CloseHandle(outwritePipe); + } + if(overrideStderr == INVALID_HANDLE_VALUE) { + _stderr = new AsyncFile(errreadPipe); + Win32Enforce!CloseHandle(errwritePipe); + } Win32Enforce!CloseHandle(pi.hThread); @@ -7914,7 +10026,7 @@ class ExternalProcess /*: AsyncOperationRequest*/ { import core.sys.posix.unistd; import core.sys.posix.fcntl; - private pid_t pid = -1; + package pid_t pid = -1; public void delegate() beforeExec; @@ -7929,7 +10041,22 @@ class ExternalProcess /*: AsyncOperationRequest*/ { if(oncomplete) oncomplete(this); } + private final void markTerminatedBySignal(int signal, bool coredumped) { + completed = true; + _status = -signal; + this.coredumped = coredumped; + if(oncomplete) + oncomplete(this); + } + private final void markStoppedBySignal(int signal) { + stopped = true; + _status = -signal; + } + private final void markContinued() { + stopped = false; + _status = int.min; + } private AsyncFile _stdin; private AsyncFile _stdout; @@ -7957,11 +10084,36 @@ class ExternalProcess /*: AsyncOperationRequest*/ { } /++ + History: + Added November 23, 2025 + +/ + void waitForChange() { + bool stoppedAtFirst = isStopped; + getThisThreadEventLoop().run(() { return this.isComplete || (stoppedAtFirst != this.isStopped); }); + } + + /++ +/ bool isComplete() { return completed; } + /++ + History: + Added November 23, 2025 + +/ + bool isStopped() { + return stopped; + } + /++ + History: + Added November 23, 2025 + +/ + bool leftCoreDump() { + return coredumped; + } + private bool coredumped; + private bool stopped; private bool completed; private int _status = int.min; @@ -8082,7 +10234,7 @@ unittest { Not actually implemented until February 6, 2025, when it changed from mixin template to class. +/ -class LoggerOf(T, size_t bufferSize = 16) { +class LoggerOf(T, size_t bufferSize = 16) : SynchronizableObject { private LoggedMessage!T[bufferSize] ring; private ulong writeBufferPosition; @@ -8185,7 +10337,13 @@ class LoggerOf(T, size_t bufferSize = 16) { logger.condition.notifyAll(); } // mark us as complete for other listeners waiting as well - event.set(); + static if (__traits(hasMember, event, "setIfInitialized")) { + // Upstream compatibility, see <https://github.com/dlang/dmd/pull/15800>. + event.setIfInitialized(); + } else { + // Old D runtime compatibility + event.set(); + } } +/ @@ -8196,7 +10354,7 @@ class LoggerOf(T, size_t bufferSize = 16) { int missedMessages = 0; long n; synchronized(logger) { - while(logger.active && connected && logger.writeBufferPosition < readBufferPosition) { + while(logger.active && connected && logger.writeBufferPosition <= readBufferPosition) { logger.condition.wait(); } @@ -8210,15 +10368,15 @@ class LoggerOf(T, size_t bufferSize = 16) { auto startPos = readBufferPosition % bufferSize; auto endPos = us.writeBufferPosition % bufferSize; if(endPos > startPos) { - buffer[0 .. n] = us.ring[startPos .. endPos]; + buffer[0 .. cast(size_t) n] = us.ring[cast(size_t) startPos .. cast(size_t) endPos]; } else { auto ourSplit = us.ring.length - startPos; - buffer[0 .. ourSplit] = us.ring[startPos .. $]; - buffer[ourSplit .. ourSplit + endPos] = us.ring[0 .. endPos]; + buffer[0 .. cast(size_t) ourSplit] = us.ring[cast(size_t) startPos .. $]; + buffer[cast(size_t) ourSplit .. cast(size_t) (ourSplit + endPos)] = us.ring[0 .. cast(size_t) endPos]; } readBufferPosition = us.writeBufferPosition; } - foreach(item; buffer[0 .. n]) { + foreach(item; buffer[0 .. cast(size_t) n]) { if(!connected) break; dg(item, missedMessages); @@ -8236,7 +10394,13 @@ class LoggerOf(T, size_t bufferSize = 16) { logger.condition.notifyAll(); } // mark us as complete for other listeners waiting as well - event.set(); + static if (__traits(hasMember, event, "setIfInitialized")) { + // Upstream compatibility, see <https://github.com/dlang/dmd/pull/15800>. + event.setIfInitialized(); + } else { + // Old D runtime compatibility + event.set(); + } } @@ -8382,6 +10546,16 @@ shared(LoggerOf!GenericEmbeddableInterpolatedSequence) logger() { return _commonLogger; } +/++ + Makes note of an exception you catch and otherwise ignore. + + History: + Added April 17, 2025 ++/ +void logSwallowedException(Exception e) { + logger.error(InterpolationHeader(), e.toString(), InterpolationFooter()); +} + /+ // using this requires a newish compiler so we just uncomment when necessary unittest { @@ -8649,103 +10823,289 @@ private void appendToBuffer(ref char[] buffer, ref int pos, scope const(char)[] } private void appendToBuffer(ref char[] buffer, ref int pos, long what) { - if(buffer.length < pos + 16) - buffer.length = pos + 16; - auto sliced = intToString(what, buffer[pos .. $]); + appendToBuffer(buffer, pos, what, IntToStringArgs.init); +} +private void appendToBuffer(ref char[] buffer, ref int pos, long what, IntToStringArgs args) { + if(buffer.length < pos + 32) + buffer.length = pos + 32; + auto sliced = intToString(what, buffer[pos .. $], args); + pos += sliced.length; +} + +private void appendToBuffer(ref char[] buffer, ref int pos, double what) { + appendToBuffer(buffer, pos, what, FloatToStringArgs.init); +} +private void appendToBuffer(ref char[] buffer, ref int pos, double what, FloatToStringArgs args) { + if(buffer.length < pos + 42) + buffer.length = pos + 42; + auto sliced = floatToString(what, buffer[pos .. $], args); pos += sliced.length; } + +/++ + You can use `mixin(dumpParams);` to put out a debug print of your current function call w/ params. ++/ +enum string dumpParams = q{ + { + import arsd.core; + arsd.core.dumpParamsImpl(__FUNCTION__, __traits(parameters)); + } +}; + +/// Don't call this directly, use `mixin(dumpParams);` instead +public void dumpParamsImpl(T...)(string func, T args) { + char[256] bufferBacking; + writeGuts(bufferBacking[], func ~ "(", ")\n", ", ", false, true, &actuallyWriteToStdout, args); +} + /++ - A `writeln` that actually works, at least for some basic types. + A `writeln` (and friends) that actually works, at least for some basic types. - It works correctly on Windows, using the correct functions to write unicode to the console. even allocating a console if needed. If the output has been redirected to a file or pipe, it writes UTF-8. + It works correctly on Windows, using the correct functions to write unicode to the console, even allocating a console if needed. If the output has been redirected to a file or pipe, it writes UTF-8. This always does text. See also WritableStream and WritableTextStream when they are implemented. +/ -void writeln(bool printInterpolatedCode = false, T...)(T t) { +void writeln(T...)(T t) { + char[256] bufferBacking; + writeGuts(bufferBacking[], null, "\n", null, false, false, &actuallyWriteToStdout, t); +} + +/// ditto +alias writelnStdOut = writeln; + +/// ditto +void writelnStderr(T...)(T t) { + char[256] bufferBacking; + writeGuts(bufferBacking[], null, "\n", null, false, false, &actuallyWriteToStderr, t); +} + +/// ditto +void writeStdout(T...)(T t) { + char[256] bufferBacking; + writeGuts(bufferBacking[], null, null, null, false, false, &actuallyWriteToStdout, t); +} + +/// ditto +void writeStderr(T...)(T t) { char[256] bufferBacking; - char[] buffer = bufferBacking[]; + writeGuts(bufferBacking[], null, null, null, false, false, &actuallyWriteToStderr, t); +} + +struct ValueWithFormattingArgs(T : double) { + double value; + FloatToStringArgs args; +} + +struct ValueWithFormattingArgs(T : long) { + long value; + IntToStringArgs args; +} + +ValueWithFormattingArgs!double formatArgs(double value, FloatToStringArgs args) { + return ValueWithFormattingArgs!double(value, args); +} +ValueWithFormattingArgs!double formatArgs(double value, int precision) { + return ValueWithFormattingArgs!double(value, FloatToStringArgs().withPrecision(precision)); + +} + +unittest { + assert(toStringInternal(5.4364.formatArgs(FloatToStringArgs().withPrecision(2))) == "5.44"); + assert(toStringInternal(5.4364.formatArgs(precision: 2)) == "5.44"); +} + +/++ + ++/ +package(arsd) string enumNameForValue(T)(T t) { + switch(t) { + foreach(memberName; __traits(allMembers, T)) { + case __traits(getMember, T, memberName): + return memberName; + } + default: + return "<unknown>"; + } +} + +/+ + Purposes: + * debugging + * writing + * converting single value to string? ++/ +private string writeGuts(T...)(char[] buffer, string prefix, string suffix, string argSeparator, bool printInterpolatedCode, bool quoteStrings, string function(scope char[] result) writer, T t) { int pos; - foreach(arg; t) { - static if(is(typeof(arg) Base == enum)) { - appendToBuffer(buffer, pos, typeof(arg).stringof); - appendToBuffer(buffer, pos, "."); - appendToBuffer(buffer, pos, toStringInternal(arg)); - appendToBuffer(buffer, pos, "("); - appendToBuffer(buffer, pos, cast(Base) arg); - appendToBuffer(buffer, pos, ")"); - } else static if(is(typeof(arg) : const char[])) { - appendToBuffer(buffer, pos, arg); - } else static if(is(typeof(arg) : stringz)) { - appendToBuffer(buffer, pos, arg.borrow); - } else static if(is(typeof(arg) : long)) { - appendToBuffer(buffer, pos, arg); - } else static if(is(typeof(arg) : double)) { - import core.stdc.stdio; - char[128] fb; - auto count = snprintf(fb.ptr, fb.length, "%.4lf", arg); - - appendToBuffer(buffer, pos, fb[0 .. count]); - } else static if(is(typeof(arg) == InterpolatedExpression!code, string code)) { - static if(printInterpolatedCode) { + if(prefix.length) + appendToBuffer(buffer, pos, prefix); + + foreach(i, arg; t) { + static if(i) + if(argSeparator.length) + appendToBuffer(buffer, pos, argSeparator); + + static if(is(typeof(arg) == InterpolatedExpression!code, string code)) { + if(printInterpolatedCode) { appendToBuffer(buffer, pos, code); appendToBuffer(buffer, pos, " = "); } - } else static if(is(typeof(arg.toString()) : const char[])) { - appendToBuffer(buffer, pos, arg.toString()); - } else static if(is(typeof(arg) A == struct)) { - appendToBuffer(buffer, pos, A.stringof); - appendToBuffer(buffer, pos, "("); - foreach(idx, item; arg.tupleof) { - if(idx) - appendToBuffer(buffer, pos, ", "); - appendToBuffer(buffer, pos, __traits(identifier, arg.tupleof[idx])); - appendToBuffer(buffer, pos, ": "); - appendToBuffer(buffer, pos, item); - } - appendToBuffer(buffer, pos, ")"); - } else static if(is(typeof(arg) == E[], E)) { - appendToBuffer(buffer, pos, "["); - foreach(idx, item; arg) { - if(idx) - appendToBuffer(buffer, pos, ", "); - appendToBuffer(buffer, pos, item); - } - appendToBuffer(buffer, pos, "]"); } else { - appendToBuffer(buffer, pos, "<" ~ typeof(arg).stringof ~ ">"); + writeIndividualArg(buffer, pos, quoteStrings, arg); } } - appendToBuffer(buffer, pos, "\n"); + if(suffix.length) + appendToBuffer(buffer, pos, suffix); - actuallyWriteToStdout(buffer[0 .. pos]); + return writer(buffer[0 .. pos]); } -debug void dump(T...)(T t, string file = __FILE__, size_t line = __LINE__) { - writeln!true(file, ":", line, ": ", t); +private void writeIndividualArg(T)(ref char[] buffer, ref int pos, bool quoteStrings, T arg) { + static if(is(typeof(arg) == ValueWithFormattingArgs!V, V)) { + appendToBuffer(buffer, pos, arg.value, arg.args); + } else static if(is(typeof(arg) Base == enum)) { + appendToBuffer(buffer, pos, typeof(arg).stringof); + appendToBuffer(buffer, pos, "."); + appendToBuffer(buffer, pos, enumNameForValue(arg)); + appendToBuffer(buffer, pos, "("); + appendToBuffer(buffer, pos, cast(Base) arg); + appendToBuffer(buffer, pos, ")"); + } else static if(is(typeof(arg) : const char[])) { + if(quoteStrings) { + appendToBuffer(buffer, pos, "\""); + appendToBuffer(buffer, pos, arg); // FIXME: escape quote and backslash in there? + appendToBuffer(buffer, pos, "\""); + } else { + appendToBuffer(buffer, pos, arg); + } + } else static if(is(typeof(arg) : stringz)) { + appendToBuffer(buffer, pos, arg.borrow); + } else static if(is(typeof(arg) : long)) { + appendToBuffer(buffer, pos, arg); + } else static if(is(typeof(arg) : double)) { + appendToBuffer(buffer, pos, arg); + } else static if(is(typeof(arg.toString()) : const char[])) { + appendToBuffer(buffer, pos, arg.toString()); + } else static if(is(typeof(arg) A == struct)) { + appendToBuffer(buffer, pos, A.stringof); + appendToBuffer(buffer, pos, "("); + foreach(idx, item; arg.tupleof) { + if(idx) + appendToBuffer(buffer, pos, ", "); + appendToBuffer(buffer, pos, __traits(identifier, arg.tupleof[idx])); + appendToBuffer(buffer, pos, ": "); + writeIndividualArg(buffer, pos, true, item); + } + appendToBuffer(buffer, pos, ")"); + } else static if(is(typeof(arg) == E[], E)) { + appendToBuffer(buffer, pos, "["); + foreach(idx, item; arg) { + if(idx) + appendToBuffer(buffer, pos, ", "); + writeIndividualArg(buffer, pos, true, item); + } + appendToBuffer(buffer, pos, "]"); + } else static if(is(typeof(arg) == delegate)) { + appendToBuffer(buffer, pos, "<" ~ typeof(arg).stringof ~ "> "); + appendToBuffer(buffer, pos, cast(size_t) arg.ptr, IntToStringArgs().withRadix(16).withPadding(12, '0')); + appendToBuffer(buffer, pos, ", "); + appendToBuffer(buffer, pos, cast(size_t) arg.funcptr, IntToStringArgs().withRadix(16).withPadding(12, '0')); + } else static if(is(typeof(arg) : const void*)) { + appendToBuffer(buffer, pos, "<" ~ typeof(arg).stringof ~ "> "); + appendToBuffer(buffer, pos, cast(size_t) arg, IntToStringArgs().withRadix(16).withPadding(12, '0')); + } else { + appendToBuffer(buffer, pos, "<" ~ typeof(arg).stringof ~ ">"); + } +} + +debug string inspect(T)(T t, string varName = null, int indent = 0) { + string str; + foreach(i; 0 .. indent) + str ~= "\t"; + if(varName.length) { + str ~= varName; + str ~= ": "; + } + str ~= T.stringof; + str ~= "("; + // hack for phobos nullable + static if(is(T == struct) && __traits(identifier, T) == "Nullable") { + if(t.isNull) { + str ~= "null)"; + } else { + str ~= "\n"; + str ~= inspect(t.get(), null, indent + 1); + foreach(i; 0 .. indent) + str ~= "\t"; + str ~= ")"; + } + } + else + // generic inspection + static if(is(T == class) || is(T == struct) || is(T == interface)) { + str ~= "\n"; + foreach(memberName; __traits(allMembers, T)) + static if(is(typeof(__traits(getMember, t, memberName).offsetof))) + { + str ~= inspect(__traits(getMember, t, memberName), memberName, indent + 1); + } + foreach(i; 0 .. indent) + str ~= "\t"; + str ~= ")"; + } else { + str ~= toStringInternal(t); + str ~= ")"; + } + + str ~= "\n"; + + return str; } -private void actuallyWriteToStdout(scope char[] buffer) @trusted { +debug void dump(T...)(T t, string file = __FILE__, size_t line = __LINE__) { + string separator; + static if(T.length && is(T[0] == InterpolationHeader)) + separator = null; + else + separator = "; "; + + char[256] bufferBacking; + writeGuts(bufferBacking[], file ~ ":" ~ toStringInternal(line) ~ ": ", "\n", separator, true, true, &actuallyWriteToStdout, t); +} +private string makeString(scope char[] buffer) @safe { + return buffer.idup; +} +private string makeStringCasting(scope /*return*/ char[] buffer) @system @nogc nothrow pure { + return cast(string) buffer; +} +private string actuallyWriteToStdout(scope char[] buffer) @safe { + return actuallyWriteToStdHandle(1, buffer); +} +private string actuallyWriteToStderr(scope char[] buffer) @safe { + return actuallyWriteToStdHandle(2, buffer); +} +private string actuallyWriteToStdHandle(int whichOne, scope char[] buffer) @trusted { version(UseStdioWriteln) { import std.stdio; - writeln(buffer); + (whichOne == 1 ? stdout : stderr).writeln(buffer); } else version(Windows) { import core.sys.windows.wincon; - auto hStdOut = GetStdHandle(STD_OUTPUT_HANDLE); + auto h = whichOne == 1 ? STD_OUTPUT_HANDLE : STD_ERROR_HANDLE; + + auto hStdOut = GetStdHandle(h); if(hStdOut == null || hStdOut == INVALID_HANDLE_VALUE) { AllocConsole(); - hStdOut = GetStdHandle(STD_OUTPUT_HANDLE); + hStdOut = GetStdHandle(h); } if(GetFileType(hStdOut) == FILE_TYPE_CHAR) { - wchar[256] wbuffer; - auto toWrite = makeWindowsString(buffer, wbuffer, WindowsStringConversionFlags.convertNewLines); + WCharzBuffer toWrite = WCharzBuffer(buffer, WindowsStringConversionFlags.convertNewLines); DWORD written; WriteConsoleW(hStdOut, toWrite.ptr, cast(DWORD) toWrite.length, &written, null); @@ -8755,8 +11115,48 @@ private void actuallyWriteToStdout(scope char[] buffer) @trusted { } } else { import unix = core.sys.posix.unistd; - unix.write(1, buffer.ptr, buffer.length); + unix.write(whichOne, buffer.ptr, buffer.length); } + + return null; +} + +/++ + As the C function it calls, this is not thread safe. + + Returns: + `null` if `name` not found. Note this is distinct from an empty string. ++/ +string getEnvironmentVariable(scope const(char)[] name) { + version(Posix) { + import core.stdc.stdlib; + CharzBuffer namez = name; + auto e = getenv(namez.ptr); + if(e is null) + return null; + return stringz(e).borrow.idup; + } else version(Windows) { + WCharzBuffer namew = name; + wchar[128] staticBuffer; + wchar[] buffer = staticBuffer; + auto ret = GetEnvironmentVariableW(namew.ptr, buffer.ptr, cast(DWORD) buffer.length); + if(ret > buffer.length) { + buffer.length = ret; + ret = GetEnvironmentVariableW(namew.ptr, buffer.ptr, cast(DWORD) buffer.length); + } + if(ret == 0) { + auto err = GetLastError(); + if(err == ERROR_SUCCESS) { + return ""; + } else if(err == ERROR_ENVVAR_NOT_FOUND) { + return null; + } else { + throw new WindowsApiException("GetEnvironmentVariable", err); + } + } + + return makeUtf8StringFromWindowsString(buffer[0 .. ret]); + } else static assert(0); } /+ @@ -9153,12 +11553,14 @@ package(arsd) version(Windows) extern(Windows) { int WSARecvFrom(SOCKET, LPWSABUF, DWORD, LPDWORD, LPDWORD, sockaddr*, LPINT, LPOVERLAPPED, LPOVERLAPPED_COMPLETION_ROUTINE); } -package(arsd) version(OSXCocoa) { +package(arsd) version(UseCocoa) { /* Copy/paste chunk from Jacob Carlborg { */ // from https://raw.githubusercontent.com/jacob-carlborg/druntime/550edd0a64f0eb2c4f35d3ec3d88e26b40ac779e/src/core/stdc/clang_block.d // with comments stripped (see docs in the original link), code reformatted, and some names changed to avoid potential conflicts +// note these should always be passed by pointer! + import core.stdc.config; struct ObjCBlock(R = void, Params...) { private: @@ -9177,18 +11579,30 @@ private: this.isa = isa; this.flags = flags; this.invoke = invoke; - this.dg = dg; this.descriptor = &.objcblock_descriptor; + + // FIXME: is this needed or not? it could be held by the OS and not be visible to GC i think + // import core.memory; GC.addRoot(dg.ptr); + + this.dg = dg; } } -ObjCBlock!(R, Params) block(R, Params...)(R delegate(Params) dg) { +ObjCBlock!(R, Params) blockOnStack(R, Params...)(R delegate(Params) dg) { static if (Params.length == 0) - enum flags = 0x50000000; + enum flags = 0x50000000; else enum flags = 0x40000000; return ObjCBlock!(R, Params)(&_NSConcreteStackBlock, flags, &objcblock_invoke!(R, Params), dg); } +ObjCBlock!(R, Params)* block(R, Params...)(R delegate(Params) dg) { + static if (Params.length == 0) + enum flags = 0x50000000; + else + enum flags = 0x40000000; + + return new ObjCBlock!(R, Params)(&_NSConcreteStackBlock, flags, &objcblock_invoke!(R, Params), dg); +} private struct Descriptor { c_ulong reserved; @@ -9279,8 +11693,19 @@ If you are not sure if Cocoa thinks your application is multithreaded or not, yo void getCharacters(wchar* buffer, NSRange range) @selector("getCharacters:range:"); bool getBytes(void* buffer, NSUInteger maxBufferCount, NSUInteger* usedBufferCount, NSStringEncoding encoding, NSStringEncodingConversionOptions options, NSRange range, NSRange* leftover) @selector("getBytes:maxLength:usedLength:encoding:options:range:remainingRange:"); + + CGSize sizeWithAttributes(NSDictionary attrs) @selector("sizeWithAttributes:"); + } + + // FIXME: it is a generic in objc with <KeyType, ObjectType> + extern class NSDictionary : NSObject { + static NSDictionary dictionaryWithObject(NSObject object, NSid key) @selector("dictionaryWithObject:forKey:"); + // static NSDictionary initWithObjects(NSArray objects, NSArray forKeys) @selector("initWithObjects:forKeys:"); } + alias NSAttributedStringKey = NSString; + /* const */extern __gshared NSAttributedStringKey NSFontAttributeName; + struct NSRange { NSUInteger loc; NSUInteger len; @@ -9406,7 +11831,42 @@ If you are not sure if Cocoa thinks your application is multithreaded or not, yo void run() @selector("run"); + void stop(NSid sender) @selector("stop:"); + + void finishLaunching() @selector("finishLaunching"); + void terminate(void*) @selector("terminate:"); + + void sendEvent(NSEvent event) @selector("sendEvent:"); + NSEvent nextEventMatchingMask( + NSEventMask mask, + NSDate untilDate, + NSRunLoopMode inMode, + bool dequeue + ) @selector("nextEventMatchingMask:untilDate:inMode:dequeue:"); + } + + enum NSEventMask : ulong { + NSEventMaskAny = ulong.max + } + + version(OSX) + extern class NSRunLoop : NSObject { + static @property NSRunLoop currentRunLoop() @selector("currentRunLoop"); + static @property NSRunLoop mainRunLoop() @selector("mainRunLoop"); + bool runMode(NSRunLoopMode mode, NSDate beforeDate) @selector("runMode:beforeDate:"); + } + + alias NSRunLoopMode = NSString; + + extern __gshared NSRunLoopMode NSDefaultRunLoopMode; + + version(OSX) + extern class NSDate : NSObject { + static @property NSDate distantFuture() @selector("distantFuture"); + static @property NSDate distantPast() @selector("distantPast"); + static @property NSDate now() @selector("now"); + } version(OSX) @@ -9523,6 +11983,7 @@ If you are not sure if Cocoa thinks your application is multithreaded or not, yo NSRect frame() @selector("frame"); NSRect contentRectForFrameRect(NSRect frameRect) @selector("contentRectForFrameRect:"); + NSRect frameRectForContentRect(NSRect contentRect) @selector("frameRectForContentRect:"); NSString title() @selector("title"); void title(NSString value) @selector("setTitle:"); @@ -9533,6 +11994,8 @@ If you are not sure if Cocoa thinks your application is multithreaded or not, yo void delegate_(NSWindowDelegate) @selector("setDelegate:"); void setBackgroundColor(NSColor color) @selector("setBackgroundColor:"); + + void setIsVisible(bool b) @selector("setIsVisible:"); } version(OSX) @@ -9749,4 +12212,95 @@ If you are not sure if Cocoa thinks your application is multithreaded or not, yo extern(C) __gshared void* _D4arsd4core6NSView7__ClassZ = null; extern(C) __gshared void* _D4arsd4core8NSWindow7__ClassZ = null; } + + + + extern(C) { // grand central dispatch bindings + + // /Library/Developer/CommandLineTools/SDKs/MacOSX13.1.sdk/usr/include/dispatch + // https://swiftlang.github.io/swift-corelibs-libdispatch/tutorial/ + // https://man.freebsd.org/cgi/man.cgi?query=dispatch_main&sektion=3&apropos=0&manpath=macOS+14.3.1 + + struct dispatch_source_type_s {} + private __gshared immutable extern { + dispatch_source_type_s _dispatch_source_type_timer; + dispatch_source_type_s _dispatch_source_type_proc; + dispatch_source_type_s _dispatch_source_type_signal; + dispatch_source_type_s _dispatch_source_type_read; + dispatch_source_type_s _dispatch_source_type_write; + dispatch_source_type_s _dispatch_source_type_vnode; + // also memory pressure and some others + } + + immutable DISPATCH_SOURCE_TYPE_TIMER = &_dispatch_source_type_timer; + immutable DISPATCH_SOURCE_TYPE_PROC = &_dispatch_source_type_proc; + immutable DISPATCH_SOURCE_TYPE_SIGNAL = &_dispatch_source_type_signal; + immutable DISPATCH_SOURCE_TYPE_READ = &_dispatch_source_type_read; + immutable DISPATCH_SOURCE_TYPE_WRITE = &_dispatch_source_type_write; + immutable DISPATCH_SOURCE_TYPE_VNODE = &_dispatch_source_type_vnode; + // also are some for internal data change things and a couple others + + enum DISPATCH_PROC_EXIT = 0x80000000; // process exited + enum DISPATCH_PROC_FORK = 0x40000000; // it forked + enum DISPATCH_PROC_EXEC = 0x20000000; // it execed + enum DISPATCH_PROC_SIGNAL = 0x08000000; // it received a signal + + enum DISPATCH_VNODE_DELETE = 0x1; + enum DISPATCH_VNODE_WRITE = 0x2; + enum DISPATCH_VNODE_EXTEND = 0x4; + enum DISPATCH_VNODE_ATTRIB = 0x8; + enum DISPATCH_VNODE_LINK = 0x10; + enum DISPATCH_VNODE_RENAME = 0x20; + enum DISPATCH_VNODE_REVOKE = 0x40; + enum DISPATCH_VNODE_FUNLOCK = 0x100; + + private struct dispatch_source_s; + private struct dispatch_queue_s {} + + alias dispatch_source_type_t = const(dispatch_source_type_s)*; + + alias dispatch_source_t = dispatch_source_s*; // NSObject<OS_dispatch_source> + alias dispatch_queue_t = dispatch_queue_s*; // NSObject<OS_dispatch_queue> + alias dispatch_object_t = void*; // actually a "transparent union" of the dispatch_source_t, dispatch_queue_t, and others + alias dispatch_block_t = ObjCBlock!(void)*; + static if(typeof(null).sizeof == 8) + alias uintptr_t = ulong; + else + alias uintptr_t = uint; + + dispatch_source_t dispatch_source_create(dispatch_source_type_t type, uintptr_t handle, c_ulong mask, dispatch_queue_t queue); + void dispatch_source_set_event_handler(dispatch_source_t source, dispatch_block_t handler); + void dispatch_source_set_cancel_handler(dispatch_source_t source, dispatch_block_t handler); + void dispatch_source_cancel(dispatch_source_t source); + + // DISPATCH_DECL_SUBCLASS(dispatch_queue_main, dispatch_queue_serial); + // dispatch_queue_t dispatch_get_main_queue(); + + extern __gshared dispatch_queue_s _dispatch_main_q; + + extern(D) dispatch_queue_t dispatch_get_main_queue() { + return &_dispatch_main_q; + } + + // FIXME: what is dispatch_time_t ??? + // dispatch_time + // dispatch_walltime + + // void dispatch_source_set_timer(dispatch_source_t source, dispatch_time_t start, ulong interval, ulong leeway); + + void dispatch_retain(dispatch_object_t object); + void dispatch_release(dispatch_object_t object); + + void dispatch_resume(dispatch_object_t object); + void dispatch_pause(dispatch_object_t object); + + void* dispatch_get_context(dispatch_object_t object); + void dispatch_set_context(dispatch_object_t object, void* context); + + // sends a function to the given queue + void dispatch_sync(dispatch_queue_t queue, scope dispatch_block_t block); + void dispatch_async(dispatch_queue_t queue, dispatch_block_t block); + + } // grand central dispatch bindings + } diff --git a/src/sisudoc/COPYRIGHT b/src/sisudoc/COPYRIGHT index fa0f004..6d7ee50 100644 --- a/src/sisudoc/COPYRIGHT +++ b/src/sisudoc/COPYRIGHT @@ -5,7 +5,7 @@ - Author: Ralph Amissah [ralph.amissah@gmail.com] - - Copyright: (C) 2015 - 2025 Ralph Amissah + - Copyright: (C) 2015 - 2026 Ralph Amissah - code under src/* src/sisudoc/* - License: AGPL 3 or later: diff --git a/src/sisudoc/spine_search.d b/src/sisudoc/spine_search.d index 8f9c16f..25321d8 100755 --- a/src/sisudoc/spine_search.d +++ b/src/sisudoc/spine_search.d @@ -7,7 +7,7 @@ - Author: Ralph Amissah [ralph.amissah@gmail.com] - - Copyright: (C) 2015 - 2025 Ralph Amissah, All Rights Reserved. + - Copyright: (C) 2015 (continuously updated, current 2026) Ralph Amissah, All Rights Reserved. - License: AGPL 3 or later: @@ -1911,7 +1911,7 @@ LIMIT %s OFFSET %s <a href=\"https://sisudoc.org/\" target=\"_top\"><b>≅ SiSU Spine ፨</b></a> </label> <label for=\"find\">(object numbering & object search)</label> -<label for=\"find\">2025</label> +<label for=\"find\">2026</label> <br><label for=\"find\">(web 1993, object numbering 1997, object search 2002 ...)</label> <br><a href=\"https://git.sisudoc.org/\" target=\"_top\">Δ git</a> </p> |