diff --git a/.gitignore b/.gitignore index 38da6d9..53cdd5b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ __pycache__/ .vscode/ +venv/ +*.egg-info/ +docs/_build/ diff --git a/builders.py b/builders.py deleted file mode 100644 index 5f25a4e..0000000 --- a/builders.py +++ /dev/null @@ -1,321 +0,0 @@ -from contextlib import contextmanager -import functools -import os -from pathlib import Path -import re -import shutil -import subprocess -import tempfile -from typing import Dict, List, Set, Sequence, Tuple - -import git -import tw -from util import BuildError, pushd - - -def _lazy_evaluable(func): - """ - Decorator which makes a function lazy-evaluable: that is, when it's - initially called, it returns a zero-argument lambda with the arguments - initially passed wrapped up in it. Calling that lambda has the effect - of executing the function. - - We use this in TZK to allow the user to use function calls in her config - to define the build steps, while not requiring her to write a bunch of - ugly and confusing lambda:'s in the config. The functions that will be called - are prepared during the config and executed later. - """ - - @functools.wraps(func) - def new_func(*args, **kwargs): - my_args = args - my_kwargs = kwargs - @functools.wraps(new_func) - def inner(): - func(*my_args, **my_kwargs) - return inner - return new_func - -# Now a more descriptive name that doesn't expose inner workings -# if the user wants to write her own builder. -tzk_builder = _lazy_evaluable - - -def stop(message: str) -> None: - "Stop the build due to an error condition." - raise BuildError(message) - - -def info(message: str) -> None: - "Print information about this build step to the console." - print(message) - - -# Global state available to all builders. -build_state = {} - - -@tzk_builder -def printer(username: str) -> None: - "Display the user's name" - if username == 'Maud': - raise Exception("No Mauds allowed!") - print(f"Hallelujah, {username} built a wiki!") - - -@tzk_builder -def require_branch(branchname: str) -> None: - "Require a specific Git branch to be checked out" - if git.read("branch", "--show-current") != branchname: - stop(f"You may only run this build from the {branchname} branch.") - - -@tzk_builder -def require_clean_working_tree() -> None: - "Require the working tree of the Git repository to be clean" - pleasecommit = "Please commit or stash them before publishing." - if git.rc("diff-index", "--quiet", "--cached", "HEAD", "--") != 0: - stop(f"There are staged changes. {pleasecommit}") - if git.rc("diff-files", "--quiet") != 0: - stop(f"There are uncommitted changes. {pleasecommit}") - - -@tzk_builder -def new_output_folder(): - "Create a new temporary folder to hold output being built" - assert 'public_wiki_folder' not in build_state - build_state['public_wiki_folder'] = tempfile.mkdtemp() - -new_output_folder.cleaner = lambda: shutil.rmtree(build_state['public_wiki_folder']) - - -@tzk_builder -def export_public_tiddlers(export_filter: str) -> None: - "Export public tiddlers to a temp wiki" - assert 'public_wiki_folder' in build_state, "new_output_folder builder must run first" - tw.exec(( - ("savewikifolder", build_state['public_wiki_folder'], export_filter), - )) - - -def _find_kill_phrases(phrases: Set[str]): - regexes = [re.compile(phrase) for phrase in phrases] - failures = [] - tid_files = (Path(build_state['public_wiki_folder']) / "tiddlers").glob("**/*.tid") - - for tid_file in tid_files: - with tid_file.open() as f: - for line in f: - for regex in regexes: - if re.search(regex, line): - failures.append((regex, str(tid_file), line)) - - return failures - -@tzk_builder -def check_for_kill_phrases(kill_phrase_file: str = None) -> None: - "Fail build if any of a series of regexes appears in a tiddler's source in the temp wiki" - assert 'public_wiki_folder' in build_state, "new_output_folder builder must run first" - if kill_phrase_file is None: - kill_phrase_file = "tiddlers/_system/config/zettelkasten/Build/KillPhrases.tid" - - kill_phrases = set() - with open(kill_phrase_file) as f: - reading = False - for line in f: - if not line.strip(): - reading = True - if line.strip() and reading: - kill_phrases.add(line) - - failures = _find_kill_phrases(kill_phrases) - if failures: - result = ["Kill phrases were found in your public wiki:"] - for failed_regex, failed_file, failed_line in failures: - trimmed_file = failed_file.replace(build_state['public_wiki_folder'], '') - result.append(f"'{failed_regex.pattern}' matched file {trimmed_file}:\n" - f" {failed_line.strip()}") - stop('\n'.join(result)) - -@tzk_builder -def save_attachments_externally(attachment_filter: str = "[is[image]]", - extimage_folder: str = "extimage") -> None: # TODO: or must this be extimage as the template suggests? - "Save embedded files in the temp wiki into an external folder" - assert 'public_wiki_folder' in build_state - - tw.exec( - ( - ("savetiddlers", attachment_filter, extimage_folder), - ), - base_wiki_folder=build_state['public_wiki_folder'] - ) - - -@tzk_builder -def compile_html_file( - wiki_name: str = "index.html", - output_folder: str = "output/public_site/", - overwrite: bool = True, - externalize_attachments: bool = False, - attachment_filter: str = "[is[image]]", - canonical_uri_template: str = "$:/core/templates/canonical-uri-external-image", - ) -> None: - "Compile a single HTML file from the temp wiki" - assert 'public_wiki_folder' in build_state - - commands: List[Tuple[str, ...]] = [] - if externalize_attachments: - commands.extend([ - ("setfield", attachment_filter, "_canonical_uri", - canonical_uri_template, "text/plain"), - ("setfield", attachment_filter, "text", "", "text/plain"), - ]) - commands.append(("render", "$:/core/save/all", wiki_name, "text/plain")) - - tw.exec(commands, base_wiki_folder=build_state['public_wiki_folder']) - if os.path.exists(output_folder) and not overwrite: - stop(f"The output folder '{os.path.abspath(output_folder)}' already exists. " - f"(To overwrite any files existing in the output folder, " - f"set overwrite = True for this builder.)") - - shutil.copytree( - Path(build_state['public_wiki_folder']) / "output", - Path(output_folder), - dirs_exist_ok=True - ) - info(f"Successfully copied built output to {os.path.abspath(output_folder)}.") - - -def _private_people_replacement_table() -> Dict[str, str]: - def _initials_from_tiddler_name(name: str) -> str: - m = re.match(r"^(?:Mr|Ms|Mx|The)(?P.*?)\.tid", name) - assert m - return '.'.join(i for i in m.group('camel_case_name') if i.isupper()) + '.' - - - # Build table of private people and their transformed initials. - tiddlers = (Path.cwd() / "tiddlers").glob("**/*.tid") - person_tiddlers = (i for i in tiddlers if re.match("^(Mr|Ms|Mx|The)", i.name)) - private_person_tiddlers = [] - for pt in person_tiddlers: - with pt.open() as f: - for line in f: - if line.startswith("tags:"): - if re.search(r'\bPublic\b', line): - # If there's a tags line in the file and it contains the - # Public tag, we skip it. - break - else: - # Otherwise, if there's a tags line in the file and it - # doesn't contain the Public tag, it's private. - private_person_tiddlers.append(pt) - break - if not line.strip(): - # And if there's no tags line in the file at all, - # it's private by default. - private_person_tiddlers.append(pt) - break - return { - i.name.replace('.tid', ''): _initials_from_tiddler_name(i.name) - for i in private_person_tiddlers - } - - -@tzk_builder -def replace_private_people() -> None: - "Replace the names of people who are not marked Public with their initials" - assert 'public_wiki_folder' in build_state - - replacement_table = _private_people_replacement_table() - tid_files = (Path(build_state['public_wiki_folder']) / "tiddlers").glob("**/*.tid") - - for tiddler in tid_files: - dirty = False - with tiddler.open() as f: - lines = f.readlines() - for idx, line in enumerate(lines): - for replace_person, replace_initials in replacement_table.items(): - if replace_person in line: - if '|' + replace_person + ']]' in line: - # link with the person as the target only; - # beware that you might have put something private in the text - lines[idx] = line.replace(replace_person, - 'PrivatePerson') - elif '[[' + replace_person + ']]' in line: - # link with the person as the target and text - lines[idx] = line.replace(replace_person, - replace_initials + '|PrivatePerson') - else: - # camel-case link or unlinked reference in text; - # or spurious substring, so rule that out with the '\b' search - lines[idx] = re.sub( - r"\b" + re.escape(replace_person) + r"\b", - f'<>', - line - ) - dirty = True - if dirty: - with tiddler.open("w") as f: - f.writelines(lines) - - -@tzk_builder -def set_tiddler_values(mappings: Dict[str, str]) -> None: - "Set the 'text' field of selected config or other tiddlers to new values" - assert 'public_wiki_folder' in build_state - - for tiddler, new_text in mappings.items(): - tiddler_path = (Path(build_state['public_wiki_folder']) / "tiddlers" / tiddler) - with tiddler_path.open("r") as f: - tiddler_lines = f.readlines() - first_blank_line_index = next(idx - for idx, value in enumerate(tiddler_lines) - if not value.strip()) - with tiddler_path.open("w") as f: - f.writelines(tiddler_lines[0:first_blank_line_index+1]) - f.write(new_text) - - -@tzk_builder -def publish_wiki_to_github( - output_folder: str = "output/public_site/", - commit_message: str = "publish checkpoint", - remote: str = "origin", - refspec: str = "master", - push = True) -> None: - "Publish the wiki to GitHub" - - with pushd(output_folder): - if not os.path.isdir(".git"): - info(f"The output folder {output_folder} doesn't appear to be a Git repository. " - f"I'll try to make it one.") - git.exec("init") - - git.exec("add", "-A") - rc = git.rc("commit", "-m", commit_message) - if rc == 0: - if push: - git.exec("push", remote, refspec) - elif rc == 1: - info("No changes to commit or publish. " - "You probably rebuilt without changing the wiki in between.") - else: - stop(f"'git commit' returned unknown return code {rc}.") - - -@tzk_builder -def shell(shell_command: str) -> None: - "Run an arbitrary shell command" - info("$ " + shell_command) - try: - output = subprocess.check_output(shell_command, shell=True, text=True) - except subprocess.CalledProcessError as e: - if e.output.strip(): - stop(f"Command exited with return code {e.returncode}:\n{e.output}") - else: - stop(f"Command exited with return code {e.returncode} (no output).") - else: - if output.strip(): - info(f"Command exited with return code 0:\n{output}") - else: - info(f"Command exited with return code 0 (no output).") diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..298ea9e --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,19 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/filesystem.py b/docs/_static/.gitkeep similarity index 100% rename from filesystem.py rename to docs/_static/.gitkeep diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..54920c1 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,108 @@ +# -*- coding: utf-8 -*- +# +# Configuration file for the Sphinx documentation builder. +# +# This file does only contain a selection of the most common options. For a +# full list see the documentation: +# http://www.sphinx-doc.org/en/master/config + +# -- Path setup -------------------------------------------------------------- +# Instead of adding things manually to the path, we just ensure we install esc +# in editable mode in our environment. + + +# -- Project information ----------------------------------------------------- + +project = 'tzk' +copyright = '2021 Soren Bjornstad' +author = 'Soren Bjornstad' + +# The short X.Y version +version = "0.0.1" +# The full version, including alpha/beta/rc tags +release = "0.0.1" + + +# -- General configuration --------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# +needs_sphinx = '1.8.5' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.autosectionlabel', + 'sphinx.ext.doctest', + 'sphinx.ext.todo', + 'sphinx.ext.coverage', + 'sphinx.ext.mathjax', + 'sphinx.ext.viewcode', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string. +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = None + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'sphinx_rtd_theme' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# The default sidebars (for documents that don't match any pattern) are +# defined by theme itself. Builtin themes are using these templates by +# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', +# 'searchbox.html']``. +# +# html_sidebars = {} + + + +# -- Extension configuration ------------------------------------------------- + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True + +# Prefix section labels with the names of their documents, to avoid ambiguity +# when the same heading appears on several pages. +autosectionlabel_prefix_document = False diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..6046cdd --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,20 @@ +tzk +=== + +**tzk** (pronounced /tə.zɪːk/) +is a custom build tool and utility CLI +for Soren Bjornstad's Zettelkasten edition of TiddlyWiki. + +.. toctree:: + :maxdepth: 3 + :caption: Contents + + operations + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/operations.rst b/docs/operations.rst new file mode 100644 index 0000000..b5e2953 --- /dev/null +++ b/docs/operations.rst @@ -0,0 +1,20 @@ +======== +Builders +======== + + +.. automodule:: tzk.builders + :members: check_for_kill_phrases, compile_html_file, export_public_tiddlers, new_output_folder, publish_wiki_to_github, replace_private_people, require_branch, require_clean_working_tree, save_attachments_externally, say_hi, set_tiddler_values, shell + + +Builder helper functions +======================== + +These helper functions, also defined in :mod:`tzk.builders`, +are intended for use with any custom builders you create. + +.. autofunction:: tzk.builders::info + +.. autofunction:: tzk.builders::stop + +.. autodecorator:: tzk.builders::tzk_builder diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..50d8e0e --- /dev/null +++ b/requirements.txt @@ -0,0 +1,11 @@ +# this package +-e . + +# docs +sphinx==3.5.4 # newer versions display function names in the wrong color? +sphinx_rtd_theme==0.5.2 + +# dev tools +setuptools +twine +wheel diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..e9817df --- /dev/null +++ b/setup.py @@ -0,0 +1,34 @@ +""" +setup.py - setuptools configuration for esc +""" + +import setuptools + +#with open("README.md", "r") as fh: +# long_description = fh.read() +long_description = "my nice long *description*" + +setuptools.setup( + name="tzk", + version="0.0.1", + author="Soren I. Bjornstad", + author_email="contact@sorenbjornstad.com", + description="Build tool for TiddlyWiki Zettelkasten", + long_description=long_description, + long_description_content_type="text/markdown", + url="TODO", + packages=setuptools.find_packages(), + classifiers=[ + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", + "Operating System :: OS Independent", + ], + entry_points={ + "console_scripts": [ + "tzk = tzk.__main__:launch" + ], + }, + python_requires='>=3.8', +) diff --git a/tzk/__init__.py b/tzk/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tzk.py b/tzk/__main__.py similarity index 75% rename from tzk.py rename to tzk/__main__.py index fc95626..d763b36 100644 --- a/tzk.py +++ b/tzk/__main__.py @@ -6,10 +6,10 @@ import sys import traceback from typing import Optional -from config import cm -import git -import tw -from util import BuildError, fail, numerize +from tzk.config import cm +from tzk import git +from tzk import tw +from tzk.util import BuildError, fail, numerize class CliCommand(ABC): @@ -35,13 +35,13 @@ class CommitCommand(CliCommand): "-m", "--message", metavar="MSG", help="Commit message to use.", - default=(cm.commit_message or "checkpoint") + default=(cm().commit_message or "checkpoint") ) parser.add_argument( "-r", "--remote", metavar="REMOTE", help="Name of the configured Git remote to push to.", - default=(cm.commit_remote or "origin"), + default=(cm().commit_remote or "origin"), ) parser.add_argument( "-l", "--local", @@ -50,12 +50,12 @@ class CommitCommand(CliCommand): ) def execute(self, args: argparse.Namespace) -> None: - if cm.commit_require_branch: + if cm().commit_require_branch: current_branch = git.read("rev-parse", "--abbrev-ref", "HEAD") - if current_branch != cm.commit_require_branch: + if current_branch != cm().commit_require_branch: fail(f"You are on the '{current_branch}' branch, " f"but your TZK configuration requires you to be on the " - f"'{cm.commit_require_branch}' branch to commit.") + f"'{cm().commit_require_branch}' branch to commit.") git.exec("add", "-A") git.exec("commit", "-m", args.message) @@ -73,18 +73,18 @@ class ListenCommand(CliCommand): "-p", "--port", metavar="PORT", help="Port to listen on.", - default=str(cm.listen_port or "8080"), + default=str(cm().listen_port or "8080"), ) parser.add_argument( "--username", metavar="USERNAME", - default=cm.listen_username or "", + default=cm().listen_username or "", help="Username to use for basic authentication, if any.", ) parser.add_argument( "--password", metavar="PASSWORD", - default=cm.listen_password or "", + default=cm().listen_password or "", help="Password to use for basic authentication, if any.", ) @@ -170,25 +170,26 @@ class BuildCommand(CliCommand): ) def _precheck(self, product: str) -> None: - if cm.products is None: + if cm().products is None: fail("No 'products' dictionary is defined in your config file.") - if product not in cm.products: + if product not in cm().products: fail(f"No '{product}' product found in the products dictionary " - f"in your config file. (Available: {', '.join(cm.products.keys())})") - if not cm.products[product]: + f"in your config file. (Available: {', '.join(cm().products.keys())})") + if not cm().products[product]: fail(f"No build steps are defined in the '{product}' product " f"in your config file.") def execute(self, args: argparse.Namespace) -> None: self._precheck(args.product) - steps = cm.products[args.product] + steps = cm().products[args.product] print(f"tzk: Starting build of product '{args.product}'.") print(f"tzk: Found {len(steps)} build {numerize(len(steps), 'step')}.") for idx, step in enumerate(steps, 1): if hasattr(step, '__doc__'): - print(f"tzk: Step {idx}/{len(steps)}: {step.__doc__}") + short_description = step.__doc__.strip().split('\n')[0].rstrip('.') + print(f"tzk: Step {idx}/{len(steps)}: {short_description}") else: print(f"tzk: Step {idx}/{len(steps)}") @@ -219,31 +220,40 @@ class BuildCommand(CliCommand): print(f"tzk: Build of product '{args.product}' completed successfully.") -parser = argparse.ArgumentParser() -subparsers = parser.add_subparsers() -for command in sorted(CliCommand.__subclasses__(), key=lambda i: i.__name__): - subparser = subparsers.add_parser(command.cmd, help=command.help) - subparser.set_defaults(_cls=command) - command.setup_arguments(subparser) # type: ignore +def launch(): + parser = argparse.ArgumentParser() -args = parser.parse_args() + subparsers = parser.add_subparsers() + for command in sorted(CliCommand.__subclasses__(), key=lambda i: i.__name__): + subparser = subparsers.add_parser(command.cmd, help=command.help) + subparser.set_defaults(_cls=command) + command.setup_arguments(subparser) # type: ignore -# For all operations except 'init', we start in the wiki folder. -if not args._cls.cmd == "init": - if not cm.wiki_folder: - fail("No 'wiki_folder' option found in config. Set this option to the name " - "of the wiki subfolder within the current directory.") + args = parser.parse_args() + if not hasattr(args, '_cls'): + parser.print_help() + sys.exit(0) - try: - os.chdir(cm.wiki_folder) - except FileNotFoundError: - fail(f"Tried to change directory into the wiki_folder '{cm.wiki_folder}' " - f"specified in your config file, but that directory does not exist.") + # For all operations except 'init', we start in the wiki folder. + if not args._cls.cmd == "init": + if not cm().wiki_folder: + fail("No 'wiki_folder' option found in config. Set this option to the name " + "of the wiki subfolder within the current directory.") - if not os.path.exists("tiddlywiki.info"): - fail(f"After changing directory into {cm.wiki_folder} per your config file: " - f"Expected a 'tiddlywiki.info' file in {os.getcwd()}. " - f"Please check that your wiki is initialized " - f"and you specified the correct wiki_folder_name.") + try: + os.chdir(cm().wiki_folder) + except FileNotFoundError: + fail(f"Tried to change directory into the wiki_folder '{cm().wiki_folder}' " + f"specified in your config file, but that directory does not exist.") -args._cls().execute(args) + if not os.path.exists("tiddlywiki.info"): + fail(f"After changing directory into {cm().wiki_folder} per your config file: " + f"Expected a 'tiddlywiki.info' file in {os.getcwd()}. " + f"Please check that your wiki is initialized " + f"and you specified the correct wiki_folder_name.") + + args._cls().execute(args) + + +if __name__ == '__main__': + launch() \ No newline at end of file diff --git a/tzk/builders.py b/tzk/builders.py new file mode 100644 index 0000000..f0808bb --- /dev/null +++ b/tzk/builders.py @@ -0,0 +1,526 @@ +""" +*Builders* are small executable chunks that together can be linked into a useful build +process. Aside from two helper functions (:func:`stop()` and :func:`info()`) +to fail the build and display an informational message, respectively, all other +functions in this module are builders. + +Builders are decorated with ``@tzk_builder``, a synonym for +:func:`_lazy_evaluable()`, which causes them to be lazy-evaluated: that is, +when they're initially called in the configuration, instead of running the +function and returning its result, a zero-argument function with all of the +arguments wrapped up is returned, to be run at a later time. This allows the +configuration file to be read at any time to retrieve information about the +defined products without actually running any build steps. + +You can write and use custom builders right within your config file +by decorating them with ``@builders.tzk_builder``. +""" + +from contextlib import contextmanager +import functools +import os +from pathlib import Path +import re +import shutil +import subprocess +import tempfile +from typing import Callable, Dict, List, Set, Sequence, Tuple + +from tzk import git +from tzk import tw +from tzk.util import BuildError, pushd + + +def tzk_builder(func): + """ + Decorator which makes a function lazy-evaluable: that is, when it's + initially called, it returns a zero-argument lambda with the arguments + initially passed wrapped up in it. Calling that lambda has the effect + of executing the function. + + We use this in TZK to allow the user to use function calls in her config + to define the build steps, while not requiring her to write a bunch of + ugly and confusing lambda:'s in the config. The functions that will be called + are prepared during the config and executed later. + """ + @functools.wraps(func) + def new_func(*args, **kwargs): + my_args = args + my_kwargs = kwargs + @functools.wraps(new_func) + def inner(): + func(*my_args, **my_kwargs) + return inner + return new_func + + +def stop(message: str) -> None: + "Stop the build due to an error condition." + raise BuildError(message) + + +def info(message: str) -> None: + "Print information about this build step to the console." + print(message) + + +# Global state available to all builders. +build_state = {} + + +@tzk_builder +def say_hi(username: str) -> None: + """ + Say hi to the specified user. + + This function is intended for testing. It will normally succeed and print + the provided *username*, but if you give "Jeff" as the username, the step + will fail, and if you give "General Failure" as the username, it will + cause an unhandled exception. + + :param username: The name of the person to say hi to. + """ + if username.lower() == 'general failure': + raise Exception("") + elif username.lower() == 'jeff': + # https://twitter.com/yephph/status/1249246702126546944 + stop("Sorry, the name Jeff does not work with our database schema.") + else: + print(f"Hello {username}!") + + +@tzk_builder +def require_branch(branchname: str) -> None: + """ + Require a specific Git branch to be checked out. + + If the branch isn't checked out, the build will fail immediately. This may + be helpful if you want to be sure you aren't accidentally building a wiki + from tentative changes or an old version. + + :param branchname: The name of the branch that must be checked out. + """ + if git.read("branch", "--show-current") != branchname: + stop(f"You may only run this build from the {branchname} branch.") + + +@tzk_builder +def require_clean_working_tree() -> None: + """ + Require the working tree of the Git repository to be clean. + + If there are any unstaged changes to existing files or staged changes, + the build will fail immediately. + + For the standard build process, it is not necessary for the working tree + to be clean. However, if you use any custom build steps that compare history, + or you simply want to ensure that you always have a recent checkpoint in your + local version whenever you publishing another version, this may be a useful + requirement. + """ + pleasecommit = "Please commit or stash them before publishing (try 'tzk commit')." + if git.rc("diff-index", "--quiet", "--cached", "HEAD", "--") != 0: + stop(f"There are staged changes. {pleasecommit}") + if git.rc("diff-files", "--quiet") != 0: + stop(f"There are uncommitted changes. {pleasecommit}") + + +@tzk_builder +def new_output_folder(): + """ + Create a new temporary folder to hold intermediate steps of the product being built. + + The path to this temporary folder will be stored in the 'public_wiki_folder' + key of the builders.build_state dictionary. Future build steps can access + the work in progress here. A cleaner is registered to delete this folder + when all steps complete, so any finished product should be copied out by a + later build step once it is complete. + """ + assert 'public_wiki_folder' not in build_state + build_state['public_wiki_folder'] = tempfile.mkdtemp() + +new_output_folder.cleaner = lambda: shutil.rmtree(build_state['public_wiki_folder']) + + +@tzk_builder +def export_public_tiddlers(export_filter: str) -> None: + """ + Export specified tiddlers to a new wiki in the temporary build folder. + + :func:`new_output_folder()` must be run prior to this builder. + + :param export_filter: A TiddlyWiki filter describing the tiddlers to be selected + for inclusion in the new wiki. + """ + assert 'public_wiki_folder' in build_state, "new_output_folder builder must run first" + tw.exec(( + ("savewikifolder", build_state['public_wiki_folder'], export_filter), + )) + + +def _find_kill_phrases(phrases: Set[str]): + """ + Search all tiddlers in the public_wiki_folder for the specified kill phrases. + """ + regexes = [re.compile(phrase) for phrase in phrases] + failures = [] + tid_files = (Path(build_state['public_wiki_folder']) / "tiddlers").glob("**/*.tid") + + for tid_file in tid_files: + with tid_file.open() as f: + for line in f: + for regex in regexes: + if re.search(regex, line): + failures.append((regex, str(tid_file), line)) + + return failures + + +@tzk_builder +def check_for_kill_phrases(kill_phrase_file: str = None) -> None: + """ + Fail the build if any of a series of regexes matches a tiddler's source in the temp wiki. + + The temp wiki should be created first + using the :func:`export_public_tiddlers()` builder. + + The kill phrases are Python-format regular expressions and may be configured + within the wiki, currently in $:/sib/gui/KillPhrases. + + :param kill_phrase_file: The path from the source wiki's root directory to the + config tiddler containing kill phrases. In the default + Zettelkasten edition, this is + tiddlers/_system/config/zettelkasten/Build/KillPhrases.tid; + if you change the way paths are determined, you can give + a different path here. + """ + assert 'public_wiki_folder' in build_state, "new_output_folder builder must run first" + if kill_phrase_file is None: + kill_phrase_file = "tiddlers/_system/config/zettelkasten/Build/KillPhrases.tid" + + kill_phrases = set() + with open(kill_phrase_file) as f: + reading = False + for line in f: + if not line.strip(): + reading = True + if line.strip() and reading: + kill_phrases.add(line) + + failures = _find_kill_phrases(kill_phrases) + if failures: + result = ["Kill phrases were found in your public wiki:"] + for failed_regex, failed_file, failed_line in failures: + trimmed_file = failed_file.replace(build_state['public_wiki_folder'], '') + result.append(f"'{failed_regex.pattern}' matched file {trimmed_file}:\n" + f" {failed_line.strip()}") + stop('\n'.join(result)) + + +@tzk_builder +def save_attachments_externally(attachment_filter: str = "[is[image]]", + extimage_folder: str = "extimage") -> None: + """ + Save embedded files in the temp wiki into an external folder. + + The temp wiki should be created first + using the :func:`export_public_tiddlers()` builder. + + Note that this builder **does not finish externalizing images**. + It saves the images outside the wiki, + but it does not change the ``_canonical_uri`` and ``text`` fields + on each image tiddler to point to this new location. + For the latter step, use the + ``externalize_attachments``, ``attachment_filter``, and ``canonical_uri_template`` + parameters to the ``compile_html_file`` step. + + :param attachment_filter: The tiddlers to be saved to the external folder; + by default, ``[is[image]]``. + :param extimage_folder: The name of the external folder to save to. This must + be the default of ``extimage`` to work with the default + ``canonical_uri_template`` + in the :func:`compile_html_file()` builder, + but you can use a different name and a different + canonical URI template if you prefer. + """ + assert 'public_wiki_folder' in build_state + + tw.exec( + ( + ("savetiddlers", attachment_filter, extimage_folder), + ), + base_wiki_folder=build_state['public_wiki_folder'] + ) + + +@tzk_builder +def compile_html_file( + wiki_name: str = "index.html", + output_folder: str = "output/public_site/", + overwrite: bool = True, + externalize_attachments: bool = False, + attachment_filter: str = "[is[image]]", + canonical_uri_template: str = "$:/core/templates/canonical-uri-external-image", + ) -> None: + """ + Compile a single HTML file from the temp wiki. + + Before compiling an HTML file, + you should create a temp wiki using the :func:`export_public_tiddlers()` builder, + then run any other build steps you want to use + to make changes to the wiki being built. + Once you compile the HTML file, + it will be copied out to an output location outside the temp folder, + and the content can no longer be changed through tzk. + + :param wiki_name: The filename of the single-file wiki to create. + Default ``index.html``. + :param output_folder: The path to the folder + where the single-file wiki and any externalized attachments + will be placed, relative to the private wiki's root directory. + Default ``output/public_site``. + :param overwrite: If the ``output_folder`` already exists, + should we overwrite it? Default True. + :param externalize_attachments: If True, update tiddlers that match + the ``attachment_filter`` parameter to point to + external versions of their content. + Only useful if you have previously run + the ``save_attachments_externally`` builder. + Default False. + :param attachment_filter: If externalizing attachments, + which tiddlers should be externalized? + Default ``[is[image]]``. + :param canonical_uri_template: What TiddlyWiki template should be used + to determine the new content + of the ``_canonical_uri`` field? + Default ``$:/core/templates/canonical-uri-external-image``. + If you're not sure what this is, don't change it. + """ + assert 'public_wiki_folder' in build_state + + commands: List[Tuple[str, ...]] = [] + if externalize_attachments: + commands.extend([ + ("setfield", attachment_filter, "_canonical_uri", + canonical_uri_template, "text/plain"), + ("setfield", attachment_filter, "text", "", "text/plain"), + ]) + commands.append(("render", "$:/core/save/all", wiki_name, "text/plain")) + + tw.exec(commands, base_wiki_folder=build_state['public_wiki_folder']) + if os.path.exists(output_folder) and not overwrite: + stop(f"The output folder '{os.path.abspath(output_folder)}' already exists. " + f"(To overwrite any files existing in the output folder, " + f"set overwrite = True for this builder.)") + + shutil.copytree( + Path(build_state['public_wiki_folder']) / "output", + Path(output_folder), + dirs_exist_ok=True + ) + info(f"Successfully copied built output to {os.path.abspath(output_folder)}.") + + +def _private_people_replacement_table( + initialer: Callable[[str], str] = None) -> Dict[str, str]: + "Build table of private people and their transformed initials." + + def _initials_from_tiddler_name(name: str) -> str: + m = re.match(r"^(?:Mr|Ms|Mx|The)(?P.*?)\.tid", name) + assert m + return '.'.join(i for i in m.group('camel_case_name') if i.isupper()) + '.' + + if initialer is None: + initialer = _initials_from_tiddler_name + + tiddlers = (Path.cwd() / "tiddlers").glob("**/*.tid") + person_tiddlers = (i for i in tiddlers if re.match("^(Mr|Ms|Mx|The)", i.name)) + private_person_tiddlers = [] + for pt in person_tiddlers: + with pt.open() as f: + for line in f: + if line.startswith("tags:"): + if re.search(r'\bPublic\b', line): + # If there's a tags line in the file and it contains the + # Public tag, we skip it. + break + else: + # Otherwise, if there's a tags line in the file and it + # doesn't contain the Public tag, it's private. + private_person_tiddlers.append(pt) + break + if not line.strip(): + # And if there's no tags line in the file at all, + # it's private by default. + private_person_tiddlers.append(pt) + break + return { + i.name.replace('.tid', ''): initialer(i.name) + for i in private_person_tiddlers + } + + +@tzk_builder +def replace_private_people(initialer: Callable[[str], str] = None) -> None: + """ + Replace the names of people who are not marked Public with their initials. + + If you have lots of PAO (People, Animals, and Organizations) in your Zettelkasten + and many of them are personal friends, + you might prefer not to make everything you said about them + easily searchable on the internet. + This is more challenging than simply not marking their tiddlers public, + since + + This builder replaces all links, bracketed or WikiCamelCase, + to the names of all people *not* tagged Public + with the initials suggested by their CamelCase titles + (e.g., MsJaneDoe becomes J.D.). The links point to the tiddler ``PrivatePerson``, + which explains this process. + + :param initialer: If you don't like the way that initials + are generated from tiddler filenames by default, + you can customize it by passing a callable + that takes one string argument + (a tiddler filename without the full path, e.g., ``MsJaneDoe.tid``) + and returns a string to be considered the "initials" of that person. + """ + assert 'public_wiki_folder' in build_state + + replacement_table = _private_people_replacement_table(initialer) + tid_files = (Path(build_state['public_wiki_folder']) / "tiddlers").glob("**/*.tid") + + for tiddler in tid_files: + dirty = False + with tiddler.open() as f: + lines = f.readlines() + for idx, line in enumerate(lines): + for replace_person, replace_initials in replacement_table.items(): + if replace_person in line: + if '|' + replace_person + ']]' in line: + # link with the person as the target only; + # beware that you might have put something private in the text + lines[idx] = line.replace(replace_person, + 'PrivatePerson') + elif '[[' + replace_person + ']]' in line: + # link with the person as the target and text + lines[idx] = line.replace(replace_person, + replace_initials + '|PrivatePerson') + else: + # camel-case link or unlinked reference in text; + # or spurious substring, so rule that out with the '\b' search + lines[idx] = re.sub( + r"\b" + re.escape(replace_person) + r"\b", + f'<>', + line + ) + dirty = True + if dirty: + with tiddler.open("w") as f: + f.writelines(lines) + + +@tzk_builder +def set_tiddler_values(mappings: Dict[str, str]) -> None: + """ + Set the 'text' field of selected config or other tiddlers to arbitrary new values. + + This can be used to make customizations that can't easily be done with feature + flags or other wikitext solutions within the wiki -- for instance, changing the + subtitle or what buttons are visible. It's also used to implement feature flags + in the first place by changing the ``$:/config/sib/CurrentEditionPublicity`` + tiddler to ``public``, so the minimal functional wiki will use: + + ```python + {'$__config_sib_CurrentEditionPublicity.tid': 'public',} + ``` + + :param mappings: A dictionary whose keys are tiddler names + and whose values are the values to be inserted + in those tiddlers' ``text`` fields. + """ + assert 'public_wiki_folder' in build_state + + for tiddler, new_text in mappings.items(): + tiddler_path = (Path(build_state['public_wiki_folder']) / "tiddlers" / tiddler) + with tiddler_path.open("r") as f: + tiddler_lines = f.readlines() + first_blank_line_index = next(idx + for idx, value in enumerate(tiddler_lines) + if not value.strip()) + with tiddler_path.open("w") as f: + f.writelines(tiddler_lines[0:first_blank_line_index+1]) + f.write(new_text) + + +@tzk_builder +def publish_wiki_to_github( + output_folder: str = "output/public_site/", + commit_message: str = "publish checkpoint", + remote: str = "origin", + refspec: str = "master", + push = True) -> None: + """ + Publish the built wiki to GitHub. + + :param output_folder: Path to a folder containing the Git repository you'd like + to publish, relative to your private wiki's root directory. + This folder should contain the version of your wiki + built by the :func:`compile_html_file()` builder, + either directly or in a subfolder somewhere. + You need to clone or create the repository yourself + and set up the remotes as appropriate + before running this step. + Default ``output/public_site``. + :param commit_message: Message to use when committing the newly built wiki. + Note that all changes in the repository will be committed. + Default ``publish checkpoint``. + :param remote: The repository remote to push changes to. + If you don't know what that is, + the default of ``origin`` is probably correct. + :param refspec: The local branch or refspec to push. + Default ``master``. + :param push: If set to False, don't push after committing the changes + (mostly useful for testing purposes). + Default True. + """ + with pushd(output_folder): + if not os.path.isdir(".git"): + stop(f"The output folder {output_folder} isn't a Git repository. " + f"Please go initialize it and then try again.") + + git.exec("add", "-A") + rc = git.rc("commit", "-m", commit_message) + if rc == 0: + if push: + git.exec("push", remote, refspec) + elif rc == 1: + info("No changes to commit or publish. " + "You probably rebuilt without changing the wiki in between.") + else: + stop(f"'git commit' returned unknown return code {rc}.") + + +@tzk_builder +def shell(shell_command: str) -> None: + """ + Run an arbitrary shell command. + + The builder will fail if a return code other than 0 is returned, + otherwise it will succeed. Output will be printed to stdout. + + :param shell_command: A string to be passed to your system's shell. + """ + info("$ " + shell_command) + try: + output = subprocess.check_output(shell_command, shell=True, text=True) + except subprocess.CalledProcessError as e: + if e.output.strip(): + stop(f"Command exited with return code {e.returncode}:\n{e.output}") + else: + stop(f"Command exited with return code {e.returncode} (no output).") + else: + if output.strip(): + info(f"Command exited with return code 0:\n{output}") + else: + info(f"Command exited with return code 0 (no output).") diff --git a/config.py b/tzk/config.py similarity index 93% rename from config.py rename to tzk/config.py index 611cc66..9b78a79 100644 --- a/config.py +++ b/tzk/config.py @@ -6,7 +6,7 @@ from pathlib import Path import sys from typing import Any -from util import fail +from tzk.util import fail class ConfigurationManager: @@ -56,4 +56,7 @@ class ConfigurationManager: return True -cm = ConfigurationManager() +def cm(cache=[]): + if not cache: + cache.append(ConfigurationManager()) + return cache[0] diff --git a/git.py b/tzk/git.py similarity index 100% rename from git.py rename to tzk/git.py diff --git a/tw.py b/tzk/tw.py similarity index 94% rename from tw.py rename to tzk/tw.py index 2a15c44..0aa989d 100644 --- a/tw.py +++ b/tzk/tw.py @@ -6,9 +6,9 @@ import subprocess from textwrap import dedent from typing import Optional, Sequence -import config -import git -from util import pushd +from tzk import config +from tzk import git +from tzk.util import pushd @functools.lru_cache(1) @@ -85,12 +85,12 @@ def _save_wikifolder_to_config(wiki_name: str) -> bool: Return True if the option ended set to wiki_name, False otherwise. """ print("tzk: Writing new wiki folder to config file...") - if not config.cm.write_attr("wiki_folder", wiki_name): - if config.cm.wiki_folder == wiki_name: + if not config.cm().write_attr("wiki_folder", wiki_name): + if config.cm().wiki_folder == wiki_name: print("tzk: (Looks like it was already there.)") else: print(f"tzk: WARNING: The wiki_folder option in your config appears " - f"to be set to '{config.cm.wiki_folder}', rather than the wiki folder " + f"to be set to '{config.cm().wiki_folder}', rather than the wiki folder " f"you're initializing, {wiki_name}. Please check your config file " "and update this option if necessary.") return False diff --git a/util.py b/tzk/util.py similarity index 100% rename from util.py rename to tzk/util.py