jjkiers/tzk
1
0
Fork 0
Code Issues Pull Requests Releases Activity

initialize Sphinx and document builders

Browse Source
This commit is contained in:
Soren I. Bjornstad 2021-08-27 09:53:18 -05:00
parent 5d4a3164de
commit 0321b7d7b1
16 changed files with 803 additions and 370 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -1,2 +1,5 @@
__pycache__/ __pycache__/
.vscode/ .vscode/
venv/
*.egg-info/
docs/_build/

321
builders.py
View File

@ -1,321 +0,0 @@
from contextlib import contextmanager
import functools
import os
from pathlib import Path
import re
import shutil
import subprocess
import tempfile
from typing import Dict, List, Set, Sequence, Tuple
import git
import tw
from util import BuildError, pushd
def _lazy_evaluable(func):
"""
Decorator which makes a function lazy-evaluable: that is, when it's
initially called, it returns a zero-argument lambda with the arguments
initially passed wrapped up in it. Calling that lambda has the effect
of executing the function.
We use this in TZK to allow the user to use function calls in her config
to define the build steps, while not requiring her to write a bunch of
ugly and confusing lambda:'s in the config. The functions that will be called
are prepared during the config and executed later.
"""
@functools.wraps(func)
def new_func(*args, **kwargs):
my_args = args
my_kwargs = kwargs
@functools.wraps(new_func)
def inner():
func(*my_args, **my_kwargs)
return inner
return new_func
# Now a more descriptive name that doesn't expose inner workings
# if the user wants to write her own builder.
tzk_builder = _lazy_evaluable
def stop(message: str) -> None:
"Stop the build due to an error condition."
raise BuildError(message)
def info(message: str) -> None:
"Print information about this build step to the console."
print(message)
# Global state available to all builders.
build_state = {}
@tzk_builder
def printer(username: str) -> None:
"Display the user's name"
if username == 'Maud':
raise Exception("No Mauds allowed!")
print(f"Hallelujah, {username} built a wiki!")
@tzk_builder
def require_branch(branchname: str) -> None:
"Require a specific Git branch to be checked out"
if git.read("branch", "--show-current") != branchname:
stop(f"You may only run this build from the {branchname} branch.")
@tzk_builder
def require_clean_working_tree() -> None:
"Require the working tree of the Git repository to be clean"
pleasecommit = "Please commit or stash them before publishing."
if git.rc("diff-index", "--quiet", "--cached", "HEAD", "--") != 0:
stop(f"There are staged changes. {pleasecommit}")
if git.rc("diff-files", "--quiet") != 0:
stop(f"There are uncommitted changes. {pleasecommit}")
@tzk_builder
def new_output_folder():
"Create a new temporary folder to hold output being built"
assert 'public_wiki_folder' not in build_state
build_state['public_wiki_folder'] = tempfile.mkdtemp()
new_output_folder.cleaner = lambda: shutil.rmtree(build_state['public_wiki_folder'])
@tzk_builder
def export_public_tiddlers(export_filter: str) -> None:
"Export public tiddlers to a temp wiki"
assert 'public_wiki_folder' in build_state, "new_output_folder builder must run first"
tw.exec((
("savewikifolder", build_state['public_wiki_folder'], export_filter),
))
def _find_kill_phrases(phrases: Set[str]):
regexes = [re.compile(phrase) for phrase in phrases]
failures = []
tid_files = (Path(build_state['public_wiki_folder']) / "tiddlers").glob("**/*.tid")
for tid_file in tid_files:
with tid_file.open() as f:
for line in f:
for regex in regexes:
if re.search(regex, line):
failures.append((regex, str(tid_file), line))
return failures
@tzk_builder
def check_for_kill_phrases(kill_phrase_file: str = None) -> None:
"Fail build if any of a series of regexes appears in a tiddler's source in the temp wiki"
assert 'public_wiki_folder' in build_state, "new_output_folder builder must run first"
if kill_phrase_file is None:
kill_phrase_file = "tiddlers/_system/config/zettelkasten/Build/KillPhrases.tid"
kill_phrases = set()
with open(kill_phrase_file) as f:
reading = False
for line in f:
if not line.strip():
reading = True
if line.strip() and reading:
kill_phrases.add(line)
failures = _find_kill_phrases(kill_phrases)
if failures:
result = ["Kill phrases were found in your public wiki:"]
for failed_regex, failed_file, failed_line in failures:
trimmed_file = failed_file.replace(build_state['public_wiki_folder'], '')
result.append(f"'{failed_regex.pattern}' matched file {trimmed_file}:\n"
f" {failed_line.strip()}")
stop('\n'.join(result))
@tzk_builder
def save_attachments_externally(attachment_filter: str = "[is[image]]",
extimage_folder: str = "extimage") -> None: # TODO: or must this be extimage as the template suggests?
"Save embedded files in the temp wiki into an external folder"
assert 'public_wiki_folder' in build_state
tw.exec(
(
("savetiddlers", attachment_filter, extimage_folder),
),
base_wiki_folder=build_state['public_wiki_folder']
)
@tzk_builder
def compile_html_file(
wiki_name: str = "index.html",
output_folder: str = "output/public_site/",
overwrite: bool = True,
externalize_attachments: bool = False,
attachment_filter: str = "[is[image]]",
canonical_uri_template: str = "$:/core/templates/canonical-uri-external-image",
) -> None:
"Compile a single HTML file from the temp wiki"
assert 'public_wiki_folder' in build_state
commands: List[Tuple[str, ...]] = []
if externalize_attachments:
commands.extend([
("setfield", attachment_filter, "_canonical_uri",
canonical_uri_template, "text/plain"),
("setfield", attachment_filter, "text", "", "text/plain"),
])
commands.append(("render", "$:/core/save/all", wiki_name, "text/plain"))
tw.exec(commands, base_wiki_folder=build_state['public_wiki_folder'])
if os.path.exists(output_folder) and not overwrite:
stop(f"The output folder '{os.path.abspath(output_folder)}' already exists. "
f"(To overwrite any files existing in the output folder, "
f"set overwrite = True for this builder.)")
shutil.copytree(
Path(build_state['public_wiki_folder']) / "output",
Path(output_folder),
dirs_exist_ok=True
)
info(f"Successfully copied built output to {os.path.abspath(output_folder)}.")
def _private_people_replacement_table() -> Dict[str, str]:
def _initials_from_tiddler_name(name: str) -> str:
m = re.match(r"^(?:Mr|Ms|Mx|The)(?P<camel_case_name>.*?)\.tid", name)
assert m
return '.'.join(i for i in m.group('camel_case_name') if i.isupper()) + '.'
# Build table of private people and their transformed initials.
tiddlers = (Path.cwd() / "tiddlers").glob("**/*.tid")
person_tiddlers = (i for i in tiddlers if re.match("^(Mr|Ms|Mx|The)", i.name))
private_person_tiddlers = []
for pt in person_tiddlers:
with pt.open() as f:
for line in f:
if line.startswith("tags:"):
if re.search(r'\bPublic\b', line):
# If there's a tags line in the file and it contains the
# Public tag, we skip it.
break
else:
# Otherwise, if there's a tags line in the file and it
# doesn't contain the Public tag, it's private.
private_person_tiddlers.append(pt)
break
if not line.strip():
# And if there's no tags line in the file at all,
# it's private by default.
private_person_tiddlers.append(pt)
break
return {
i.name.replace('.tid', ''): _initials_from_tiddler_name(i.name)
for i in private_person_tiddlers
}
@tzk_builder
def replace_private_people() -> None:
"Replace the names of people who are not marked Public with their initials"
assert 'public_wiki_folder' in build_state
replacement_table = _private_people_replacement_table()
tid_files = (Path(build_state['public_wiki_folder']) / "tiddlers").glob("**/*.tid")
for tiddler in tid_files:
dirty = False
with tiddler.open() as f:
lines = f.readlines()
for idx, line in enumerate(lines):
for replace_person, replace_initials in replacement_table.items():
if replace_person in line:
if '|' + replace_person + ']]' in line:
# link with the person as the target only;
# beware that you might have put something private in the text
lines[idx] = line.replace(replace_person,
'PrivatePerson')
elif '[[' + replace_person + ']]' in line:
# link with the person as the target and text
lines[idx] = line.replace(replace_person,
replace_initials + '|PrivatePerson')
else:
# camel-case link or unlinked reference in text;
# or spurious substring, so rule that out with the '\b' search
lines[idx] = re.sub(
r"\b" + re.escape(replace_person) + r"\b",
f'<<privateperson "{replace_initials}">>',
line
)
dirty = True
if dirty:
with tiddler.open("w") as f:
f.writelines(lines)
@tzk_builder
def set_tiddler_values(mappings: Dict[str, str]) -> None:
"Set the 'text' field of selected config or other tiddlers to new values"
assert 'public_wiki_folder' in build_state
for tiddler, new_text in mappings.items():
tiddler_path = (Path(build_state['public_wiki_folder']) / "tiddlers" / tiddler)
with tiddler_path.open("r") as f:
tiddler_lines = f.readlines()
first_blank_line_index = next(idx
for idx, value in enumerate(tiddler_lines)
if not value.strip())
with tiddler_path.open("w") as f:
f.writelines(tiddler_lines[0:first_blank_line_index+1])
f.write(new_text)
@tzk_builder
def publish_wiki_to_github(
output_folder: str = "output/public_site/",
commit_message: str = "publish checkpoint",
remote: str = "origin",
refspec: str = "master",
push = True) -> None:
"Publish the wiki to GitHub"
with pushd(output_folder):
if not os.path.isdir(".git"):
info(f"The output folder {output_folder} doesn't appear to be a Git repository. "
f"I'll try to make it one.")
git.exec("init")
git.exec("add", "-A")
rc = git.rc("commit", "-m", commit_message)
if rc == 0:
if push:
git.exec("push", remote, refspec)
elif rc == 1:
info("No changes to commit or publish. "
"You probably rebuilt without changing the wiki in between.")
else:
stop(f"'git commit' returned unknown return code {rc}.")
@tzk_builder
def shell(shell_command: str) -> None:
"Run an arbitrary shell command"
info("$ " + shell_command)
try:
output = subprocess.check_output(shell_command, shell=True, text=True)
except subprocess.CalledProcessError as e:
if e.output.strip():
stop(f"Command exited with return code {e.returncode}:\n{e.output}")
else:
stop(f"Command exited with return code {e.returncode} (no output).")
else:
if output.strip():
info(f"Command exited with return code 0:\n{output}")
else:
info(f"Command exited with return code 0 (no output).")

19
docs/Makefile Normal file
View File

@ -0,0 +1,19 @@
# Minimal makefile for Sphinx documentation
#
# You can set these variables from the command line.
SPHINXOPTS =
SPHINXBUILD = sphinx-build
SOURCEDIR = .
BUILDDIR = _build
# Put it first so that "make" without argument is like "make help".
help:
@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
.PHONY: help Makefile
# Catch-all target: route all unknown targets to Sphinx using the new
# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
%: Makefile
@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

0
filesystem.py → docs/_static/.gitkeep vendored
View File

108
docs/conf.py Normal file
View File

@ -0,0 +1,108 @@
# -*- coding: utf-8 -*-
#
# Configuration file for the Sphinx documentation builder.
#
# This file does only contain a selection of the most common options. For a
# full list see the documentation:
# http://www.sphinx-doc.org/en/master/config
# -- Path setup --------------------------------------------------------------
# Instead of adding things manually to the path, we just ensure we install esc
# in editable mode in our environment.
# -- Project information -----------------------------------------------------
project = 'tzk'
copyright = '2021 Soren Bjornstad'
author = 'Soren Bjornstad'
# The short X.Y version
version = "0.0.1"
# The full version, including alpha/beta/rc tags
release = "0.0.1"
# -- General configuration ---------------------------------------------------
# If your documentation needs a minimal Sphinx version, state it here.
#
needs_sphinx = '1.8.5'
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
'sphinx.ext.autodoc',
'sphinx.ext.autosectionlabel',
'sphinx.ext.doctest',
'sphinx.ext.todo',
'sphinx.ext.coverage',
'sphinx.ext.mathjax',
'sphinx.ext.viewcode',
]
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string.
source_suffix = '.rst'
# The master toctree document.
master_doc = 'index'
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path.
exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = None
# -- Options for HTML output -------------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
#
html_theme = 'sphinx_rtd_theme'
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
#
# html_theme_options = {}
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']
# Custom sidebar templates, must be a dictionary that maps document names
# to template names.
#
# The default sidebars (for documents that don't match any pattern) are
# defined by theme itself. Builtin themes are using these templates by
# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
# 'searchbox.html']``.
#
# html_sidebars = {}
# -- Extension configuration -------------------------------------------------
# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = True
# Prefix section labels with the names of their documents, to avoid ambiguity
# when the same heading appears on several pages.
autosectionlabel_prefix_document = False

20
docs/index.rst Normal file
View File

@ -0,0 +1,20 @@
tzk
===
**tzk** (pronounced /tə.zɪːk/)
is a custom build tool and utility CLI
for Soren Bjornstad's Zettelkasten edition of TiddlyWiki.
.. toctree::
:maxdepth: 3
:caption: Contents
operations
Indices and tables
==================
* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`

20
docs/operations.rst Normal file
View File

@ -0,0 +1,20 @@
========
Builders
========
.. automodule:: tzk.builders
:members: check_for_kill_phrases, compile_html_file, export_public_tiddlers, new_output_folder, publish_wiki_to_github, replace_private_people, require_branch, require_clean_working_tree, save_attachments_externally, say_hi, set_tiddler_values, shell
Builder helper functions
========================
These helper functions, also defined in :mod:`tzk.builders`,
are intended for use with any custom builders you create.
.. autofunction:: tzk.builders::info
.. autofunction:: tzk.builders::stop
.. autodecorator:: tzk.builders::tzk_builder

11
requirements.txt Normal file
View File

@ -0,0 +1,11 @@
# this package
-e .
# docs
sphinx==3.5.4 # newer versions display function names in the wrong color?
sphinx_rtd_theme==0.5.2
# dev tools
setuptools
twine
wheel

34
setup.py Normal file
View File

@ -0,0 +1,34 @@
"""
setup.py - setuptools configuration for esc
"""
import setuptools
#with open("README.md", "r") as fh:
# long_description = fh.read()
long_description = "my nice long *description*"
setuptools.setup(
name="tzk",
version="0.0.1",
author="Soren I. Bjornstad",
author_email="contact@sorenbjornstad.com",
description="Build tool for TiddlyWiki Zettelkasten",
long_description=long_description,
long_description_content_type="text/markdown",
url="TODO",
packages=setuptools.find_packages(),
classifiers=[
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.6",
"Programming Language :: Python :: 3.7",
"License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
"Operating System :: OS Independent",
],
entry_points={
"console_scripts": [
"tzk = tzk.__main__:launch"
],
},
python_requires='>=3.8',
)

0
tzk/__init__.py Normal file
View File

92
tzk.py → tzk/__main__.py
View File

@ -6,10 +6,10 @@ import sys
import traceback import traceback
from typing import Optional from typing import Optional
from config import cm from tzk.config import cm
import git from tzk import git
import tw from tzk import tw
from util import BuildError, fail, numerize from tzk.util import BuildError, fail, numerize
class CliCommand(ABC): class CliCommand(ABC):
@ -35,13 +35,13 @@ class CommitCommand(CliCommand):
"-m", "--message", "-m", "--message",
metavar="MSG", metavar="MSG",
help="Commit message to use.", help="Commit message to use.",
default=(cm.commit_message or "checkpoint") default=(cm().commit_message or "checkpoint")
) )
parser.add_argument( parser.add_argument(
"-r", "--remote", "-r", "--remote",
metavar="REMOTE", metavar="REMOTE",
help="Name of the configured Git remote to push to.", help="Name of the configured Git remote to push to.",
default=(cm.commit_remote or "origin"), default=(cm().commit_remote or "origin"),
) )
parser.add_argument( parser.add_argument(
"-l", "--local", "-l", "--local",
@ -50,12 +50,12 @@ class CommitCommand(CliCommand):
) )
def execute(self, args: argparse.Namespace) -> None: def execute(self, args: argparse.Namespace) -> None:
if cm.commit_require_branch: if cm().commit_require_branch:
current_branch = git.read("rev-parse", "--abbrev-ref", "HEAD") current_branch = git.read("rev-parse", "--abbrev-ref", "HEAD")
if current_branch != cm.commit_require_branch: if current_branch != cm().commit_require_branch:
fail(f"You are on the '{current_branch}' branch, " fail(f"You are on the '{current_branch}' branch, "
f"but your TZK configuration requires you to be on the " f"but your TZK configuration requires you to be on the "
f"'{cm.commit_require_branch}' branch to commit.") f"'{cm().commit_require_branch}' branch to commit.")
git.exec("add", "-A") git.exec("add", "-A")
git.exec("commit", "-m", args.message) git.exec("commit", "-m", args.message)
@ -73,18 +73,18 @@ class ListenCommand(CliCommand):
"-p", "--port", "-p", "--port",
metavar="PORT", metavar="PORT",
help="Port to listen on.", help="Port to listen on.",
default=str(cm.listen_port or "8080"), default=str(cm().listen_port or "8080"),
) )
parser.add_argument( parser.add_argument(
"--username", "--username",
metavar="USERNAME", metavar="USERNAME",
default=cm.listen_username or "", default=cm().listen_username or "",
help="Username to use for basic authentication, if any.", help="Username to use for basic authentication, if any.",
) )
parser.add_argument( parser.add_argument(
"--password", "--password",
metavar="PASSWORD", metavar="PASSWORD",
default=cm.listen_password or "", default=cm().listen_password or "",
help="Password to use for basic authentication, if any.", help="Password to use for basic authentication, if any.",
) )
@ -170,25 +170,26 @@ class BuildCommand(CliCommand):
) )
def _precheck(self, product: str) -> None: def _precheck(self, product: str) -> None:
if cm.products is None: if cm().products is None:
fail("No 'products' dictionary is defined in your config file.") fail("No 'products' dictionary is defined in your config file.")
if product not in cm.products: if product not in cm().products:
fail(f"No '{product}' product found in the products dictionary " fail(f"No '{product}' product found in the products dictionary "
f"in your config file. (Available: {', '.join(cm.products.keys())})") f"in your config file. (Available: {', '.join(cm().products.keys())})")
if not cm.products[product]: if not cm().products[product]:
fail(f"No build steps are defined in the '{product}' product " fail(f"No build steps are defined in the '{product}' product "
f"in your config file.") f"in your config file.")
def execute(self, args: argparse.Namespace) -> None: def execute(self, args: argparse.Namespace) -> None:
self._precheck(args.product) self._precheck(args.product)
steps = cm.products[args.product] steps = cm().products[args.product]
print(f"tzk: Starting build of product '{args.product}'.") print(f"tzk: Starting build of product '{args.product}'.")
print(f"tzk: Found {len(steps)} build {numerize(len(steps), 'step')}.") print(f"tzk: Found {len(steps)} build {numerize(len(steps), 'step')}.")
for idx, step in enumerate(steps, 1): for idx, step in enumerate(steps, 1):
if hasattr(step, '__doc__'): if hasattr(step, '__doc__'):
print(f"tzk: Step {idx}/{len(steps)}: {step.__doc__}") short_description = step.__doc__.strip().split('\n')[0].rstrip('.')
print(f"tzk: Step {idx}/{len(steps)}: {short_description}")
else: else:
print(f"tzk: Step {idx}/{len(steps)}") print(f"tzk: Step {idx}/{len(steps)}")
@ -219,31 +220,40 @@ class BuildCommand(CliCommand):
print(f"tzk: Build of product '{args.product}' completed successfully.") print(f"tzk: Build of product '{args.product}' completed successfully.")
parser = argparse.ArgumentParser() def launch():
subparsers = parser.add_subparsers() parser = argparse.ArgumentParser()
for command in sorted(CliCommand.__subclasses__(), key=lambda i: i.__name__):
subparser = subparsers.add_parser(command.cmd, help=command.help)
subparser.set_defaults(_cls=command)
command.setup_arguments(subparser) # type: ignore
args = parser.parse_args() subparsers = parser.add_subparsers()
for command in sorted(CliCommand.__subclasses__(), key=lambda i: i.__name__):
subparser = subparsers.add_parser(command.cmd, help=command.help)
subparser.set_defaults(_cls=command)
command.setup_arguments(subparser) # type: ignore
# For all operations except 'init', we start in the wiki folder. args = parser.parse_args()
if not args._cls.cmd == "init": if not hasattr(args, '_cls'):
if not cm.wiki_folder: parser.print_help()
fail("No 'wiki_folder' option found in config. Set this option to the name " sys.exit(0)
"of the wiki subfolder within the current directory.")
try: # For all operations except 'init', we start in the wiki folder.
os.chdir(cm.wiki_folder) if not args._cls.cmd == "init":
except FileNotFoundError: if not cm().wiki_folder:
fail(f"Tried to change directory into the wiki_folder '{cm.wiki_folder}' " fail("No 'wiki_folder' option found in config. Set this option to the name "
f"specified in your config file, but that directory does not exist.") "of the wiki subfolder within the current directory.")
if not os.path.exists("tiddlywiki.info"): try:
fail(f"After changing directory into {cm.wiki_folder} per your config file: " os.chdir(cm().wiki_folder)
f"Expected a 'tiddlywiki.info' file in {os.getcwd()}. " except FileNotFoundError:
f"Please check that your wiki is initialized " fail(f"Tried to change directory into the wiki_folder '{cm().wiki_folder}' "
f"and you specified the correct wiki_folder_name.") f"specified in your config file, but that directory does not exist.")
args._cls().execute(args) if not os.path.exists("tiddlywiki.info"):
fail(f"After changing directory into {cm().wiki_folder} per your config file: "
f"Expected a 'tiddlywiki.info' file in {os.getcwd()}. "
f"Please check that your wiki is initialized "
f"and you specified the correct wiki_folder_name.")
args._cls().execute(args)
if __name__ == '__main__':
launch()

526
tzk/builders.py Normal file
View File

@ -0,0 +1,526 @@
"""
*Builders* are small executable chunks that together can be linked into a useful build
process. Aside from two helper functions (:func:`stop()` and :func:`info()`)
to fail the build and display an informational message, respectively, all other
functions in this module are builders.
Builders are decorated with ``@tzk_builder``, a synonym for
:func:`_lazy_evaluable()`, which causes them to be lazy-evaluated: that is,
when they're initially called in the configuration, instead of running the
function and returning its result, a zero-argument function with all of the
arguments wrapped up is returned, to be run at a later time. This allows the
configuration file to be read at any time to retrieve information about the
defined products without actually running any build steps.
You can write and use custom builders right within your config file
by decorating them with ``@builders.tzk_builder``.
"""
from contextlib import contextmanager
import functools
import os
from pathlib import Path
import re
import shutil
import subprocess
import tempfile
from typing import Callable, Dict, List, Set, Sequence, Tuple
from tzk import git
from tzk import tw
from tzk.util import BuildError, pushd
def tzk_builder(func):
"""
Decorator which makes a function lazy-evaluable: that is, when it's
initially called, it returns a zero-argument lambda with the arguments
initially passed wrapped up in it. Calling that lambda has the effect
of executing the function.
We use this in TZK to allow the user to use function calls in her config
to define the build steps, while not requiring her to write a bunch of
ugly and confusing lambda:'s in the config. The functions that will be called
are prepared during the config and executed later.
"""
@functools.wraps(func)
def new_func(*args, **kwargs):
my_args = args
my_kwargs = kwargs
@functools.wraps(new_func)
def inner():
func(*my_args, **my_kwargs)
return inner
return new_func
def stop(message: str) -> None:
"Stop the build due to an error condition."
raise BuildError(message)
def info(message: str) -> None:
"Print information about this build step to the console."
print(message)
# Global state available to all builders.
build_state = {}
@tzk_builder
def say_hi(username: str) -> None:
"""
Say hi to the specified user.
This function is intended for testing. It will normally succeed and print
the provided *username*, but if you give "Jeff" as the username, the step
will fail, and if you give "General Failure" as the username, it will
cause an unhandled exception.
:param username: The name of the person to say hi to.
"""
if username.lower() == 'general failure':
raise Exception("")
elif username.lower() == 'jeff':
# https://twitter.com/yephph/status/1249246702126546944
stop("Sorry, the name Jeff does not work with our database schema.")
else:
print(f"Hello {username}!")
@tzk_builder
def require_branch(branchname: str) -> None:
"""
Require a specific Git branch to be checked out.
If the branch isn't checked out, the build will fail immediately. This may
be helpful if you want to be sure you aren't accidentally building a wiki
from tentative changes or an old version.
:param branchname: The name of the branch that must be checked out.
"""
if git.read("branch", "--show-current") != branchname:
stop(f"You may only run this build from the {branchname} branch.")
@tzk_builder
def require_clean_working_tree() -> None:
"""
Require the working tree of the Git repository to be clean.
If there are any unstaged changes to existing files or staged changes,
the build will fail immediately.
For the standard build process, it is not necessary for the working tree
to be clean. However, if you use any custom build steps that compare history,
or you simply want to ensure that you always have a recent checkpoint in your
local version whenever you publishing another version, this may be a useful
requirement.
"""
pleasecommit = "Please commit or stash them before publishing (try 'tzk commit')."
if git.rc("diff-index", "--quiet", "--cached", "HEAD", "--") != 0:
stop(f"There are staged changes. {pleasecommit}")
if git.rc("diff-files", "--quiet") != 0:
stop(f"There are uncommitted changes. {pleasecommit}")
@tzk_builder
def new_output_folder():
"""
Create a new temporary folder to hold intermediate steps of the product being built.
The path to this temporary folder will be stored in the 'public_wiki_folder'
key of the builders.build_state dictionary. Future build steps can access
the work in progress here. A cleaner is registered to delete this folder
when all steps complete, so any finished product should be copied out by a
later build step once it is complete.
"""
assert 'public_wiki_folder' not in build_state
build_state['public_wiki_folder'] = tempfile.mkdtemp()
new_output_folder.cleaner = lambda: shutil.rmtree(build_state['public_wiki_folder'])
@tzk_builder
def export_public_tiddlers(export_filter: str) -> None:
"""
Export specified tiddlers to a new wiki in the temporary build folder.
:func:`new_output_folder()` must be run prior to this builder.
:param export_filter: A TiddlyWiki filter describing the tiddlers to be selected
for inclusion in the new wiki.
"""
assert 'public_wiki_folder' in build_state, "new_output_folder builder must run first"
tw.exec((
("savewikifolder", build_state['public_wiki_folder'], export_filter),
))
def _find_kill_phrases(phrases: Set[str]):
"""
Search all tiddlers in the public_wiki_folder for the specified kill phrases.
"""
regexes = [re.compile(phrase) for phrase in phrases]
failures = []
tid_files = (Path(build_state['public_wiki_folder']) / "tiddlers").glob("**/*.tid")
for tid_file in tid_files:
with tid_file.open() as f:
for line in f:
for regex in regexes:
if re.search(regex, line):
failures.append((regex, str(tid_file), line))
return failures
@tzk_builder
def check_for_kill_phrases(kill_phrase_file: str = None) -> None:
"""
Fail the build if any of a series of regexes matches a tiddler's source in the temp wiki.
The temp wiki should be created first
using the :func:`export_public_tiddlers()` builder.
The kill phrases are Python-format regular expressions and may be configured
within the wiki, currently in $:/sib/gui/KillPhrases.
:param kill_phrase_file: The path from the source wiki's root directory to the
config tiddler containing kill phrases. In the default
Zettelkasten edition, this is
tiddlers/_system/config/zettelkasten/Build/KillPhrases.tid;
if you change the way paths are determined, you can give
a different path here.
"""
assert 'public_wiki_folder' in build_state, "new_output_folder builder must run first"
if kill_phrase_file is None:
kill_phrase_file = "tiddlers/_system/config/zettelkasten/Build/KillPhrases.tid"
kill_phrases = set()
with open(kill_phrase_file) as f:
reading = False
for line in f:
if not line.strip():
reading = True
if line.strip() and reading:
kill_phrases.add(line)
failures = _find_kill_phrases(kill_phrases)
if failures:
result = ["Kill phrases were found in your public wiki:"]
for failed_regex, failed_file, failed_line in failures:
trimmed_file = failed_file.replace(build_state['public_wiki_folder'], '')
result.append(f"'{failed_regex.pattern}' matched file {trimmed_file}:\n"
f" {failed_line.strip()}")
stop('\n'.join(result))
@tzk_builder
def save_attachments_externally(attachment_filter: str = "[is[image]]",
extimage_folder: str = "extimage") -> None:
"""
Save embedded files in the temp wiki into an external folder.
The temp wiki should be created first
using the :func:`export_public_tiddlers()` builder.
Note that this builder **does not finish externalizing images**.
It saves the images outside the wiki,
but it does not change the ``_canonical_uri`` and ``text`` fields
on each image tiddler to point to this new location.
For the latter step, use the
``externalize_attachments``, ``attachment_filter``, and ``canonical_uri_template``
parameters to the ``compile_html_file`` step.
:param attachment_filter: The tiddlers to be saved to the external folder;
by default, ``[is[image]]``.
:param extimage_folder: The name of the external folder to save to. This must
be the default of ``extimage`` to work with the default
``canonical_uri_template``
in the :func:`compile_html_file()` builder,
but you can use a different name and a different
canonical URI template if you prefer.
"""
assert 'public_wiki_folder' in build_state
tw.exec(
(
("savetiddlers", attachment_filter, extimage_folder),
),
base_wiki_folder=build_state['public_wiki_folder']
)
@tzk_builder
def compile_html_file(
wiki_name: str = "index.html",
output_folder: str = "output/public_site/",
overwrite: bool = True,
externalize_attachments: bool = False,
attachment_filter: str = "[is[image]]",
canonical_uri_template: str = "$:/core/templates/canonical-uri-external-image",
) -> None:
"""
Compile a single HTML file from the temp wiki.
Before compiling an HTML file,
you should create a temp wiki using the :func:`export_public_tiddlers()` builder,
then run any other build steps you want to use
to make changes to the wiki being built.
Once you compile the HTML file,
it will be copied out to an output location outside the temp folder,
and the content can no longer be changed through tzk.
:param wiki_name: The filename of the single-file wiki to create.
Default ``index.html``.
:param output_folder: The path to the folder
where the single-file wiki and any externalized attachments
will be placed, relative to the private wiki's root directory.
Default ``output/public_site``.
:param overwrite: If the ``output_folder`` already exists,
should we overwrite it? Default True.
:param externalize_attachments: If True, update tiddlers that match
the ``attachment_filter`` parameter to point to
external versions of their content.
Only useful if you have previously run
the ``save_attachments_externally`` builder.
Default False.
:param attachment_filter: If externalizing attachments,
which tiddlers should be externalized?
Default ``[is[image]]``.
:param canonical_uri_template: What TiddlyWiki template should be used
to determine the new content
of the ``_canonical_uri`` field?
Default ``$:/core/templates/canonical-uri-external-image``.
If you're not sure what this is, don't change it.
"""
assert 'public_wiki_folder' in build_state
commands: List[Tuple[str, ...]] = []
if externalize_attachments:
commands.extend([
("setfield", attachment_filter, "_canonical_uri",
canonical_uri_template, "text/plain"),
("setfield", attachment_filter, "text", "", "text/plain"),
])
commands.append(("render", "$:/core/save/all", wiki_name, "text/plain"))
tw.exec(commands, base_wiki_folder=build_state['public_wiki_folder'])
if os.path.exists(output_folder) and not overwrite:
stop(f"The output folder '{os.path.abspath(output_folder)}' already exists. "
f"(To overwrite any files existing in the output folder, "
f"set overwrite = True for this builder.)")
shutil.copytree(
Path(build_state['public_wiki_folder']) / "output",
Path(output_folder),
dirs_exist_ok=True
)
info(f"Successfully copied built output to {os.path.abspath(output_folder)}.")
def _private_people_replacement_table(
initialer: Callable[[str], str] = None) -> Dict[str, str]:
"Build table of private people and their transformed initials."
def _initials_from_tiddler_name(name: str) -> str:
m = re.match(r"^(?:Mr|Ms|Mx|The)(?P<camel_case_name>.*?)\.tid", name)
assert m
return '.'.join(i for i in m.group('camel_case_name') if i.isupper()) + '.'
if initialer is None:
initialer = _initials_from_tiddler_name
tiddlers = (Path.cwd() / "tiddlers").glob("**/*.tid")
person_tiddlers = (i for i in tiddlers if re.match("^(Mr|Ms|Mx|The)", i.name))
private_person_tiddlers = []
for pt in person_tiddlers:
with pt.open() as f:
for line in f:
if line.startswith("tags:"):
if re.search(r'\bPublic\b', line):
# If there's a tags line in the file and it contains the
# Public tag, we skip it.
break
else:
# Otherwise, if there's a tags line in the file and it
# doesn't contain the Public tag, it's private.
private_person_tiddlers.append(pt)
break
if not line.strip():
# And if there's no tags line in the file at all,
# it's private by default.
private_person_tiddlers.append(pt)
break
return {
i.name.replace('.tid', ''): initialer(i.name)
for i in private_person_tiddlers
}
@tzk_builder
def replace_private_people(initialer: Callable[[str], str] = None) -> None:
"""
Replace the names of people who are not marked Public with their initials.
If you have lots of PAO (People, Animals, and Organizations) in your Zettelkasten
and many of them are personal friends,
you might prefer not to make everything you said about them
easily searchable on the internet.
This is more challenging than simply not marking their tiddlers public,
since
This builder replaces all links, bracketed or WikiCamelCase,
to the names of all people *not* tagged Public
with the initials suggested by their CamelCase titles
(e.g., MsJaneDoe becomes J.D.). The links point to the tiddler ``PrivatePerson``,
which explains this process.
:param initialer: If you don't like the way that initials
are generated from tiddler filenames by default,
you can customize it by passing a callable
that takes one string argument
(a tiddler filename without the full path, e.g., ``MsJaneDoe.tid``)
and returns a string to be considered the "initials" of that person.
"""
assert 'public_wiki_folder' in build_state
replacement_table = _private_people_replacement_table(initialer)
tid_files = (Path(build_state['public_wiki_folder']) / "tiddlers").glob("**/*.tid")
for tiddler in tid_files:
dirty = False
with tiddler.open() as f:
lines = f.readlines()
for idx, line in enumerate(lines):
for replace_person, replace_initials in replacement_table.items():
if replace_person in line:
if '|' + replace_person + ']]' in line:
# link with the person as the target only;
# beware that you might have put something private in the text
lines[idx] = line.replace(replace_person,
'PrivatePerson')
elif '[[' + replace_person + ']]' in line:
# link with the person as the target and text
lines[idx] = line.replace(replace_person,
replace_initials + '|PrivatePerson')
else:
# camel-case link or unlinked reference in text;
# or spurious substring, so rule that out with the '\b' search
lines[idx] = re.sub(
r"\b" + re.escape(replace_person) + r"\b",
f'<<privateperson "{replace_initials}">>',
line
)
dirty = True
if dirty:
with tiddler.open("w") as f:
f.writelines(lines)
@tzk_builder
def set_tiddler_values(mappings: Dict[str, str]) -> None:
"""
Set the 'text' field of selected config or other tiddlers to arbitrary new values.
This can be used to make customizations that can't easily be done with feature
flags or other wikitext solutions within the wiki -- for instance, changing the
subtitle or what buttons are visible. It's also used to implement feature flags
in the first place by changing the ``$:/config/sib/CurrentEditionPublicity``
tiddler to ``public``, so the minimal functional wiki will use:
```python
{'$__config_sib_CurrentEditionPublicity.tid': 'public',}
```
:param mappings: A dictionary whose keys are tiddler names
and whose values are the values to be inserted
in those tiddlers' ``text`` fields.
"""
assert 'public_wiki_folder' in build_state
for tiddler, new_text in mappings.items():
tiddler_path = (Path(build_state['public_wiki_folder']) / "tiddlers" / tiddler)
with tiddler_path.open("r") as f:
tiddler_lines = f.readlines()
first_blank_line_index = next(idx
for idx, value in enumerate(tiddler_lines)
if not value.strip())
with tiddler_path.open("w") as f:
f.writelines(tiddler_lines[0:first_blank_line_index+1])
f.write(new_text)
@tzk_builder
def publish_wiki_to_github(
output_folder: str = "output/public_site/",
commit_message: str = "publish checkpoint",
remote: str = "origin",
refspec: str = "master",
push = True) -> None:
"""
Publish the built wiki to GitHub.
:param output_folder: Path to a folder containing the Git repository you'd like
to publish, relative to your private wiki's root directory.
This folder should contain the version of your wiki
built by the :func:`compile_html_file()` builder,
either directly or in a subfolder somewhere.
You need to clone or create the repository yourself
and set up the remotes as appropriate
before running this step.
Default ``output/public_site``.
:param commit_message: Message to use when committing the newly built wiki.
Note that all changes in the repository will be committed.
Default ``publish checkpoint``.
:param remote: The repository remote to push changes to.
If you don't know what that is,
the default of ``origin`` is probably correct.
:param refspec: The local branch or refspec to push.
Default ``master``.
:param push: If set to False, don't push after committing the changes
(mostly useful for testing purposes).
Default True.
"""
with pushd(output_folder):
if not os.path.isdir(".git"):
stop(f"The output folder {output_folder} isn't a Git repository. "
f"Please go initialize it and then try again.")
git.exec("add", "-A")
rc = git.rc("commit", "-m", commit_message)
if rc == 0:
if push:
git.exec("push", remote, refspec)
elif rc == 1:
info("No changes to commit or publish. "
"You probably rebuilt without changing the wiki in between.")
else:
stop(f"'git commit' returned unknown return code {rc}.")
@tzk_builder
def shell(shell_command: str) -> None:
"""
Run an arbitrary shell command.
The builder will fail if a return code other than 0 is returned,
otherwise it will succeed. Output will be printed to stdout.
:param shell_command: A string to be passed to your system's shell.
"""
info("$ " + shell_command)
try:
output = subprocess.check_output(shell_command, shell=True, text=True)
except subprocess.CalledProcessError as e:
if e.output.strip():
stop(f"Command exited with return code {e.returncode}:\n{e.output}")
else:
stop(f"Command exited with return code {e.returncode} (no output).")
else:
if output.strip():
info(f"Command exited with return code 0:\n{output}")
else:
info(f"Command exited with return code 0 (no output).")

7
config.py → tzk/config.py
View File

@ -6,7 +6,7 @@ from pathlib import Path
import sys import sys
from typing import Any from typing import Any
from util import fail from tzk.util import fail
class ConfigurationManager: class ConfigurationManager:
@ -56,4 +56,7 @@ class ConfigurationManager:
return True return True
cm = ConfigurationManager() def cm(cache=[]):
if not cache:
cache.append(ConfigurationManager())
return cache[0]

0
git.py → tzk/git.py
View File

12
tw.py → tzk/tw.py
View File

@ -6,9 +6,9 @@ import subprocess
from textwrap import dedent from textwrap import dedent
from typing import Optional, Sequence from typing import Optional, Sequence
import config from tzk import config
import git from tzk import git
from util import pushd from tzk.util import pushd
@functools.lru_cache(1) @functools.lru_cache(1)
@ -85,12 +85,12 @@ def _save_wikifolder_to_config(wiki_name: str) -> bool:
Return True if the option ended set to wiki_name, False otherwise. Return True if the option ended set to wiki_name, False otherwise.
""" """
print("tzk: Writing new wiki folder to config file...") print("tzk: Writing new wiki folder to config file...")
if not config.cm.write_attr("wiki_folder", wiki_name): if not config.cm().write_attr("wiki_folder", wiki_name):
if config.cm.wiki_folder == wiki_name: if config.cm().wiki_folder == wiki_name:
print("tzk: (Looks like it was already there.)") print("tzk: (Looks like it was already there.)")
else: else:
print(f"tzk: WARNING: The wiki_folder option in your config appears " print(f"tzk: WARNING: The wiki_folder option in your config appears "
f"to be set to '{config.cm.wiki_folder}', rather than the wiki folder " f"to be set to '{config.cm().wiki_folder}', rather than the wiki folder "
f"you're initializing, {wiki_name}. Please check your config file " f"you're initializing, {wiki_name}. Please check your config file "
"and update this option if necessary.") "and update this option if necessary.")
return False return False

0
util.py → tzk/util.py
View File