shell builder: subscriptions and clean target

builders.py

@@ -4,12 +4,13 @@ import os
 from pathlib import Path
 import re
 import shutil
+import subprocess
 import tempfile
-from typing import Dict, List, Set, Tuple
+from typing import Dict, List, Set, Sequence, Tuple
 
 import git
 import tw
-from util import BuildError
+from util import BuildError, pushd
 
 
 def _lazy_evaluable(func):
@@ -153,8 +154,8 @@ def save_attachments_externally(attachment_filter: str = "[is[image]]",
 @tzk_builder
 def compile_html_file(
         wiki_name: str = "index.html",
-        output_folder: str = "public_site/",
+        output_folder: str = "output/public_site/",
-        remove_output: bool = False,
+        overwrite: bool = True,
         externalize_attachments: bool = False,
         attachment_filter: str = "[is[image]]",
         canonical_uri_template: str = "$:/core/templates/canonical-uri-external-image",
@@ -172,17 +173,15 @@ def compile_html_file(
     commands.append(("render", "$:/core/save/all", wiki_name, "text/plain"))
 
     tw.exec(commands, base_wiki_folder=build_state['public_wiki_folder'])
-    if os.path.exists(output_folder) and not remove_output:
+    if os.path.exists(output_folder) and not overwrite:
         stop(f"The output folder '{os.path.abspath(output_folder)}' already exists. "
-             f"(To delete the output folder if it exists, set remove_output = True "
+             f"(To overwrite any files existing in the output folder, "
-             f"for this builder.)")
+             f"set overwrite = True for this builder.)")
-    elif os.path.exists(output_folder) and remove_output:
-        info(f"Removing existing output folder {os.path.abspath(output_folder)}.")
-        shutil.rmtree(output_folder)
 
     shutil.copytree(
         Path(build_state['public_wiki_folder']) / "output",
-        Path(output_folder)
+        Path(output_folder),
+        dirs_exist_ok=True
     )
     info(f"Successfully copied built output to {os.path.abspath(output_folder)}.")
 
@@ -282,15 +281,41 @@ def publish_wiki_to_github(
         output_folder: str = "output/public_site/",
         commit_message: str = "publish checkpoint",
         remote: str = "origin",
-        refspec: str = "master") -> None:
+        refspec: str = "master",
+        push = True) -> None:
     "Publish the wiki to GitHub"
 
-    os.chdir(output_folder)
+    with pushd(output_folder):
-    if not os.path.isdir(".git"):
+        if not os.path.isdir(".git"):
-        info(f"The output folder {output_folder} doesn't appear to be a Git repository. "
+            info(f"The output folder {output_folder} doesn't appear to be a Git repository. "
-             f"I'll try to make it one.")
+                f"I'll try to make it one.")
-        git.exec("init")
+            git.exec("init")
 
-    git.exec("add", "-A")
+        git.exec("add", "-A")
-    git.exec("commit", "-m", commit_message)
+        rc = git.rc("commit", "-m", commit_message)
-    git.exec("push", remote, refspec)
+        if rc == 0:
+            if push:
+                git.exec("push", remote, refspec)
+        elif rc == 1:
+            info("No changes to commit or publish. "
+                "You probably rebuilt without changing the wiki in between.")
+        else:
+            stop(f"'git commit' returned unknown return code {rc}.")
+
+
+@tzk_builder
+def shell(shell_command: str) -> None:
+    "Run an arbitrary shell command"
+    info("$ " + shell_command)
+    try:
+        output = subprocess.check_output(shell_command, shell=True, text=True)
+    except subprocess.CalledProcessError as e:
+        if e.output.strip():
+            stop(f"Command exited with return code {e.returncode}:\n{e.output}")
+        else:
+            stop(f"Command exited with return code {e.returncode} (no output).")
+    else:
+        if output.strip():
+            info(f"Command exited with return code 0:\n{output}")
+        else:
+            info(f"Command exited with return code 0 (no output).")

tw.py

@@ -1,4 +1,3 @@
-from contextlib import contextmanager
 import functools
 import json
 import os
@@ -9,25 +8,13 @@ from typing import Optional, Sequence
 
 import config
 import git
+from util import pushd
 
 
 @functools.lru_cache(1)
 def _npm_bin() -> str:
     return subprocess.check_output(("npm", "bin"), text=True).strip()
 
-@contextmanager
-def _pushd(directory: str):
-    """
-    Change directory into the directory /directory/ until the end of the with-block,
-    then return to previous directory.
-    """
-    old_directory = os.getcwd()
-    try:
-        os.chdir(directory)
-        yield
-    finally:
-        os.chdir(old_directory)
-
 def _tw_path() -> str:
     return _npm_bin() + "/tiddlywiki"
 
@@ -86,7 +73,7 @@ def _init_tw(wiki_name: str) -> None:
         os.mkdir(wiki_name)
     except FileExistsError:
         pass
-    with _pushd(wiki_name):
+    with pushd(wiki_name):
         subprocess.check_call((_tw_path(), "--init"))
 
 

tzk.py

@@ -9,7 +9,7 @@ from typing import Optional
 from config import cm
 import git
 import tw
-from util import BuildError, fail
+from util import BuildError, fail, numerize
 
 
 class CliCommand(ABC):
@@ -166,6 +166,7 @@ class BuildCommand(CliCommand):
             help="Function name of a builder to skip even if part of the PRODUCT. "
                  "This option can be specified multiple times.",
             action="append",
+            default=[],
         )
 
     def _precheck(self, product: str) -> None:
@@ -183,7 +184,7 @@ class BuildCommand(CliCommand):
 
         steps = cm.products[args.product]
         print(f"tzk: Starting build of product '{args.product}'.")
-        print(f"tzk: Found {len(steps)} build steps.")
+        print(f"tzk: Found {len(steps)} build {numerize(len(steps), 'step')}.")
 
         for idx, step in enumerate(steps, 1):
             if hasattr(step, '__doc__'):

util.py

@@ -1,3 +1,5 @@
+from contextlib import contextmanager
+import os
 import sys
 from typing import NoReturn
 
@@ -9,3 +11,27 @@ class BuildError(Exception):
 def fail(msg: str, exit_code: int = 1) -> NoReturn:
     print(msg, file=sys.stderr)
     sys.exit(exit_code)
+
+
+def numerize(number: int, singular: str, plural: str = None):
+    if plural is None:
+        plural = singular + 's'
+
+    if number == 1:
+        return singular
+    else:
+        return plural
+
+
+@contextmanager
+def pushd(directory: str):
+    """
+    Change directory into the directory /directory/ until the end of the with-block,
+    then return to previous directory.
+    """
+    old_directory = os.getcwd()
+    try:
+        os.chdir(directory)
+        yield
+    finally:
+        os.chdir(old_directory)