Clean up tutorial notebook output noise and version drift

README.md

@@ -73,6 +73,29 @@ docker compose exec docs jupyter nbconvert --to notebook --execute --inplace \
     /main/src/tutorials/YOUR_NOTEBOOK.ipynb
 ```
 
+### Notebook execution policy
+
+Tutorial and how-to notebooks under `src/tutorials/` and `src/how-to/` are
+committed **with their executed cell outputs**. The static site renders these
+outputs verbatim via `mkdocs-jupyter` — the build container does not execute
+notebooks, so the rendered page always matches what was committed.
+
+Refresh outputs whenever the DataJoint version pinned in `mkdocs.yaml`
+(`extra.datajoint_version`) changes, or whenever a tutorial's code changes:
+
+```bash
+# Re-execute against the bind-mounted local datajoint-python checkout
+DJ_PYTHON_PATH=../datajoint-python MODE=EXECUTE    docker compose up --build
+DJ_PYTHON_PATH=../datajoint-python MODE=EXECUTE_PG docker compose up --build
+```
+
+A guard script flags notebooks whose committed `DataJoint X.Y.Z connected`
+banner doesn't match `extra.datajoint_version`:
+
+```bash
+python scripts/check_notebook_versions.py
+```
+
 ## Related
 
 - [datajoint-python](https://github.com/datajoint/datajoint-python) — Core library

docker-compose.yaml

@@ -96,15 +96,15 @@ services:
         elif echo "$${MODE}" | grep -i execute_pg &>/dev/null; then
             # EXECUTE_PG mode: execute notebooks against PostgreSQL
             pip install -e "/datajoint-python[postgres]"
-            pip install scikit-image pooch nbconvert matplotlib faker zarr
+            pip install scikit-image pooch nbconvert matplotlib faker zarr ipywidgets
             mkdir -p /tmp/datajoint-tutorials
             echo "Executing notebooks against PostgreSQL..."
             export DJ_HOST=postgres
             python scripts/execute_notebooks.py --backend postgresql
         elif echo "$${MODE}" | grep -i execute &>/dev/null; then
             # EXECUTE mode: execute notebooks against MySQL (default)
             pip install -e /datajoint-python
-            pip install scikit-image pooch nbconvert matplotlib faker zarr
+            pip install scikit-image pooch nbconvert matplotlib faker zarr ipywidgets
             mkdir -p /tmp/datajoint-tutorials
             echo "Executing notebooks against MySQL..."
             python scripts/execute_notebooks.py --backend mysql

scripts/check_notebook_versions.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+"""
+Verify that every executed notebook's DataJoint connection banner
+matches the version configured in mkdocs.yaml (extra.datajoint_version).
+
+The banner DataJoint prints on connection looks like:
+    [2026-02-19 18:32:59] DataJoint 2.2.2 connected to postgres@postgres:5432
+
+We accept any patch within the configured major.minor (e.g., "2.2"
+matches 2.2.0, 2.2.1, 2.2.2, ...). Banners with a different major.minor
+fail the check.
+
+Notebooks with no banner are skipped (they may not connect to a database).
+
+Exit codes:
+    0 - all banners current (or absent)
+    1 - one or more notebooks stale
+    2 - configuration / parsing error
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import sys
+from pathlib import Path
+
+BANNER_RE = re.compile(r"DataJoint\s+(\d+)\.(\d+)\.(\d+)\s+connected")
+# mkdocs.yaml uses Material-specific YAML tags (!!python/name:...) that PyYAML's
+# safe_load rejects, so pull the version line out with a regex instead.
+VERSION_KEY_RE = re.compile(
+    r'^\s*datajoint_version:\s*["\']?(\d+)\.(\d+)(?:\.\d+)?["\']?',
+    re.MULTILINE,
+)
+
+
+def load_target_version(mkdocs_yaml: Path) -> tuple[int, int]:
+    text = mkdocs_yaml.read_text()
+    m = VERSION_KEY_RE.search(text)
+    if not m:
+        print(
+            f"error: could not find datajoint_version in {mkdocs_yaml}",
+            file=sys.stderr,
+        )
+        sys.exit(2)
+    return int(m.group(1)), int(m.group(2))
+
+
+def iter_banner_versions(notebook_path: Path):
+    with notebook_path.open() as f:
+        nb = json.load(f)
+    for cell in nb.get("cells", []):
+        for out in cell.get("outputs", []) or []:
+            chunks = []
+            text = out.get("text")
+            if isinstance(text, list):
+                chunks.extend(text)
+            elif isinstance(text, str):
+                chunks.append(text)
+            data = out.get("data", {}) or {}
+            plain = data.get("text/plain")
+            if isinstance(plain, list):
+                chunks.extend(plain)
+            elif isinstance(plain, str):
+                chunks.append(plain)
+            for chunk in chunks:
+                for m in BANNER_RE.finditer(chunk):
+                    yield (int(m.group(1)), int(m.group(2)), int(m.group(3)))
+
+
+def main() -> int:
+    repo = Path(__file__).resolve().parent.parent
+    mkdocs_yaml = repo / "mkdocs.yaml"
+    target_major, target_minor = load_target_version(mkdocs_yaml)
+
+    search_dirs = [repo / "src" / "tutorials", repo / "src" / "how-to"]
+    notebooks: list[Path] = []
+    for d in search_dirs:
+        if d.exists():
+            notebooks.extend(p for p in d.rglob("*.ipynb") if ".ipynb_checkpoints" not in str(p))
+    notebooks.sort()
+
+    stale: list[tuple[Path, tuple[int, int, int]]] = []
+    checked = 0
+    for nb in notebooks:
+        had_banner = False
+        for ver in iter_banner_versions(nb):
+            had_banner = True
+            if ver[0] != target_major or ver[1] != target_minor:
+                stale.append((nb.relative_to(repo), ver))
+                break
+        if had_banner:
+            checked += 1
+
+    if stale:
+        print(f"Stale DataJoint connection banner(s) (target: {target_major}.{target_minor}.x):")
+        for path, ver in stale:
+            print(f"  {path}: found {ver[0]}.{ver[1]}.{ver[2]}")
+        print(f"\nRe-execute notebooks with MODE=EXECUTE or MODE=EXECUTE_PG.")
+        return 1
+
+    print(f"OK: {checked} notebook(s) with banners are on DataJoint {target_major}.{target_minor}.x")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/execute_notebooks.py

@@ -173,6 +173,17 @@ def main():
     print(f"  DJ_PORT: {env.get('DJ_PORT')}")
     print(f"  DJ_USER: {env.get('DJ_USER')}")
 
+    # Pre-cache scikit-image datasets so the one-time "Downloading file ..."
+    # message doesn't leak into committed notebook outputs.
+    print("\nPre-caching scikit-image datasets...")
+    import skimage.data as _sk_data
+    for _loader in (_sk_data.hubble_deep_field, _sk_data.human_mitosis):
+        try:
+            _loader()
+            print(f"  cached: {_loader.__name__}")
+        except Exception as _e:
+            print(f"  pre-cache warn: {_loader.__name__}: {_e}")
+
     # Find notebooks
     notebooks = find_notebooks(args.base_path)
     print(f"\nFound {len(notebooks)} notebooks to execute\n")

src/how-to/master-part.ipynb

@@ -16,18 +16,18 @@
    "id": "cell-setup",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2026-02-19T18:32:08.746158Z",
-     "iopub.status.busy": "2026-02-19T18:32:08.746031Z",
-     "iopub.status.idle": "2026-02-19T18:32:09.113193Z",
-     "shell.execute_reply": "2026-02-19T18:32:09.112827Z"
+     "iopub.execute_input": "2026-05-18T19:42:27.675571Z",
+     "iopub.status.busy": "2026-05-18T19:42:27.675251Z",
+     "iopub.status.idle": "2026-05-18T19:42:28.026876Z",
+     "shell.execute_reply": "2026-05-18T19:42:28.026279Z"
     }
    },
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "[2026-02-19 18:32:09] DataJoint 2.1.1 connected to postgres@postgres:5432\n"
+      "[2026-05-18 19:42:28] DataJoint 2.2.2 connected to postgres@postgres:5432\n"
      ]
     }
    ],
@@ -84,10 +84,10 @@
    "id": "cell-define",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2026-02-19T18:32:09.114731Z",
-     "iopub.status.busy": "2026-02-19T18:32:09.114545Z",
-     "iopub.status.idle": "2026-02-19T18:32:09.402980Z",
-     "shell.execute_reply": "2026-02-19T18:32:09.402650Z"
+     "iopub.execute_input": "2026-05-18T19:42:28.027920Z",
+     "iopub.status.busy": "2026-05-18T19:42:28.027760Z",
+     "iopub.status.idle": "2026-05-18T19:42:28.311816Z",
+     "shell.execute_reply": "2026-05-18T19:42:28.311281Z"
     }
    },
    "outputs": [
@@ -156,7 +156,7 @@
        "</svg>"
       ],
       "text/plain": [
-       "<datajoint.diagram.Diagram at 0xffff96138110>"
+       "<datajoint.diagram.Diagram at 0xffff764a9d90>"
       ]
      },
      "execution_count": 2,
@@ -225,10 +225,10 @@
    "id": "cell-insert",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2026-02-19T18:32:09.404028Z",
-     "iopub.status.busy": "2026-02-19T18:32:09.403932Z",
-     "iopub.status.idle": "2026-02-19T18:32:09.411007Z",
-     "shell.execute_reply": "2026-02-19T18:32:09.410764Z"
+     "iopub.execute_input": "2026-05-18T19:42:28.313033Z",
+     "iopub.status.busy": "2026-05-18T19:42:28.312945Z",
+     "iopub.status.idle": "2026-05-18T19:42:28.319781Z",
+     "shell.execute_reply": "2026-05-18T19:42:28.319299Z"
     }
    },
    "outputs": [
@@ -379,10 +379,10 @@
    "id": "cell-query",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2026-02-19T18:32:09.411953Z",
-     "iopub.status.busy": "2026-02-19T18:32:09.411863Z",
-     "iopub.status.idle": "2026-02-19T18:32:09.416047Z",
-     "shell.execute_reply": "2026-02-19T18:32:09.415764Z"
+     "iopub.execute_input": "2026-05-18T19:42:28.320926Z",
+     "iopub.status.busy": "2026-05-18T19:42:28.320831Z",
+     "iopub.status.idle": "2026-05-18T19:42:28.324757Z",
+     "shell.execute_reply": "2026-05-18T19:42:28.324278Z"
     }
    },
    "outputs": [
@@ -497,10 +497,10 @@
    "id": "cell-join",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2026-02-19T18:32:09.416915Z",
-     "iopub.status.busy": "2026-02-19T18:32:09.416834Z",
-     "iopub.status.idle": "2026-02-19T18:32:09.420923Z",
-     "shell.execute_reply": "2026-02-19T18:32:09.420692Z"
+     "iopub.execute_input": "2026-05-18T19:42:28.325531Z",
+     "iopub.status.busy": "2026-05-18T19:42:28.325457Z",
+     "iopub.status.idle": "2026-05-18T19:42:28.330016Z",
+     "shell.execute_reply": "2026-05-18T19:42:28.329459Z"
     }
    },
    "outputs": [
@@ -618,10 +618,10 @@
    "id": "cell-aggr",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2026-02-19T18:32:09.421737Z",
-     "iopub.status.busy": "2026-02-19T18:32:09.421664Z",
-     "iopub.status.idle": "2026-02-19T18:32:09.425961Z",
-     "shell.execute_reply": "2026-02-19T18:32:09.425681Z"
+     "iopub.execute_input": "2026-05-18T19:42:28.330816Z",
+     "iopub.status.busy": "2026-05-18T19:42:28.330742Z",
+     "iopub.status.idle": "2026-05-18T19:42:28.335817Z",
+     "shell.execute_reply": "2026-05-18T19:42:28.335320Z"
     }
    },
    "outputs": [
@@ -736,10 +736,10 @@
    "id": "cell-computed",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2026-02-19T18:32:09.426708Z",
-     "iopub.status.busy": "2026-02-19T18:32:09.426642Z",
-     "iopub.status.idle": "2026-02-19T18:32:09.469961Z",
-     "shell.execute_reply": "2026-02-19T18:32:09.469669Z"
+     "iopub.execute_input": "2026-05-18T19:42:28.336815Z",
+     "iopub.status.busy": "2026-05-18T19:42:28.336587Z",
+     "iopub.status.idle": "2026-05-18T19:42:28.376859Z",
+     "shell.execute_reply": "2026-05-18T19:42:28.376290Z"
     }
    },
    "outputs": [
@@ -891,39 +891,39 @@
    "id": "cell-delete",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2026-02-19T18:32:09.470903Z",
-     "iopub.status.busy": "2026-02-19T18:32:09.470819Z",
-     "iopub.status.idle": "2026-02-19T18:32:09.494950Z",
-     "shell.execute_reply": "2026-02-19T18:32:09.494673Z"
+     "iopub.execute_input": "2026-05-18T19:42:28.377784Z",
+     "iopub.status.busy": "2026-05-18T19:42:28.377695Z",
+     "iopub.status.idle": "2026-05-18T19:42:28.415908Z",
+     "shell.execute_reply": "2026-05-18T19:42:28.415372Z"
     }
    },
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "[2026-02-19 18:32:09] Deleting 3 rows from \"howto_masterpart\".\"__session_analysis__trial_score\"\n"
+      "[2026-05-18 19:42:28] Deleting 3 rows from \"howto_masterpart\".\"__session_analysis__trial_score\"\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "[2026-02-19 18:32:09] Deleting 3 rows from \"howto_masterpart\".\"session__trial\"\n"
+      "[2026-05-18 19:42:28] Deleting 1 rows from \"howto_masterpart\".\"__session_analysis\"\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "[2026-02-19 18:32:09] Deleting 1 rows from \"howto_masterpart\".\"__session_analysis\"\n"
+      "[2026-05-18 19:42:28] Deleting 3 rows from \"howto_masterpart\".\"session__trial\"\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "[2026-02-19 18:32:09] Deleting 1 rows from \"howto_masterpart\".\"session\"\n"
+      "[2026-05-18 19:42:28] Deleting 1 rows from \"howto_masterpart\".\"session\"\n"
      ]
     },
     {
@@ -973,10 +973,10 @@
    "id": "cell-cleanup",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2026-02-19T18:32:09.495889Z",
-     "iopub.status.busy": "2026-02-19T18:32:09.495806Z",
-     "iopub.status.idle": "2026-02-19T18:32:09.501035Z",
-     "shell.execute_reply": "2026-02-19T18:32:09.500610Z"
+     "iopub.execute_input": "2026-05-18T19:42:28.416778Z",
+     "iopub.status.busy": "2026-05-18T19:42:28.416689Z",
+     "iopub.status.idle": "2026-05-18T19:42:28.420860Z",
+     "shell.execute_reply": "2026-05-18T19:42:28.420466Z"
     }
    },
    "outputs": [],
@@ -1001,7 +1001,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.12.12"
+   "version": "3.12.13"
   }
  },
  "nbformat": 4,