From e326fbca7443dfb1a873ace572090fbd72206936 Mon Sep 17 00:00:00 2001 From: lauramurgatroyd Date: Fri, 8 May 2026 16:12:43 +0000 Subject: [PATCH 1/5] WIP --- README.md | 53 ++++++++++++++++++++++++++ test_notebooks_sequential.py | 73 +++++++++++++++++++++++------------- 2 files changed, 100 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 4a7a8cb..5930fc2 100644 --- a/README.md +++ b/README.md @@ -4,3 +4,56 @@ Static hosting: - Source: https://github.com/TomographicImaging/scripts - Rendered: https://tomographicimaging.github.io/scripts + +## Environment files for installing software + +### CIL-Demos +`cil_demos.yml` - for installing the latest release of CIL-Demos on a system with a GPU. +`cil_demos_cpu.yml` - for installing the latest release of CIL-Demos on a system without a GPU. + +### CIL +`cil.yml` - for installing the latest release of CIL. +`cil_test.yml` - for installing the latest + +### CILViewer +`cilviewer_ui` - for installing the latest release of CILViewer, with +## Script for Testing CIL demos + +`test_notebooks_sequential.py` is a script for testing the CIL-Demos + +To use it: +1. Clone [CIL-Demos](https://github.com/TomographicImaging/CIL-Demos), if you haven't already, and move to the branch of CIL-Demos you would like to test. +2. Clone [scripts](https://github.com/TomographicImaging/scripts) +3. Download all datasets used by the [CIL-Demos](https://github.com/TomographicImaging/CIL-Demos) notebooks you would like to test. +4. Create an environment using: + ```sh + conda env create -f https://tomographicimaging.github.io/scripts/env/cil_test_demos.yml + ``` +5. Clone [CIL](https://github.com/TomographicImaging/CIL), if you haven't already, and move to the branch of CIL you would like to test the demos with. +6. Activate the environment you created: + ```sh + conda activate cil_test_demos + ``` +7. Navigate into the CIL directory and install CIL into this environment: + ```sh + pip install -e . + ``` +8. The script uses `nbmake` and needs to point to the kernel to run the notebooks with. We need to make sure our conda environment is registered with the correct jupyter kernel name. Run: + ```sh + jupyter kernelspec list + ``` +If there is a kernel with name `cil_test_demos` then move to step 9. If not, run: + ```sh + python -m ipykernel install --user --name cil_test_demos --display-name "cil_test_demos" + ``` +9. At the top of the `test_notebooks_sequential.py` file, update these variables: + `CIL_DEMOS_DIR` - should point to your clone of CIL-Demos + `DATA_PATH` and `DATA_PATH_ALT` should point to directories where the CIL-Demos data is saved. +10. Run `test_notebooks_sequential.py`. This is likely to take quite a long time. It will create the following inside the `scripts` directory: + `test_notebooks_*.log` log file (with name including date and time of run) + This shows which notebooks have run and whether they have passed or failed. + `tmp_notebooks` directory + This contains the run and rendered `.ipynb` notebook files. + `html_outputs` directory + This contains run and rendered notebooks as `html` files. + diff --git a/test_notebooks_sequential.py b/test_notebooks_sequential.py index 5a58497..df8633f 100644 --- a/test_notebooks_sequential.py +++ b/test_notebooks_sequential.py @@ -10,7 +10,7 @@ DATA_PATH_ALT = '/mnt/share/materials/CIL/' SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) -CIL_DEMOS_DIR = os.path.expanduser(os.path.join('~', 'CIL-Demos')) +CIL_DEMOS_DIR = os.path.expanduser("/home/lhe97136/CIL-Demos") PDF_OUTPUT_DIR = os.path.join(SCRIPT_DIR, 'html_outputs') TMP_OUTPUT_DIR = os.path.join(SCRIPT_DIR, 'tmp_notebooks') @@ -25,15 +25,42 @@ LOG_NAME = "test_notebooks_" + datetime.now().strftime("%Y%m%d_%H%M%S") + ".log" -def preprocess_notebook(original_path): - rel_path = os.path.relpath(original_path, CIL_DEMOS_DIR) - rel_path = rel_path.replace("..", "_") - tmp_path = os.path.join(TMP_OUTPUT_DIR, rel_path) - tmp_dir = os.path.dirname(tmp_path) - os.makedirs(tmp_dir, exist_ok=True) - tmp_path = tmp_path.replace('.ipynb', '_tmp.ipynb') +def copy_all_files(): + """ + Copy all *.ipynb and *.py files from `folders` into TMP_OUTPUT_DIR, + preserving relative directory structure. + Renames any *.ipynb files to *_tmp.ipynb + + Returns a list of tuples: (original_path, copied_path) for all notebooks. + """ + notebook_paths = [] + for folder in folders: + for root, _, files in os.walk(folder): + for file in files: + if file.endswith('.ipynb') or file.endswith('.py'): + original_path = os.path.join(root, file) + + rel_path = os.path.relpath(original_path, CIL_DEMOS_DIR) + rel_path = rel_path.replace("..", "_") + tmp_path = os.path.join(TMP_OUTPUT_DIR, rel_path) + tmp_dir = os.path.dirname(tmp_path) + os.makedirs(tmp_dir, exist_ok=True) + if file.endswith('.ipynb') and not file.endswith('_tmp.ipynb'): + tmp_path = tmp_path.replace('.ipynb', '_tmp.ipynb') + if os.path.exists(tmp_path): + with open(LOG_NAME, "a") as log_file: + log_file.write(f"Skipping (tmp exists): {original_path}\n") + + if not os.path.exists(tmp_path): + shutil.copy(original_path, tmp_path) + + if file.endswith('.ipynb'): + notebook_paths.append((original_path, tmp_path)) + return notebook_paths + - shutil.copy(original_path, tmp_path) + +def preprocess_notebook(original_path, tmp_path): with open(tmp_path, 'r') as f: notebook = nbformat.read(f, as_version=4) @@ -183,25 +210,19 @@ def cleanup_notebook_files(tmp_path): def main(): results = {} all_warnings = {} - for folder in folders: - for root, _, files in os.walk(folder): - for file in files: - if file.endswith('.ipynb') and not file.endswith('_tmp.ipynb'): - original_path = os.path.join(root, file) - rel_path = os.path.relpath(original_path, CIL_DEMOS_DIR) - rel_path = rel_path.replace("..", "_") - tmp_path = os.path.join(TMP_OUTPUT_DIR, rel_path).replace('.ipynb', '_tmp.ipynb') + with open(LOG_NAME, "a") as log: + log.write("=== Copying Files ===\n") - if os.path.exists(tmp_path): - with open(LOG_NAME, "a") as log_file: - log_file.write(f"Skipping (tmp exists): {original_path}\n") - continue + notebooks = copy_all_files() - with open(LOG_NAME, "a") as log_file: - log_file.write(f"\n=== Processing notebook: {original_path} ===\n") - tmp_path = preprocess_notebook(original_path) - passed = run_notebook_test(tmp_path, original_path, all_warnings) - results[original_path] = 'passed' if passed else 'failed' + for original_path, copied_path in notebooks: + + + with open(LOG_NAME, "a") as log_file: + log_file.write(f"\n=== Processing notebook: {original_path} ===\n") + tmp_path = preprocess_notebook(original_path, copied_path) + passed = run_notebook_test(tmp_path, original_path, all_warnings) + results[original_path] = 'passed' if passed else 'failed' with open(LOG_NAME, "a") as log_file: log_file.write("\n======== Notebook Test Summary =========\n") From 373a112c107588e045d06e4365c835a375e277f1 Mon Sep 17 00:00:00 2001 From: lauramurgatroyd Date: Fri, 8 May 2026 16:15:43 +0000 Subject: [PATCH 2/5] spacing --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5930fc2..f43980b 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ To use it: ```sh jupyter kernelspec list ``` -If there is a kernel with name `cil_test_demos` then move to step 9. If not, run: + If there is a kernel with name `cil_test_demos` then move to step 9. If not, run: ```sh python -m ipykernel install --user --name cil_test_demos --display-name "cil_test_demos" ``` From 4436ea6954c1c8eb71c09d2355d1a31e1fb943a1 Mon Sep 17 00:00:00 2001 From: lauramurgatroyd Date: Fri, 8 May 2026 16:16:26 +0000 Subject: [PATCH 3/5] spacing --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index f43980b..67f88f3 100644 --- a/README.md +++ b/README.md @@ -50,10 +50,13 @@ To use it: `CIL_DEMOS_DIR` - should point to your clone of CIL-Demos `DATA_PATH` and `DATA_PATH_ALT` should point to directories where the CIL-Demos data is saved. 10. Run `test_notebooks_sequential.py`. This is likely to take quite a long time. It will create the following inside the `scripts` directory: + `test_notebooks_*.log` log file (with name including date and time of run) This shows which notebooks have run and whether they have passed or failed. + `tmp_notebooks` directory This contains the run and rendered `.ipynb` notebook files. + `html_outputs` directory This contains run and rendered notebooks as `html` files. From e179d60208509e161517b13a1388af2f9dbfd493 Mon Sep 17 00:00:00 2001 From: lauramurgatroyd Date: Fri, 15 May 2026 17:15:11 +0000 Subject: [PATCH 4/5] docs --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 67f88f3..6bbca85 100644 --- a/README.md +++ b/README.md @@ -13,10 +13,12 @@ Static hosting: ### CIL `cil.yml` - for installing the latest release of CIL. -`cil_test.yml` - for installing the latest +`cil_test.yml` - for installing the latest release of CIL with the requirements for running the tests. ### CILViewer -`cilviewer_ui` - for installing the latest release of CILViewer, with +`cilviewer_ui` - for installing the latest release of CILViewer, with packages needed to run the GUI. + + ## Script for Testing CIL demos `test_notebooks_sequential.py` is a script for testing the CIL-Demos From 87e9bf16664e4e994d6e805c560476b747594f4c Mon Sep 17 00:00:00 2001 From: Laura Murgatroyd <60604372+lauramurgatroyd@users.noreply.github.com> Date: Fri, 15 May 2026 18:25:05 +0100 Subject: [PATCH 5/5] Update test_notebooks_sequential.py --- test_notebooks_sequential.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_notebooks_sequential.py b/test_notebooks_sequential.py index df8633f..1ebbecb 100644 --- a/test_notebooks_sequential.py +++ b/test_notebooks_sequential.py @@ -10,7 +10,7 @@ DATA_PATH_ALT = '/mnt/share/materials/CIL/' SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) -CIL_DEMOS_DIR = os.path.expanduser("/home/lhe97136/CIL-Demos") +CIL_DEMOS_DIR = os.path.expanduser(os.path.join('~', 'CIL-Demos')) PDF_OUTPUT_DIR = os.path.join(SCRIPT_DIR, 'html_outputs') TMP_OUTPUT_DIR = os.path.join(SCRIPT_DIR, 'tmp_notebooks')