From 9ac6e2b70658edeabae3b50418b7a4e808448047 Mon Sep 17 00:00:00 2001 From: Joe Zuntz Date: Mon, 27 Jul 2026 09:10:36 +0100 Subject: [PATCH 1/3] Allow jinja templates in config as well as pipeline files --- ceci/pipeline/mini.py | 2 +- ceci/pipeline/pipeline.py | 3 +-- ceci/stage.py | 5 +++-- ceci/tools/ancestors.py | 4 ++-- tests/config.yml | 11 ++++++++++- tests/test_executables.py | 3 +-- tests/test_flow_chart.py | 3 ++- tests/test_pipeline.py | 3 ++- 8 files changed, 22 insertions(+), 12 deletions(-) diff --git a/ceci/pipeline/mini.py b/ceci/pipeline/mini.py index 5747370..bdba2d8 100644 --- a/ceci/pipeline/mini.py +++ b/ceci/pipeline/mini.py @@ -113,7 +113,7 @@ def run_jobs(self): runner = minirunner.Runner( nodes, graph, log_dir, callback=self.callback, sleep=self.sleep ) - interval = self.launcher_config.get("interval", 3) + interval = self.launcher_config.get("interval", 1) try: runner.run(interval) except minirunner.FailedJob as error: diff --git a/ceci/pipeline/pipeline.py b/ceci/pipeline/pipeline.py index b3a66cc..548626c 100644 --- a/ceci/pipeline/pipeline.py +++ b/ceci/pipeline/pipeline.py @@ -657,8 +657,7 @@ def read_stages_config(self, stages_config): self.stages_config = stages_config if isinstance(self.stages_config, str): - with open(self.stages_config) as stage_config_file: - self.stage_config_data = yaml.safe_load(stage_config_file) + self.stage_config_data = yaml.safe_load(read_and_apply_template(self.stages_config)) if self.stage_config_data is None: self.stage_config_data = {} elif isinstance(self.stages_config, dict): diff --git a/ceci/stage.py b/ceci/stage.py index 890b566..3de3c1f 100644 --- a/ceci/stage.py +++ b/ceci/stage.py @@ -1301,6 +1301,7 @@ def read_config(self, args): """ # Try to load configuration file if provided import yaml + from .pipeline.templates import read_and_apply_template config_file = self.get_input("config") @@ -1309,8 +1310,8 @@ def read_config(self, args): if isinstance(config_file, dict): overall_config = config_file elif config_file is not None: - with open(config_file) as _config_file: - overall_config = yaml.safe_load(_config_file) + overall_config = read_and_apply_template(config_file) + overall_config = yaml.safe_load(overall_config) else: overall_config = {} diff --git a/ceci/tools/ancestors.py b/ceci/tools/ancestors.py index aac10f6..2f58cfc 100644 --- a/ceci/tools/ancestors.py +++ b/ceci/tools/ancestors.py @@ -1,12 +1,12 @@ import ceci +from ceci.pipeline.templates import read_and_apply_template import yaml import argparse import networkx def print_ancestors(pipeline_config_file, target): - with open(pipeline_config_file) as f: - pipe_config = yaml.safe_load(f) + pipe_config = yaml.safe_load(read_and_apply_template(pipeline_config_file)) # need to manually switch off resume mode because it # would stop jobs from being properly in the DAG. diff --git a/tests/config.yml b/tests/config.yml index b91ced0..280ab68 100644 --- a/tests/config.yml +++ b/tests/config.yml @@ -1,11 +1,20 @@ global: metacalibration: True +{% set apply_flag = false %} + shearMeasurementPipe: - apply_flag: False + apply_flag: {{ apply_flag }} WLGCSelector: zbin_edges: [0.2, 0.3, 0.5] ra_range: [-5, 5] +{% if false %} +this badly-formed part should not matter +because jinja2 templates should be working +{% endif %} + + + nonsenseStageToTestNonesAreOkay: diff --git a/tests/test_executables.py b/tests/test_executables.py index e6edf12..9eb56cd 100644 --- a/tests/test_executables.py +++ b/tests/test_executables.py @@ -23,8 +23,7 @@ def test_misuse_mpi(): subprocess.check_call(cmd.split()) def test_run_template_parameters(): - if os.path.exists("tests/template_test_logfile.txt"): - os.remove("tests/template_test_logfile.txt") + shutil.rmtree("tests/template_test_logfile.txt") shutil.rmtree("tests/outputs_north", ignore_errors=True) cmd = 'ceci tests/template.yml --template-parameters logfile=template_test_logfile.txt some_directory=inputs field=north' diff --git a/tests/test_flow_chart.py b/tests/test_flow_chart.py index 8c026e8..b84bb7e 100644 --- a/tests/test_flow_chart.py +++ b/tests/test_flow_chart.py @@ -1,9 +1,10 @@ import yaml from ceci import FlowChartPipeline +from ceci.pipeline.templates import read_and_apply_template import os def test_flow_chart(): - config = yaml.safe_load(open("tests/test.yml")) + config = yaml.safe_load(read_and_apply_template("tests/test.yml")) launcher_config = {"interval": 0.5, "name": "mini"} pipeline = FlowChartPipeline(config["stages"], launcher_config) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 6bd7890..782f214 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -1,4 +1,5 @@ from ceci import PipelineStage, MiniPipeline, ParslPipeline, Pipeline, DryRunPipeline +from ceci.pipeline.templates import read_and_apply_template from ceci_example.types import TextFile from ceci.sites import load, reset_default_site from ceci.utils import extra_paths @@ -191,7 +192,7 @@ def test_dry_run(mocker): stdout_mock = mocker.patch("ceci.pipeline.pipeline.sys.stdout") stdout_mock.isatty.return_value = True - config = yaml.safe_load(open("tests/test.yml")) + config = yaml.safe_load(read_and_apply_template("tests/test.yml")) launcher_config = {"interval": 0.5, "name": "mini"} pipeline = DryRunPipeline(config["stages"], launcher_config) From fc8b72069f5d91aef223bc92104cba9929ff93a8 Mon Sep 17 00:00:00 2001 From: Joe Zuntz Date: Mon, 27 Jul 2026 09:53:41 +0100 Subject: [PATCH 2/3] Undo default change --- ceci/pipeline/mini.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ceci/pipeline/mini.py b/ceci/pipeline/mini.py index bdba2d8..5747370 100644 --- a/ceci/pipeline/mini.py +++ b/ceci/pipeline/mini.py @@ -113,7 +113,7 @@ def run_jobs(self): runner = minirunner.Runner( nodes, graph, log_dir, callback=self.callback, sleep=self.sleep ) - interval = self.launcher_config.get("interval", 1) + interval = self.launcher_config.get("interval", 3) try: runner.run(interval) except minirunner.FailedJob as error: From 8d6c8f0c26e74fac0cef5faca8d763e0bf646176 Mon Sep 17 00:00:00 2001 From: Joe Zuntz Date: Mon, 27 Jul 2026 13:38:24 +0100 Subject: [PATCH 3/3] Fix rmtree error --- tests/test_executables.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_executables.py b/tests/test_executables.py index 9eb56cd..9327ea3 100644 --- a/tests/test_executables.py +++ b/tests/test_executables.py @@ -23,7 +23,7 @@ def test_misuse_mpi(): subprocess.check_call(cmd.split()) def test_run_template_parameters(): - shutil.rmtree("tests/template_test_logfile.txt") + shutil.rmtree("tests/template_test_logfile.txt", ignore_errors=True) shutil.rmtree("tests/outputs_north", ignore_errors=True) cmd = 'ceci tests/template.yml --template-parameters logfile=template_test_logfile.txt some_directory=inputs field=north'