diff --git a/ceci/pipeline/pipeline.py b/ceci/pipeline/pipeline.py index b3a66cc..548626c 100644 --- a/ceci/pipeline/pipeline.py +++ b/ceci/pipeline/pipeline.py @@ -657,8 +657,7 @@ def read_stages_config(self, stages_config): self.stages_config = stages_config if isinstance(self.stages_config, str): - with open(self.stages_config) as stage_config_file: - self.stage_config_data = yaml.safe_load(stage_config_file) + self.stage_config_data = yaml.safe_load(read_and_apply_template(self.stages_config)) if self.stage_config_data is None: self.stage_config_data = {} elif isinstance(self.stages_config, dict): diff --git a/ceci/stage.py b/ceci/stage.py index 890b566..3de3c1f 100644 --- a/ceci/stage.py +++ b/ceci/stage.py @@ -1301,6 +1301,7 @@ def read_config(self, args): """ # Try to load configuration file if provided import yaml + from .pipeline.templates import read_and_apply_template config_file = self.get_input("config") @@ -1309,8 +1310,8 @@ def read_config(self, args): if isinstance(config_file, dict): overall_config = config_file elif config_file is not None: - with open(config_file) as _config_file: - overall_config = yaml.safe_load(_config_file) + overall_config = read_and_apply_template(config_file) + overall_config = yaml.safe_load(overall_config) else: overall_config = {} diff --git a/ceci/tools/ancestors.py b/ceci/tools/ancestors.py index aac10f6..2f58cfc 100644 --- a/ceci/tools/ancestors.py +++ b/ceci/tools/ancestors.py @@ -1,12 +1,12 @@ import ceci +from ceci.pipeline.templates import read_and_apply_template import yaml import argparse import networkx def print_ancestors(pipeline_config_file, target): - with open(pipeline_config_file) as f: - pipe_config = yaml.safe_load(f) + pipe_config = yaml.safe_load(read_and_apply_template(pipeline_config_file)) # need to manually switch off resume mode because it # would stop jobs from being properly in the DAG. diff --git a/tests/config.yml b/tests/config.yml index b91ced0..280ab68 100644 --- a/tests/config.yml +++ b/tests/config.yml @@ -1,11 +1,20 @@ global: metacalibration: True +{% set apply_flag = false %} + shearMeasurementPipe: - apply_flag: False + apply_flag: {{ apply_flag }} WLGCSelector: zbin_edges: [0.2, 0.3, 0.5] ra_range: [-5, 5] +{% if false %} +this badly-formed part should not matter +because jinja2 templates should be working +{% endif %} + + + nonsenseStageToTestNonesAreOkay: diff --git a/tests/test_executables.py b/tests/test_executables.py index e6edf12..9327ea3 100644 --- a/tests/test_executables.py +++ b/tests/test_executables.py @@ -23,8 +23,7 @@ def test_misuse_mpi(): subprocess.check_call(cmd.split()) def test_run_template_parameters(): - if os.path.exists("tests/template_test_logfile.txt"): - os.remove("tests/template_test_logfile.txt") + shutil.rmtree("tests/template_test_logfile.txt", ignore_errors=True) shutil.rmtree("tests/outputs_north", ignore_errors=True) cmd = 'ceci tests/template.yml --template-parameters logfile=template_test_logfile.txt some_directory=inputs field=north' diff --git a/tests/test_flow_chart.py b/tests/test_flow_chart.py index 8c026e8..b84bb7e 100644 --- a/tests/test_flow_chart.py +++ b/tests/test_flow_chart.py @@ -1,9 +1,10 @@ import yaml from ceci import FlowChartPipeline +from ceci.pipeline.templates import read_and_apply_template import os def test_flow_chart(): - config = yaml.safe_load(open("tests/test.yml")) + config = yaml.safe_load(read_and_apply_template("tests/test.yml")) launcher_config = {"interval": 0.5, "name": "mini"} pipeline = FlowChartPipeline(config["stages"], launcher_config) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 6bd7890..782f214 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -1,4 +1,5 @@ from ceci import PipelineStage, MiniPipeline, ParslPipeline, Pipeline, DryRunPipeline +from ceci.pipeline.templates import read_and_apply_template from ceci_example.types import TextFile from ceci.sites import load, reset_default_site from ceci.utils import extra_paths @@ -191,7 +192,7 @@ def test_dry_run(mocker): stdout_mock = mocker.patch("ceci.pipeline.pipeline.sys.stdout") stdout_mock.isatty.return_value = True - config = yaml.safe_load(open("tests/test.yml")) + config = yaml.safe_load(read_and_apply_template("tests/test.yml")) launcher_config = {"interval": 0.5, "name": "mini"} pipeline = DryRunPipeline(config["stages"], launcher_config)