Allow jinja templates in config as well as pipeline files

ceci/pipeline/pipeline.py

@@ -657,8 +657,7 @@ def read_stages_config(self, stages_config):
         self.stages_config = stages_config
 
         if isinstance(self.stages_config, str):
-            with open(self.stages_config) as stage_config_file:
-                self.stage_config_data = yaml.safe_load(stage_config_file)
+            self.stage_config_data = yaml.safe_load(read_and_apply_template(self.stages_config))
             if self.stage_config_data is None:
                 self.stage_config_data = {}
         elif isinstance(self.stages_config, dict):

ceci/stage.py

@@ -1301,6 +1301,7 @@ def read_config(self, args):
         """
         # Try to load configuration file if provided
         import yaml
+        from .pipeline.templates import read_and_apply_template
 
         config_file = self.get_input("config")
 
@@ -1309,8 +1310,8 @@ def read_config(self, args):
         if isinstance(config_file, dict):
             overall_config = config_file
         elif config_file is not None:
-            with open(config_file) as _config_file:
-                overall_config = yaml.safe_load(_config_file)
+            overall_config = read_and_apply_template(config_file)
+            overall_config = yaml.safe_load(overall_config)
         else:
             overall_config = {}
 

ceci/tools/ancestors.py

@@ -1,12 +1,12 @@
 import ceci
+from ceci.pipeline.templates import read_and_apply_template
 import yaml
 import argparse
 import networkx
 
 
 def print_ancestors(pipeline_config_file, target):
-    with open(pipeline_config_file) as f:
-        pipe_config = yaml.safe_load(f)
+    pipe_config = yaml.safe_load(read_and_apply_template(pipeline_config_file))
 
     # need to manually switch off resume mode because it
     # would stop jobs from being properly in the DAG.

tests/config.yml

@@ -1,11 +1,20 @@
 global:
     metacalibration: True
 
+{% set apply_flag = false %}
+
 shearMeasurementPipe:
-    apply_flag: False
+    apply_flag: {{ apply_flag }}
 
 WLGCSelector:
     zbin_edges: [0.2, 0.3, 0.5]
     ra_range: [-5, 5]
 
+{% if false %}
+this badly-formed part should not matter
+because jinja2 templates should be working
+{% endif %}
+
+
+
 nonsenseStageToTestNonesAreOkay:

tests/test_executables.py

@@ -23,8 +23,7 @@ def test_misuse_mpi():
         subprocess.check_call(cmd.split())
 
 def test_run_template_parameters():
-    if os.path.exists("tests/template_test_logfile.txt"):
-        os.remove("tests/template_test_logfile.txt")
+    shutil.rmtree("tests/template_test_logfile.txt", ignore_errors=True)
     shutil.rmtree("tests/outputs_north", ignore_errors=True)
 
     cmd = 'ceci tests/template.yml --template-parameters logfile=template_test_logfile.txt some_directory=inputs field=north'

tests/test_flow_chart.py

@@ -1,9 +1,10 @@
 import yaml
 from ceci import FlowChartPipeline
+from ceci.pipeline.templates import read_and_apply_template
 import os
 
 def test_flow_chart():
-    config = yaml.safe_load(open("tests/test.yml"))
+    config = yaml.safe_load(read_and_apply_template("tests/test.yml"))
     launcher_config = {"interval": 0.5, "name": "mini"}
 
     pipeline = FlowChartPipeline(config["stages"], launcher_config)

tests/test_pipeline.py

@@ -1,4 +1,5 @@
 from ceci import PipelineStage, MiniPipeline, ParslPipeline, Pipeline, DryRunPipeline
+from ceci.pipeline.templates import read_and_apply_template
 from ceci_example.types import TextFile
 from ceci.sites import load, reset_default_site
 from ceci.utils import extra_paths
@@ -191,7 +192,7 @@ def test_dry_run(mocker):
     stdout_mock = mocker.patch("ceci.pipeline.pipeline.sys.stdout")
     stdout_mock.isatty.return_value = True
 
-    config = yaml.safe_load(open("tests/test.yml"))
+    config = yaml.safe_load(read_and_apply_template("tests/test.yml"))
     launcher_config = {"interval": 0.5, "name": "mini"}
 
     pipeline = DryRunPipeline(config["stages"], launcher_config)