diff --git a/README.md b/README.md index d4dce1a..7d34b7e 100644 --- a/README.md +++ b/README.md @@ -148,6 +148,32 @@ Synthetic data is only schema-valid against the dictionary that generated it, so refuses a batch that doesn't match the version being uploaded (override with `--allow-version-mismatch`). +### Synthetic data: LLM configuration + +`synth generate --llm` and `synth deploy` generate LLM-realistic values with +[gen3-metadata-simulator](https://github.com/AustralianBioCommons/gen3-metadata-simulator). +The provider and model resolve with precedence **CLI flags > SSM > default**: +the CDK config's optional `llm` block publishes `app/llm_provider` / +`app/llm_model` to SSM, so every operator gets the deployment's values, and +`--llm-provider` / `--llm-model` override them for one run (e.g. to try a +model before adding it to the CDK config). Environments deployed without the +block fall back to provider `anthropic`, and the `--llm` path errors with +guidance when no model is configured anywhere. + +Only the API key stays local — as a *path* to the file holding it, never the +key itself, set once per operator: + +```bash +g3dt config set llm_api_key_file ~/.g3dt/anthropic_api_key +g3dt synth generate AusDiab_Simulated --llm -n 5 -e test +``` + +(or per run with `--llm-api-key-file`; the vendor env var `ANTHROPIC_API_KEY` +/ `OPENAI_API_KEY` also works as a fallback.) The old `~/.g3dt/.env` +(`LLM_PROVIDER`/`LLM_MODEL`/`LLM_API_KEY_FILE`) is **no longer read**. +`g3dt config show --env ` prints the resolved provider, model, and key +path. + ## Verifying download access (check-download) Registration alone does not prove a file can be downloaded. Two failure modes diff --git a/pyproject.toml b/pyproject.toml index 6281333..a3af157 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "gen3-dataops-toolkit" -version = "3.3.0" +version = "3.4.0" description = "Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments" authors = ["JoshuaHarris391 "] readme = "README.md" diff --git a/src/g3dt/cli/config_cmds.py b/src/g3dt/cli/config_cmds.py index 6bfa703..35f5444 100644 --- a/src/g3dt/cli/config_cmds.py +++ b/src/g3dt/cli/config_cmds.py @@ -118,6 +118,11 @@ def show( typer.echo(f" namespace : {e.namespace}") typer.echo(f" cluster_name : {e.cluster_name}") typer.echo(f" ec2_instance_id : {e.ec2_instance_id}") + # Synthetic-data LLM facts: provider/model from SSM (the CDK's optional + # llm block); only the key *path* is local, from the marker. + typer.echo(f" llm_provider : {e.llm_provider}") + typer.echo(f" llm_model : {e.llm_model or '(not set — pass --llm-model or add the llm block to the CDK config)'}") + typer.echo(f" llm_api_key_file : {config.llm_api_key_file() or '(not set — g3dt config set llm_api_key_file )'}") if study: s = study_of(study, env) typer.secho(f"Study: {study} -> {s.key}", bold=True) @@ -199,6 +204,20 @@ def check(label: str, file_value, ssm_value) -> None: check(f"gen3.{camel}", gen3.get(camel), rc.get(f"app/{leaf}")) check("toolkitVersion", inputs.get("toolkitVersion"), rc.get("meta/toolkitVersion")) + # Optional inputs: compared only when the file defines them — an absent + # input legitimately publishes no SSM parameter (or, for the dictionary + # fields, predates their addition), so absence on both sides is not drift. + for camel, leaf in { + "dictionaryBaseUrl": "dictionary_base_url", + "dictionaryPath": "dictionary_path", + }.items(): + if camel in gen3: + check(f"gen3.{camel}", gen3.get(camel), rc.get(f"app/{leaf}")) + llm = inputs.get("llm") or {} + for camel, leaf in {"provider": "llm_provider", "model": "llm_model"}.items(): + if camel in llm: + check(f"llm.{camel}", llm.get(camel), rc.get(f"app/{leaf}")) + if not drift: typer.secho( f"No drift: SSM /{project}/{config.env_base(env)} matches {file}.", diff --git a/src/g3dt/cli/synth.py b/src/g3dt/cli/synth.py index 998645a..7ebad35 100644 --- a/src/g3dt/cli/synth.py +++ b/src/g3dt/cli/synth.py @@ -9,9 +9,14 @@ whose name contains ``prod``) shows a warning and requires typing the env name to confirm — it cannot be bypassed. -Generation defaults to keyless ``random`` data (no API calls). Pass ``--llm`` for -LLM-realistic values, which needs an API key configured in a ``.env`` in the -working directory (``LLM_PROVIDER`` / ``LLM_MODEL`` / ``LLM_API_KEY_FILE``). +Generation defaults to keyless ``random`` data (no API calls). Pass ``--llm`` +for LLM-realistic values. The LLM provider and model come from the +environment's SSM tree (the CDK config's optional ``llm`` block, published as +``app/llm_provider`` / ``app/llm_model``) and can be overridden per run with +``--llm-provider`` / ``--llm-model``. Only the API key stays local: point at +its file once with ``g3dt config set llm_api_key_file `` (or per run +with ``--llm-api-key-file``); the vendor env var ``ANTHROPIC_API_KEY`` / +``OPENAI_API_KEY`` also works. The old ``~/.g3dt/.env`` is no longer read. """ from __future__ import annotations @@ -21,6 +26,7 @@ import typer +from g3dt import config from g3dt.config import ( dictionary_filename, dictionary_url, @@ -38,6 +44,45 @@ SYNTH_DIR = Path("~/.g3dt/synth_metadata").expanduser() + +def _llm_env_overrides( + e, + llm_provider: Optional[str], + llm_model: Optional[str], + llm_api_key_file: Optional[Path], +) -> dict: + """Resolve the effective LLM settings for a run: flags > SSM > default. + + Returns env-var overrides for the generator script, which forwards + provider/model to gen3-metadata-simulator as CLI flags (the simulator's + own precedence puts flags first). Exits with guidance when no model is + configured anywhere; the key-file path is optional — the simulator falls + back to the vendor env var and raises its own error if neither exists. + """ + effective_provider = llm_provider or e.llm_provider + effective_model = llm_model or e.llm_model + if not effective_model: + typer.secho( + "No LLM model configured. Set the llm block in the CDK config " + "(published to SSM as app/llm_model) and redeploy, or pass " + "--llm-model for this run.", + fg=typer.colors.RED, + err=True, + ) + raise typer.Exit(1) + overrides = { + "G3DT_LLM_PROVIDER": effective_provider, + "G3DT_LLM_MODEL": effective_model, + } + key_file = ( + str(Path(llm_api_key_file).expanduser()) + if llm_api_key_file + else config.llm_api_key_file() + ) + if key_file: + overrides["LLM_API_KEY_FILE"] = key_file + return overrides + #: Written into each generated batch so `synth upload` can tell which dictionary #: produced it. A batch is only valid against that dictionary, and the directory #: name alone cannot be trusted: --schema and --version are separate options, so @@ -110,19 +155,35 @@ def deploy( env: str = typer.Option( "test", "--env", "-e", help="Target environment (prod requires typed confirmation)." ), + llm_provider: Optional[str] = typer.Option( + None, "--llm-provider", + help="LLM vendor override (anthropic|openai); default: the env's SSM app/llm_provider.", + ), + llm_model: Optional[str] = typer.Option( + None, "--llm-model", + help="LLM model override; default: the env's SSM app/llm_model.", + ), + llm_api_key_file: Optional[Path] = typer.Option( + None, "--llm-api-key-file", exists=True, dir_okay=False, + help="Path to the file holding the LLM API key; default: the marker's " + "llm_api_key_file (set once: g3dt config set llm_api_key_file ).", + ), ) -> None: """Full end-to-end synthetic deploy (dict + LLM-generate + upload + restarts). Wraps services/synthetic_data/full_deploy_dd_and_synth.sh (LLM-backed - generation). Requires an LLM key configured in .env. + generation). Provider/model come from the env's SSM tree unless + overridden; the API key path comes from --llm-api-key-file or the marker. """ e = env_of(env) safety.confirm_prod_strict("synthetic full deploy", env) + env_vars = script_env(e) + env_vars.update(_llm_env_overrides(e, llm_provider, llm_model, llm_api_key_file)) runner.run( runner.bash_script( "services/synthetic_data/full_deploy_dd_and_synth.sh", env ), - env=script_env(e), + env=env_vars, ) @@ -147,8 +208,22 @@ def generate( llm: bool = typer.Option( False, "--llm", - help="Generate LLM-realistic values; reads LLM config from a .env in the " - "working directory. Default is keyless random data (no API key, no API calls).", + help="Generate LLM-realistic values; provider/model resolve from the " + "env's SSM tree (override with --llm-provider/--llm-model). Default is " + "keyless random data (no API key, no API calls).", + ), + llm_provider: Optional[str] = typer.Option( + None, "--llm-provider", + help="LLM vendor override (anthropic|openai); default: the env's SSM app/llm_provider.", + ), + llm_model: Optional[str] = typer.Option( + None, "--llm-model", + help="LLM model override; default: the env's SSM app/llm_model.", + ), + llm_api_key_file: Optional[Path] = typer.Option( + None, "--llm-api-key-file", exists=True, dir_okay=False, + help="Path to the file holding the LLM API key; default: the marker's " + "llm_api_key_file (set once: g3dt config set llm_api_key_file ).", ), seed: int = typer.Option(None, "--seed", help="RNG seed for reproducible output."), schema: str = typer.Option( @@ -227,11 +302,16 @@ def generate( args += ["--num-records", num_records] if seed is not None: args += ["--seed", str(seed)] + env_vars = script_env(e, ver) + if effective_provider == "llm": + env_vars.update( + _llm_env_overrides(e, llm_provider, llm_model, llm_api_key_file) + ) runner.run( runner.bash_script( "services/synthetic_data/generate_synth_metadata.sh", *args ), - env=script_env(e, ver), + env=env_vars, ) # Only after a successful generate: runner.run raises on failure, so a # half-written batch is never stamped as valid. diff --git a/src/g3dt/config.py b/src/g3dt/config.py index 15d7c77..82e47d5 100644 --- a/src/g3dt/config.py +++ b/src/g3dt/config.py @@ -50,11 +50,21 @@ DEFAULT_DICT_BASE_URL = "https://raw.githubusercontent.com" DEFAULT_DICT_PATH = "dictionary/prod_dict/acdc_schema.json" +#: Synthetic-data LLM facts, published by the CDK's OPTIONAL ``llm`` config +#: block as ``app/llm_provider`` / ``app/llm_model`` and consumed by +#: gen3-metadata-simulator through ``g3dt synth``. Optional app inputs: +#: environments deployed without the block fall back to this provider default. +#: The model deliberately has no default — ``g3dt synth`` errors with guidance +#: when the ``--llm`` path is used and no model is configured anywhere. +DEFAULT_LLM_PROVIDER = "anthropic" + #: Marker locations, most specific first. MARKER_PATHS = ("g3dt.yaml", "~/.g3dt/g3dt.yaml", "/etc/g3dt/g3dt.yaml") -#: Marker keys `g3dt config set` may write. -SETTABLE_MARKER_KEYS = ("project", "region", "default_env") +#: Marker keys `g3dt config set` may write. ``llm_api_key_file`` is the path +#: to the file holding the synth LLM API key — the one LLM setting that stays +#: local (the provider and model come from SSM; the key never leaves the box). +SETTABLE_MARKER_KEYS = ("project", "region", "default_env", "llm_api_key_file") #: Gen3 app facts mirrored to SSM /{project}/{env}/app/* by the CDK. REQUIRED_APP_KEYS = ( @@ -150,6 +160,18 @@ def require_project(marker: Optional[dict] = None) -> str: return project +def llm_api_key_file(marker: Optional[dict] = None) -> Optional[str]: + """Path to the file holding the synth LLM API key, from the marker. + + Set once per operator with ``g3dt config set llm_api_key_file ``. + Returns ``None`` when unset — gen3-metadata-simulator then falls back to + the vendor's standard env var (``ANTHROPIC_API_KEY`` / ``OPENAI_API_KEY``). + """ + m = marker if marker is not None else load_marker() + value = m.get("llm_api_key_file") + return str(Path(str(value)).expanduser()) if value else None + + def set_marker_value(key: str, value: str) -> Tuple[Optional[str], str, Path]: """Set one bootstrap key in the user's marker file and write it back. @@ -230,6 +252,11 @@ class EnvConfig: # hand-built EnvConfig still composes a valid URL. dictionary_base_url: str = DEFAULT_DICT_BASE_URL dictionary_path: str = DEFAULT_DICT_PATH + # Optional synthetic-data LLM inputs (SSM app/llm_provider, app/llm_model). + # The model has no default on purpose: synth's --llm path checks and errors + # with guidance rather than silently picking a model. + llm_provider: str = DEFAULT_LLM_PROVIDER + llm_model: Optional[str] = None def _app_or_default(rc, leaf: str, default: str) -> str: @@ -289,6 +316,10 @@ def resolve_env(env: str, project: Optional[str] = None) -> EnvConfig: rc, "dictionary_base_url", DEFAULT_DICT_BASE_URL ), dictionary_path=_app_or_default(rc, "dictionary_path", DEFAULT_DICT_PATH), + # Same optional-app-fact contract: the CDK publishes these only when + # the config has an llm block, so absence means "use the defaults". + llm_provider=_app_or_default(rc, "llm_provider", DEFAULT_LLM_PROVIDER), + llm_model=(_app_or_default(rc, "llm_model", "") or None), ) @@ -395,6 +426,8 @@ def script_env(e: EnvConfig, version: Optional[str] = None) -> Dict[str, str]: "G3DT_NAMESPACE": e.namespace, "G3DT_CLUSTER_NAME": e.cluster_name, "G3DT_SCHEMA_REPO": e.schema_repo, + "G3DT_LLM_PROVIDER": e.llm_provider, + "G3DT_LLM_MODEL": e.llm_model, } env.update({k: v for k, v in values.items() if v is not None}) return env diff --git a/src/g3dt/services/synthetic_data/generate_synth_metadata.sh b/src/g3dt/services/synthetic_data/generate_synth_metadata.sh index 14edf77..f3effc9 100644 --- a/src/g3dt/services/synthetic_data/generate_synth_metadata.sh +++ b/src/g3dt/services/synthetic_data/generate_synth_metadata.sh @@ -8,14 +8,13 @@ # The tool takes a LOCAL bundled Gen3 schema file (pulled by pull_dict.sh into # ~/.g3dt/schemas/acdc_schema_.json, or $G3DT_SCHEMA_DIR if set). The # default provider is keyless 'random'; pass --provider llm for LLM-realistic -# values, in which case LLM config is read from ~/.g3dt/.env (or $G3DT_ENV_FILE -# if set): LLM_PROVIDER / LLM_MODEL / LLM_API_KEY_FILE. +# values. The LLM vendor/model arrive as $G3DT_LLM_PROVIDER / $G3DT_LLM_MODEL +# (resolved by g3dt: CLI flags > SSM app/llm_* > default) and are forwarded to +# the simulator as flags; the API key file path arrives as $LLM_API_KEY_FILE. +# The old ~/.g3dt/.env is no longer read. set -euo pipefail -# LLM provider config file (lives outside the installed package). -ENV_FILE="${G3DT_ENV_FILE:-$HOME/.g3dt/.env}" - usage() { cat < --version [options] @@ -32,8 +31,8 @@ Options: --num-records N|n1,n2 Records per study: one number for all, or a comma list (one per study). Default: ${DEFAULT_NUM_RECORDS} --provider random|llm Value strategy. Default: ${DEFAULT_PROVIDER} - 'random' needs no key; 'llm' reads LLM config from - ${ENV_FILE}. + 'random' needs no key; 'llm' uses \$G3DT_LLM_PROVIDER / + \$G3DT_LLM_MODEL / \$LLM_API_KEY_FILE (set by g3dt). --seed N RNG seed for reproducible output. --output-root DIR Root output dir. Default: ${DEFAULT_OUTPUT_ROOT} -h, --help Show this help and exit. @@ -123,9 +122,16 @@ for i in "${!STUDY_ARRAY[@]}"; do --num-records "$N" --provider "$PROVIDER") [[ -n "$SEED" ]] && CMD+=(--seed "$SEED") - # Point the LLM provider at the user-level env file regardless of the caller's CWD. - if [[ "$PROVIDER" == "llm" && -f "${ENV_FILE}" ]]; then - CMD+=(--env-file "${ENV_FILE}") + if [[ "$PROVIDER" == "llm" ]]; then + # Vendor/model resolved by g3dt (flags > SSM > default) and forwarded + # as simulator flags, because the simulator's own precedence puts + # flags above any .env or environment variable. + [[ -n "${G3DT_LLM_PROVIDER:-}" ]] && CMD+=(--llm-provider "$G3DT_LLM_PROVIDER") + [[ -n "${G3DT_LLM_MODEL:-}" ]] && CMD+=(--llm-model "$G3DT_LLM_MODEL") + # Neutralize any .env in the caller's CWD: /dev/null exists (satisfies + # the simulator's exists=True check) and dotenv-parses to empty, so + # only the flags above and the inherited $LLM_API_KEY_FILE apply. + CMD+=(--env-file /dev/null) fi "${CMD[@]}" done diff --git a/tests/test_cli_config.py b/tests/test_cli_config.py index cd80238..1b1a38c 100644 --- a/tests/test_cli_config.py +++ b/tests/test_cli_config.py @@ -282,3 +282,77 @@ def test_dictionary_source_treats_empty_parameter_as_unset(): e = config.resolve_env("test") assert config.dictionary_url(e).startswith("https://raw.githubusercontent.com/") + + +@mock_aws +def test_llm_facts_resolved_from_optional_app_inputs(): + """ + Inputs: an env tree with app/llm_provider and app/llm_model set (what the + CDK's optional llm config block publishes) + Expected: EnvConfig carries both, so `g3dt synth --llm` uses the + deployment's values and operators share one model with no local + configuration beyond the API key path. + """ + _seed_env("etl", "test") + ssm = boto3.client("ssm", region_name=REGION) + ssm.put_parameter(Name="/etl/test/app/llm_provider", Value="openai", Type="String") + ssm.put_parameter(Name="/etl/test/app/llm_model", Value="some-model", Type="String") + + e = config.resolve_env("test") + assert e.llm_provider == "openai" + assert e.llm_model == "some-model" + + +@mock_aws +def test_llm_facts_default_when_absent(): + """ + Inputs: an env tree WITHOUT app/llm_* (a deployment whose config has no + llm block — including every environment deployed before v3.1.0) + Expected: provider falls back to DEFAULT_LLM_PROVIDER and the model is + None. Crucially these keys are NOT in REQUIRED_APP_KEYS, whose + gate raises a "re-run cdk deploy" error — requiring them would + break every existing environment. The missing model only becomes + an error inside `synth --llm`, with guidance, when it is needed. + """ + _seed_env("etl", "test") + + e = config.resolve_env("test") + assert e.llm_provider == config.DEFAULT_LLM_PROVIDER + assert e.llm_model is None + + +@mock_aws +def test_llm_model_blank_parameter_treated_as_unset(): + """ + Inputs: app/llm_model published as whitespace (SSM rejects truly empty + values, so a "blanked out" parameter arrives as spaces) + Expected: the model resolves to None, not to a whitespace model id that + would be passed to the simulator as a real flag value. + """ + _seed_env("etl", "test") + boto3.client("ssm", region_name=REGION).put_parameter( + Name="/etl/test/app/llm_model", Value=" ", Type="String" + ) + + e = config.resolve_env("test") + assert e.llm_model is None + + +def test_script_env_exports_llm_facts_only_when_model_set(): + """ + Inputs: two hand-built EnvConfigs — one with a model, one without + Expected: G3DT_LLM_PROVIDER is always exported (it has a real default); + G3DT_LLM_MODEL is exported only when a model exists, so shell + scripts can distinguish "no model configured" from empty string. + """ + base = dict( + name="test", is_ec2=False, region=REGION, dictionary_version="v1", + aws_profile=None, aws_secret_name="s", schema_s3_uri="u", domain="d", + app_name="a", namespace="n", cluster_name="c", schema_repo="Org/r", + ) + with_model = config.script_env(config.EnvConfig(**base, llm_model="m1")) + assert with_model["G3DT_LLM_PROVIDER"] == config.DEFAULT_LLM_PROVIDER + assert with_model["G3DT_LLM_MODEL"] == "m1" + + without_model = config.script_env(config.EnvConfig(**base)) + assert "G3DT_LLM_MODEL" not in without_model diff --git a/tests/test_cli_safety.py b/tests/test_cli_safety.py index 2563493..6f479b8 100644 --- a/tests/test_cli_safety.py +++ b/tests/test_cli_safety.py @@ -31,6 +31,9 @@ def _env_cfg(name: str) -> EnvConfig: namespace="n", cluster_name="c", schema_repo="Org/schema-repo", + # synth deploy is always LLM-backed and requires a model; model an env + # deployed with the CDK's llm block so the safety paths stay testable. + llm_model="ssm-model", ) diff --git a/tests/test_cli_synth.py b/tests/test_cli_synth.py index 44c6ec5..bae0b0d 100644 --- a/tests/test_cli_synth.py +++ b/tests/test_cli_synth.py @@ -22,8 +22,20 @@ runner = CliRunner() -def _env_cfg(name: str) -> EnvConfig: - """A fully-populated EnvConfig as resolve_env would return it.""" +@pytest.fixture(autouse=True) +def _isolated_marker(tmp_path, monkeypatch): + """Keep tests hermetic: the LLM key-file fallback reads the marker, and a + developer's real ~/.g3dt/g3dt.yaml (with llm_api_key_file set) would + otherwise leak into the asserted subprocess env.""" + monkeypatch.setenv("G3DT_MARKER", str(tmp_path / "no-marker.yaml")) + + +def _env_cfg(name: str, llm_model: str = "ssm-model") -> EnvConfig: + """A fully-populated EnvConfig as resolve_env would return it. + + Carries an llm_model by default (as an env deployed with the CDK's llm + block would); pass llm_model=None to model a deployment without the block. + """ return EnvConfig( name=name, is_ec2=name.endswith("_ec2"), @@ -37,6 +49,7 @@ def _env_cfg(name: str) -> EnvConfig: namespace="n", cluster_name="c", schema_repo="Org/schema-repo", + llm_model=llm_model, ) @@ -86,12 +99,82 @@ def test_generate_passes_study_and_defaults_to_random(mock_run, _env, schema_dir def test_generate_llm_flag_enables_llm(mock_run, _env, schema_dir): """ Inputs: g3dt synth generate AusDiab_Simulated --llm - Expected Output: --provider llm is passed (opt-in; reads LLM config from .env). + Expected Output: --provider llm is passed, and the env's SSM-resolved + model reaches the script as G3DT_LLM_MODEL (the script forwards it to the + simulator as a flag) — no local .env involved. """ result = runner.invoke(app, ["synth", "generate", "AusDiab_Simulated", "--llm"]) assert result.exit_code == 0, result.output argv = _gen_argv(mock_run) assert argv[argv.index("--provider") + 1] == "llm" + env = _gen_env(mock_run) + assert env["G3DT_LLM_MODEL"] == "ssm-model" + assert env["G3DT_LLM_PROVIDER"] == "anthropic" + + +def _gen_env(mock_run): + """Return the env dict of the generate_synth_metadata.sh invocation.""" + for call in mock_run.call_args_list: + argv = list(call.args[0]) + if any(str(a).endswith("generate_synth_metadata.sh") for a in argv): + return call.kwargs["env"] + raise AssertionError(f"generate script not invoked: {mock_run.call_args_list}") + + +@patch("g3dt.cli.synth.env_of", side_effect=_env_cfg) +@patch("g3dt.cli._internal.runner.run") +def test_generate_llm_flags_override_ssm(mock_run, _env, schema_dir, tmp_path): + """ + Inputs: --llm --llm-provider openai --llm-model my-model + --llm-api-key-file + Expected Output: the flag values win over the EnvConfig's SSM-resolved + ones and the key path is exported as LLM_API_KEY_FILE. This is the + "try a model with one command, no redeploy" path. + """ + key_file = tmp_path / "key" + key_file.write_text("sk-test") + result = runner.invoke( + app, + ["synth", "generate", "AusDiab_Simulated", "--llm", + "--llm-provider", "openai", "--llm-model", "my-model", + "--llm-api-key-file", str(key_file)], + ) + assert result.exit_code == 0, result.output + env = _gen_env(mock_run) + assert env["G3DT_LLM_PROVIDER"] == "openai" + assert env["G3DT_LLM_MODEL"] == "my-model" + assert env["LLM_API_KEY_FILE"] == str(key_file) + + +@patch("g3dt.cli.synth.env_of", side_effect=lambda name: _env_cfg(name, llm_model=None)) +@patch("g3dt.cli._internal.runner.run") +def test_generate_llm_without_model_exits_with_guidance(mock_run, _env, schema_dir): + """ + Inputs: --llm against an env whose deployment has no llm block (no SSM + model) and with no --llm-model flag + Expected Output: exit 1 BEFORE the script runs, with a message pointing at + both fixes (add the llm block to the CDK config, or pass --llm-model) — + instead of the simulator failing later with its own .env-era error. + """ + result = runner.invoke(app, ["synth", "generate", "AusDiab_Simulated", "--llm"]) + assert result.exit_code == 1 + assert "No LLM model configured" in result.output + assert "--llm-model" in result.output + assert not mock_run.called + + +@patch("g3dt.cli.synth.env_of", side_effect=_env_cfg) +@patch("g3dt.cli._internal.runner.run") +def test_generate_random_provider_skips_llm_plumbing(mock_run, _env, schema_dir): + """ + Inputs: a plain random-provider generate (the default) + Expected Output: no LLM_API_KEY_FILE is injected — the keyless path stays + keyless, and a missing model in the deployment can never affect it. + """ + result = runner.invoke(app, ["synth", "generate", "AusDiab_Simulated"]) + assert result.exit_code == 0, result.output + env = _gen_env(mock_run) + assert "LLM_API_KEY_FILE" not in env @patch("g3dt.cli.synth.env_of", side_effect=_env_cfg) diff --git a/tests/test_generate_synth_sh.py b/tests/test_generate_synth_sh.py new file mode 100644 index 0000000..fe2f9d5 --- /dev/null +++ b/tests/test_generate_synth_sh.py @@ -0,0 +1,109 @@ +"""Flag-forwarding tests for the synthetic-data generator shell script. + +``services/synthetic_data/generate_synth_metadata.sh`` is the layer between +`g3dt synth` and gen3-metadata-simulator. Since v3.4.0 the LLM vendor and +model arrive as ``$G3DT_LLM_PROVIDER`` / ``$G3DT_LLM_MODEL`` (resolved by g3dt +with precedence flags > SSM > default) and must be forwarded to the simulator +as CLI flags — the simulator's own precedence puts flags above any ``.env``, +which is exactly what makes the deployment's values authoritative. The script +must also pass ``--env-file /dev/null`` so a stray ``.env`` in the caller's +working directory can never hijack resolution, and must no longer read the +retired ``~/.g3dt/.env``. + +None of that branching is reachable from the Python tests (the CLI stops at +building argv), so these run the real script with a stubbed +``gen3-metadata-simulator`` on PATH that records the command line it was +handed. +""" +import os +import subprocess +from pathlib import Path + +import pytest + +SCRIPT = ( + Path(__file__).resolve().parent.parent + / "src" / "g3dt" / "services" / "synthetic_data" / "generate_synth_metadata.sh" +) + + +@pytest.fixture +def stub_simulator(tmp_path): + """Put a fake ``gen3-metadata-simulator`` on PATH that logs its arguments.""" + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + record = tmp_path / "record.txt" + stub = bin_dir / "gen3-metadata-simulator" + stub.write_text('#!/usr/bin/env bash\necho "$*" >> "$STUB_RECORD"\nexit 0\n') + stub.chmod(0o755) + return bin_dir, record + + +def _run(stub_simulator, tmp_path, extra_args=(), extra_env=None): + bin_dir, record = stub_simulator + schema = tmp_path / "schema.json" + schema.write_text("{}") + env = dict( + os.environ, + PATH=f"{bin_dir}:{os.environ['PATH']}", + STUB_RECORD=str(record), + G3DT_SYNTH_DIR=str(tmp_path / "out"), + ) + if extra_env: + env.update(extra_env) + result = subprocess.run( + ["bash", str(SCRIPT), "--schema", str(schema), "--version", "v1", + "--studies", "Study_A", *extra_args], + env=env, capture_output=True, text=True, + ) + assert result.returncode == 0, result.stderr + return record.read_text() + + +def test_llm_provider_and_model_forwarded_as_flags(stub_simulator, tmp_path): + """ + Inputs: --provider llm with G3DT_LLM_PROVIDER/G3DT_LLM_MODEL in the env + (what g3dt exports after resolving flags > SSM > default) + Expected: the simulator receives them as --llm-provider/--llm-model flags + plus --env-file /dev/null, so the resolved values are + authoritative and no filesystem .env can override them. + """ + recorded = _run( + stub_simulator, tmp_path, + extra_args=("--provider", "llm"), + extra_env={"G3DT_LLM_PROVIDER": "anthropic", "G3DT_LLM_MODEL": "some-model"}, + ) + assert "--llm-provider anthropic" in recorded + assert "--llm-model some-model" in recorded + assert "--env-file /dev/null" in recorded + + +def test_llm_without_env_vars_still_neutralizes_cwd_env(stub_simulator, tmp_path): + """ + Inputs: --provider llm with NO G3DT_LLM_* env vars (a caller invoking the + script directly rather than through g3dt) + Expected: no --llm-provider/--llm-model flags (the simulator's own + defaults apply), but --env-file /dev/null is still passed — the + retired ~/.g3dt/.env must not resurface through the simulator's + CWD default. + """ + recorded = _run(stub_simulator, tmp_path, extra_args=("--provider", "llm")) + assert "--llm-provider" not in recorded + assert "--llm-model" not in recorded + assert "--env-file /dev/null" in recorded + + +def test_random_provider_passes_no_llm_flags(stub_simulator, tmp_path): + """ + Inputs: the default keyless random provider (G3DT_LLM_* vars present, as + script_env always exports the provider) + Expected: none of the LLM flags are passed — the random path stays exactly + as before, unaffected by any LLM configuration. + """ + recorded = _run( + stub_simulator, tmp_path, + extra_env={"G3DT_LLM_PROVIDER": "anthropic", "G3DT_LLM_MODEL": "some-model"}, + ) + assert "--provider random" in recorded + assert "--llm-provider" not in recorded + assert "--env-file" not in recorded