From 7f429618f80c4fd8803f0134257464458c14d86d Mon Sep 17 00:00:00 2001 From: Toby Jennings Date: Fri, 24 Jul 2026 10:12:44 -0500 Subject: [PATCH 1/5] feat: modularize ap daily script for cmservice - separate dynamic contents to separate script - add reference campaigns for cmservice --- scripts/LSSTCam/ap_daytime_cm_campaign.yaml | 195 +++++++++++++++++ .../LSSTCam/ap_daytime_cm_campaign_dev.yaml | 198 ++++++++++++++++++ scripts/LSSTCam/ap_daytime_env.sh | 16 ++ scripts/LSSTCam/submit_ap_daytime.sh | 15 +- 4 files changed, 410 insertions(+), 14 deletions(-) create mode 100644 scripts/LSSTCam/ap_daytime_cm_campaign.yaml create mode 100644 scripts/LSSTCam/ap_daytime_cm_campaign_dev.yaml create mode 100644 scripts/LSSTCam/ap_daytime_env.sh diff --git a/scripts/LSSTCam/ap_daytime_cm_campaign.yaml b/scripts/LSSTCam/ap_daytime_cm_campaign.yaml new file mode 100644 index 00000000..1dd9698b --- /dev/null +++ b/scripts/LSSTCam/ap_daytime_cm_campaign.yaml @@ -0,0 +1,195 @@ +# AP_DAYTIME CM Service Scheduled Campaign Definition +# - Uses embargo_future +# - Uses LSSTCam collections +# - Uses production Slack and Kafka notifications +# All env vars `${...}` must be fulfilled *at runtime* by stack setup, explicit environment +# setting or by sourcing a file during launcher execution that does explicit environment setting +# All CM template vars `${{ ... }}` are rendered when the schedule triggers the creation of new +# campaign objects. +# All CM manifest vars `{{ ... }}` are rendered when the campaign assembles a configuration +# chain for each step-group. +# All BPS variables `{...}` are fulfilled by BPS during workflow execution and must be +# already known to BPS or added to BPS configuration. +# +# This YAML contains multiple documents of different "kinds" that are supported by the CM Service API +# - `campaign`: the campaign namespace object +# - `node`: one or more "steps" or other nodes available to a campaign graph +# - `edge`: edges define the shape of the campaign graph/DAG +# - configuration manifests `lsst`; `bps`; `wms`; `site`; `butler` provide campaign-level configuration +# - `schedule`: Defines a repeatable schedule for the campaign and TURNS ALL OTHER MANIFESTS INTO TEMPLATES +# +# This YAML may be used with the IMPORT function of the CM Service Web UI (via New Campaign) +# or with the CM cli tool: `cm schedules load .yaml` +--- +apiVersion: io.lsst.cmservice/v1 +kind: campaign +metadata: + name: ap_daytime_${{ yesterday | as_day_obs }} + day_obs: ${{ yesterday | as_day_obs }} +spec: + notification_labels: + - daytime-slack +--- +apiVersion: io.lsst.cmservice/v1 +kind: node +metadata: + kind: step + name: AP_PIPE + selectors: + butler: + compute-site: slac +spec: + bps: + pipeline_yaml: "${AP_PIPE_DIR}/pipelines/LSSTCam/ApPipeDaytime.yaml" + groups: null +--- +apiVersion: io.lsst.cmservice/v1 +kind: butler +metadata: + name: embargo_future + default: true +spec: + collections: + campaign_input: + - LSSTCam/defaults + - LSSTCam/templates + - LSSTCam/runs/prompt-${{ yesterday | as_day_obs }} + # The `campaign_public_output` is the base collection name from which all other + # collection names are derived, and it is also the name of the final chained collection + # available at the completion of the campaign. + campaign_public_output: "LSSTCam/runs/daytimeAP/${{ yesterday | as_day_obs }}" + # TODO the following collection is redundant and deprecated in CM 0.9.1 + campaign_output: "LSSTCam/runs/daytimeAP/${{ yesterday | as_day_obs }}" + include_files: [] + # The ${...} environment variables must be fulfilled externally. + # See the `custom_group_payload` attribute of the step configuration for the + # source of these values. + # The ${{...}} template expressions are fulfilled by the expressions defined + # on the "schedule" object. + predicates: + - instrument='LSSTCam' + - skymap='lsst_cells_v2' + - day_obs=${{ yesterday | as_day_obs }} + - detector not in (0,1,19,20,27,30,65,68,120,122,123,158,161,168,169,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204) + - exposure.science_program in ('BLOCK-365','BLOCK-407','BLOCK-430','BLOCK-432','BLOCK-T698','BLOCK-T703','BLOCK-T704','BLOCK-T706') + repo: embargo_future +--- +apiVersion: io.lsst.cmservice/v1 +kind: bps +metadata: + name: daytime_apdb + default: true +spec: + # The ${{...}} template expressions are fulfilled by the expressions defined + # on the "schedule" object whenever a new campaign is created by that schedule. + # The {{...}} variable placeholders are fulfilled at runtime when CM builds + # a configuration chain and renders artifacts during each campaign execution. + extra_qgraph_options: + - "--skip-existing-in LSSTCam/runs/prompt-${{ yesterday | as_day_obs }}" + - "--retained-dataset-types ${AP_PIPE_DIR}/scripts/LSSTCam/retained_types.yaml" + - "--prune-unanchored-quanta getRegionTimeFromVisit:associateApdb" + - "-c parameters:release_id=1" + - "-c parameters:apdb_config=s3://embargo@rubin-summit-users/apdb_config/cassandra/pp_apdb_lsstcam.yaml" + - "-c associateApdb:doRunForcedMeasurement=False" + - "--dataset-query-constraint off" + extra_run_quantum_options: + - "--no-raise-on-partial-outputs" + extra_aggregate_options: + - "--worker-log-dir {submitPath}/jobs/finalJob" + - "--output {submitPath}/{uniqProcName}_prov.qg" + include_files: + - "${AP_PIPE_DIR}/bps/clustering/clustering_Daytime.yaml" + literals: + numberOfRetries: 3 + retryUnlessExit: + - 2 + finalJob: + command2: "${DAF_BUTLER_DIR}/bin/butler {finalPreCmdOpts} update-dimension-regions {butlerConfig} LSSTCam {outputRun} --batch-size 2000" + command3: "${DAF_BUTLER_DIR}/bin/butler provenance-report --format json '{submitPath}/{uniqProcName}_prov.qg' > '{submitPath}/{uniqProcName}_prov.json'" +--- +apiVersion: io.lsst.cmservice/v1 +kind: lsst +metadata: + name: slac_cvmfs_w44 + default: true +spec: + campaign: AP-daytime + description: Alert Production Daytime Catchup (Daily Batch) + environment: + DAX_APDB_MONITOR_CONFIG: logging:lsst.dax.apdb.monitor + MP_SKY_URL: http://usdf-mpsky.sdf.slac.stanford.edu/ephemerides/ + lsst_distrib_dir: /cvmfs/sw.lsst.eu/almalinux-x86_64/lsst_distrib/w_2026_44 + lsst_version: w_2026_44 + project: ApPipe + ticket: DM-XXXXX +--- +apiVersion: io.lsst.cmservice/v1 +kind: wms +metadata: + name: htcondor_auto + default: true +spec: + auto_provision: true + batch_system: htcondor + service_class: lsst.ctrl.bps.htcondor.HTCondorService + # These are WMS-specific provisioning options BPS uses for allocateNodes + provisioned_idle_time: 300 + provisioned_max_wall_time: '2-0:0:0' + provisioned_node_count: 125 + provisioned_extra_arguments: + - "--cpus 16" +--- +apiVersion: io.lsst.cmservice/v1 +kind: site +metadata: + name: slac_s3df + default: true + labels: + compute-site: slac + deployment: production +spec: + facility: SLAC + # These are Site-specific provisioning options BPS uses for allocateNodes + provisioned_account_group: rubin:production + provisioned_platform: s3df + provisioned_queue: milano + # These temporary settings set up site-specific botocore/butler env vars + # and will be moved to a "library" so they are not user-managed + botocore_config_file: "${HOME}/.lsst/aws-config.cmservice.ini" + botocore_credentials_file: "${HOME}/.lsst/aws-credentials.cmservice.ini" + lsst_db_auth_file: "${HOME}/.lsst/db-auth.cmservice.yaml" + lsst_data_repos_file: "${HOME}/.lsst/data-repos.cmservice.yaml" +--- +apiVersion: io.lsst.cmservice/v1 +kind: edge +metadata: + name: START--STEP +spec: + source: START + target: AP_PIPE +--- +apiVersion: io.lsst.cmservice/v1 +kind: edge +metadata: + name: STEP--END +spec: + source: AP_PIPE + target: END +--- +apiversion: io.lsst.cmservice/v1 +kind: schedule +metadata: + owner: root +spec: + auto_start: false + cron: "30 18 * * *" + # These expressions are available to other manifests as `${{...}}` variables + # where the result of the Python expression is the object. These examples are + # both `datetime.datetime` objects. + expressions: + yesterday: "datetime.now(timezone.utc) - timedelta(days=1)" + # This format string is used to identify the new campaign created by this + # schedule as `_`. Because campaign names must be unique + # in CM Service, this format string should more or less reflect the frequency + # of the schedule execution while keeping incidental one-shot runs in mind. + name_format: "%s" diff --git a/scripts/LSSTCam/ap_daytime_cm_campaign_dev.yaml b/scripts/LSSTCam/ap_daytime_cm_campaign_dev.yaml new file mode 100644 index 00000000..4fb08736 --- /dev/null +++ b/scripts/LSSTCam/ap_daytime_cm_campaign_dev.yaml @@ -0,0 +1,198 @@ +# [DEV] AP_DAYTIME CM Service Scheduled Campaign Definition +# - Uses embargo_future +# - Uses user collections +# - Uses development db-auth file +# - Uses a random component in collection names +# +# All env vars `${...}` must be fulfilled *at runtime* by stack setup, explicit environment +# setting or by sourcing a file during launcher execution that does explicit environment setting +# All CM template vars `${{ ... }}` are rendered when the schedule triggers the creation of new +# campaign objects. +# All CM manifest vars `{{ ... }}` are rendered when the campaign assembles a configuration +# chain for each step-group. +# All BPS variables `{...}` are fulfilled by BPS during workflow execution and must be +# already known to BPS or added to BPS configuration. +# +# This YAML contains multiple documents of different "kinds" that are supported by the CM Service API +# - `campaign`: the campaign namespace object +# - `node`: one or more "steps" or other nodes available to a campaign graph +# - `edge`: edges define the shape of the campaign graph/DAG +# - configuration manifests `lsst`; `bps`; `wms`; `site`; `butler` provide campaign-level configuration +# - `schedule`: Defines a repeatable schedule for the campaign and TURNS ALL OTHER MANIFESTS INTO TEMPLATES +# +# This YAML may be used with the IMPORT function of the CM Service Web UI (via New Campaign) +# or with the CM cli tool: `cm schedules load .yaml` +--- +apiVersion: io.lsst.cmservice/v1 +kind: campaign +metadata: + name: ap_daytime_dev_${{ yesterday | as_day_obs }} + day_obs: ${{ yesterday | as_day_obs }} +spec: + notification_labels: + - default + - kafka +--- +apiVersion: io.lsst.cmservice/v1 +kind: node +metadata: + kind: step + name: AP_PIPE + selectors: + butler: + compute-site: slac +spec: + bps: + pipeline_yaml: "${AP_PIPE_DIR}/pipelines/LSSTCam/ApPipeDaytime.yaml" + groups: null +--- +apiVersion: io.lsst.cmservice/v1 +kind: butler +metadata: + name: embargo_future + default: true +spec: + collections: + campaign_input: + - LSSTCam/defaults + - LSSTCam/templates + - LSSTCam/runs/prompt-${{ yesterday | as_day_obs }} + # The `campaign_public_output` is the base collection name from which all other + # collection names are derived, and it is also the name of the final chained collection + # available at the completion of the campaign. + campaign_public_output: "u/lsstsvc1/runs/daytimeAP/${{ yesterday | as_day_obs | random_n(8, sep='/') }}" + # TODO the following collection is redundant and deprecated in CM 0.9.1 + campaign_output: "u/lsstsvc1/runs/daytimeAP/${{ yesterday | as_day_obs | random_n(8, sep='/') }}" + include_files: [] + # The ${...} environment variables must be fulfilled externally. + # See the `custom_group_payload` attribute of the step configuration for the + # source of these values. + # The ${{...}} template expressions are fulfilled by the expressions defined + # on the "schedule" object. + predicates: + - instrument='LSSTCam' + - skymap='lsst_cells_v2' + - day_obs=${{ yesterday | as_day_obs }} + - detector not in (0,1,19,20,27,30,65,68,120,122,123,158,161,168,169,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204) + - exposure.science_program in ('BLOCK-365','BLOCK-407','BLOCK-430','BLOCK-432','BLOCK-T698','BLOCK-T703','BLOCK-T704','BLOCK-T706') + repo: embargo_future +--- +apiVersion: io.lsst.cmservice/v1 +kind: bps +metadata: + name: daytime_apdb_dev + default: true +spec: + # The ${{...}} template expressions are fulfilled by the expressions defined + # on the "schedule" object whenever a new campaign is created by that schedule. + # The {{...}} variable placeholders are fulfilled at runtime when CM builds + # a configuration chain and renders artifacts during each campaign execution. + extra_qgraph_options: + - "--skip-existing-in LSSTCam/runs/prompt-${{ yesterday | as_day_obs }}" + - "--retained-dataset-types ${AP_PIPE_DIR}/scripts/LSSTCam/retained_types.yaml" + - "--prune-unanchored-quanta getRegionTimeFromVisit:associateApdb" + - "-c parameters:release_id=1" + - "-c parameters:apdb_config=s3://prompt-processing-dev@rubin-pp-dev-users/apdb_config/cassandra/pp_apdb_lsstcam-dev.yaml" + - "-c associateApdb:doRunForcedMeasurement=False" + - "--dataset-query-constraint off" + extra_run_quantum_options: + - "--no-raise-on-partial-outputs" + extra_aggregate_options: + - "--worker-log-dir {submitPath}/jobs/finalJob" + - "--output {submitPath}/{uniqProcName}_prov.qg" + include_files: + - "${AP_PIPE_DIR}/bps/clustering/clustering_Daytime.yaml" + literals: + numberOfRetries: 3 + retryUnlessExit: + - 2 + finalJob: + command2: "${DAF_BUTLER_DIR}/bin/butler {finalPreCmdOpts} update-dimension-regions {butlerConfig} LSSTCam {outputRun} --batch-size 2000" + command3: "${DAF_BUTLER_DIR}/bin/butler provenance-report --format json '{submitPath}/{uniqProcName}_prov.qg' > '{submitPath}/{uniqProcName}_prov.json'" +--- +apiVersion: io.lsst.cmservice/v1 +kind: lsst +metadata: + name: slac_cvmfs_w44 + default: true +spec: + campaign: AP-daytime + description: Alert Production Daytime Catchup (Daily Batch) + environment: + DAX_APDB_MONITOR_CONFIG: logging:lsst.dax.apdb.monitor + MP_SKY_URL: http://usdf-mpsky.sdf.slac.stanford.edu/ephemerides/ + lsst_distrib_dir: /cvmfs/sw.lsst.eu/almalinux-x86_64/lsst_distrib/w_2026_44 + lsst_version: w_2026_44 + project: ApPipe + ticket: DM-XXXXX +--- +apiVersion: io.lsst.cmservice/v1 +kind: wms +metadata: + name: htcondor_auto + default: true +spec: + auto_provision: true + batch_system: htcondor + service_class: lsst.ctrl.bps.htcondor.HTCondorService + # These are WMS-specific provisioning options BPS uses for allocateNodes + provisioned_idle_time: 300 + provisioned_max_wall_time: '2-0:0:0' + provisioned_node_count: 125 + provisioned_extra_arguments: + - "--cpus 16" +--- +apiVersion: io.lsst.cmservice/v1 +kind: site +metadata: + name: slac_s3df + default: true + labels: + compute-site: slac + deployment: development +spec: + facility: SLAC + # These are Site-specific provisioning options BPS uses for allocateNodes + provisioned_account_group: rubin:production + provisioned_platform: s3df + provisioned_queue: milano + # These temporary settings set up site-specific botocore/butler env vars + # and will be moved to a "library" so they are not user-managed + botocore_config_file: "${HOME}/.lsst/aws-config.cmservice-dev.ini" + botocore_credentials_file: "${HOME}/.lsst/aws-credentials.cmservice-dev.ini" + lsst_db_auth_file: "${HOME}/.lsst/db-auth.cmservice-dev.yaml" + lsst_data_repos_file: "${HOME}/.lsst/data-repos.cmservice-dev.yaml" +--- +apiVersion: io.lsst.cmservice/v1 +kind: edge +metadata: + name: START--STEP +spec: + source: START + target: AP_PIPE +--- +apiVersion: io.lsst.cmservice/v1 +kind: edge +metadata: + name: STEP--END +spec: + source: AP_PIPE + target: END +--- +apiversion: io.lsst.cmservice/v1 +kind: schedule +metadata: + owner: root +spec: + auto_start: false + cron: "30 18 * * *" + # These expressions are available to other manifests as `${{...}}` variables + # where the result of the Python expression is the object. These examples are + # both `datetime.datetime` objects. + expressions: + yesterday: "datetime.now(timezone.utc) - timedelta(days=1)" + # This format string is used to identify the new campaign created by this + # schedule as `_`. Because campaign names must be unique + # in CM Service, this format string should more or less reflect the frequency + # of the schedule execution while keeping incidental one-shot runs in mind. + name_format: "%s" diff --git a/scripts/LSSTCam/ap_daytime_env.sh b/scripts/LSSTCam/ap_daytime_env.sh new file mode 100644 index 00000000..9c1ea912 --- /dev/null +++ b/scripts/LSSTCam/ap_daytime_env.sh @@ -0,0 +1,16 @@ +# Potentially dynamic variable definitions used by the AP Daytime submit script + +# List of detectors currently excluded from Prompt Processing +# These include the non-imaging wavefront sensors as well as some that are disabled in fan-out. +# See https://github.com/lsst-sqre/phalanx/blob/main/applications/next-visit-fan-out/values-usdfprod-prompt-processing.yaml +BAD_DETECTORS="120 122 0 20 27 65 123 161 168 188 1 19 30 68 158 169 187 \ +189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204" + +# Space-delimited list of observing blocks that generate science images +# See https://github.com/lsst-sqre/phalanx/blob/main/applications/prompt-keda-lsstcam/values-usdfprod-prompt-processing.yaml#L21-L45 +BLOCKS="BLOCK-365 BLOCK-407 BLOCK-430 BLOCK-432 \ +BLOCK-T698 BLOCK-T703 BLOCK-T704 BLOCK-T706" + +# Convert lists to SQL IN() form +export BAD_DETECTORS_SQL="($(printf '%s,' $BAD_DETECTORS | sed 's/,$//'))" +export BLOCKS_SQL="($(printf "'%s'," $BLOCKS | sed 's/,$//'))" diff --git a/scripts/LSSTCam/submit_ap_daytime.sh b/scripts/LSSTCam/submit_ap_daytime.sh index 46be3474..c493c47c 100755 --- a/scripts/LSSTCam/submit_ap_daytime.sh +++ b/scripts/LSSTCam/submit_ap_daytime.sh @@ -18,25 +18,12 @@ ulimit -n 65536 INSTRUMENT="LSSTCam" -# List of detectors currently excluded from Prompt Processing -# These include the non-imaging wavefront sensors as well as some that are disabled in fan-out. -# See https://github.com/lsst-sqre/phalanx/blob/main/applications/next-visit-fan-out/values-usdfprod-prompt-processing.yaml -BAD_DETECTORS="120 122 0 20 27 65 123 161 168 188 1 19 30 68 158 169 187 \ -189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204" - -# Space-delimited list of observing blocks that generate science images -# See https://github.com/lsst-sqre/phalanx/blob/main/applications/prompt-keda-lsstcam/values-usdfprod-prompt-processing.yaml#L21-L45 -BLOCKS="BLOCK-365 BLOCK-407 BLOCK-430 BLOCK-432 \ -BLOCK-T698 BLOCK-T703 BLOCK-T704 BLOCK-T706" +source ./ap_daytime_env.sh OUTPUT_COLLECTION="LSSTCam/runs/daytimeAP/${DAY_OBS}" LOG_FILE="output-${DAY_OBS}.out" -# Convert lists to SQL IN() form -BAD_DETECTORS_SQL="($(printf '%s,' $BAD_DETECTORS | sed 's/,$//'))" -BLOCKS_SQL="($(printf "'%s'," $BLOCKS | sed 's/,$//'))" - # Pipeline and butler config must mirror bps_Daytime.yaml — we replicate them # here because we build the quantum graph ourselves before calling BPS. PIPELINE_YAML="${AP_PIPE_DIR}/pipelines/LSSTCam/ApPipeDaytime.yaml" From 78c78935285af3322a5509e89654d946dd8ad0ae Mon Sep 17 00:00:00 2001 From: Toby Jennings Date: Wed, 4 Nov 2026 12:39:36 -0600 Subject: [PATCH 2/5] fix(daytime): exclude detectors 81,82,83 --- scripts/LSSTCam/ap_daytime_cm_campaign.yaml | 2 +- scripts/LSSTCam/ap_daytime_cm_campaign_dev.yaml | 2 +- scripts/LSSTCam/ap_daytime_env.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/LSSTCam/ap_daytime_cm_campaign.yaml b/scripts/LSSTCam/ap_daytime_cm_campaign.yaml index 1dd9698b..b5e5cce2 100644 --- a/scripts/LSSTCam/ap_daytime_cm_campaign.yaml +++ b/scripts/LSSTCam/ap_daytime_cm_campaign.yaml @@ -70,7 +70,7 @@ spec: - instrument='LSSTCam' - skymap='lsst_cells_v2' - day_obs=${{ yesterday | as_day_obs }} - - detector not in (0,1,19,20,27,30,65,68,120,122,123,158,161,168,169,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204) + - detector not in (0,1,19,20,27,30,65,68,81,82,83,120,122,123,158,161,168,169,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204) - exposure.science_program in ('BLOCK-365','BLOCK-407','BLOCK-430','BLOCK-432','BLOCK-T698','BLOCK-T703','BLOCK-T704','BLOCK-T706') repo: embargo_future --- diff --git a/scripts/LSSTCam/ap_daytime_cm_campaign_dev.yaml b/scripts/LSSTCam/ap_daytime_cm_campaign_dev.yaml index 4fb08736..e7b1f10e 100644 --- a/scripts/LSSTCam/ap_daytime_cm_campaign_dev.yaml +++ b/scripts/LSSTCam/ap_daytime_cm_campaign_dev.yaml @@ -73,7 +73,7 @@ spec: - instrument='LSSTCam' - skymap='lsst_cells_v2' - day_obs=${{ yesterday | as_day_obs }} - - detector not in (0,1,19,20,27,30,65,68,120,122,123,158,161,168,169,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204) + - detector not in (0,1,19,20,27,30,65,68,81,82,83,120,122,123,158,161,168,169,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204) - exposure.science_program in ('BLOCK-365','BLOCK-407','BLOCK-430','BLOCK-432','BLOCK-T698','BLOCK-T703','BLOCK-T704','BLOCK-T706') repo: embargo_future --- diff --git a/scripts/LSSTCam/ap_daytime_env.sh b/scripts/LSSTCam/ap_daytime_env.sh index 9c1ea912..6e3b6542 100644 --- a/scripts/LSSTCam/ap_daytime_env.sh +++ b/scripts/LSSTCam/ap_daytime_env.sh @@ -4,7 +4,7 @@ # These include the non-imaging wavefront sensors as well as some that are disabled in fan-out. # See https://github.com/lsst-sqre/phalanx/blob/main/applications/next-visit-fan-out/values-usdfprod-prompt-processing.yaml BAD_DETECTORS="120 122 0 20 27 65 123 161 168 188 1 19 30 68 158 169 187 \ -189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204" +189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 81 82 83" # Space-delimited list of observing blocks that generate science images # See https://github.com/lsst-sqre/phalanx/blob/main/applications/prompt-keda-lsstcam/values-usdfprod-prompt-processing.yaml#L21-L45 From 244925b7120ea079d5bfb8fe1a780e94f44b607c Mon Sep 17 00:00:00 2001 From: Toby Jennings Date: Wed, 4 Nov 2026 12:55:31 -0600 Subject: [PATCH 3/5] fix(daytime): remove legacy daytime scripts --- scripts/LSSTCam/ap_daytime_env.sh | 16 ------ scripts/LSSTCam/submit_ap_daytime.sh | 84 ---------------------------- 2 files changed, 100 deletions(-) delete mode 100644 scripts/LSSTCam/ap_daytime_env.sh delete mode 100755 scripts/LSSTCam/submit_ap_daytime.sh diff --git a/scripts/LSSTCam/ap_daytime_env.sh b/scripts/LSSTCam/ap_daytime_env.sh deleted file mode 100644 index 6e3b6542..00000000 --- a/scripts/LSSTCam/ap_daytime_env.sh +++ /dev/null @@ -1,16 +0,0 @@ -# Potentially dynamic variable definitions used by the AP Daytime submit script - -# List of detectors currently excluded from Prompt Processing -# These include the non-imaging wavefront sensors as well as some that are disabled in fan-out. -# See https://github.com/lsst-sqre/phalanx/blob/main/applications/next-visit-fan-out/values-usdfprod-prompt-processing.yaml -BAD_DETECTORS="120 122 0 20 27 65 123 161 168 188 1 19 30 68 158 169 187 \ -189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 81 82 83" - -# Space-delimited list of observing blocks that generate science images -# See https://github.com/lsst-sqre/phalanx/blob/main/applications/prompt-keda-lsstcam/values-usdfprod-prompt-processing.yaml#L21-L45 -BLOCKS="BLOCK-365 BLOCK-407 BLOCK-430 BLOCK-432 \ -BLOCK-T698 BLOCK-T703 BLOCK-T704 BLOCK-T706" - -# Convert lists to SQL IN() form -export BAD_DETECTORS_SQL="($(printf '%s,' $BAD_DETECTORS | sed 's/,$//'))" -export BLOCKS_SQL="($(printf "'%s'," $BLOCKS | sed 's/,$//'))" diff --git a/scripts/LSSTCam/submit_ap_daytime.sh b/scripts/LSSTCam/submit_ap_daytime.sh deleted file mode 100755 index c493c47c..00000000 --- a/scripts/LSSTCam/submit_ap_daytime.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/bash -set -eu - -# An executable script that will prepare and submit the daytime Alert Production pipeline - -if [ "$#" -ne 1 ]; then - echo "Usage: $0 YYYYMMDD" >&2 - exit 1 -fi - -DAY_OBS="$1" - -# Redirect Cassandra logs -export DAX_APDB_MONITOR_CONFIG="logging:lsst.dax.apdb.monitor" - -# Configure the filesystem to allow many open files -ulimit -n 65536 - -INSTRUMENT="LSSTCam" - -source ./ap_daytime_env.sh - -OUTPUT_COLLECTION="LSSTCam/runs/daytimeAP/${DAY_OBS}" - -LOG_FILE="output-${DAY_OBS}.out" - -# Pipeline and butler config must mirror bps_Daytime.yaml — we replicate them -# here because we build the quantum graph ourselves before calling BPS. -PIPELINE_YAML="${AP_PIPE_DIR}/pipelines/LSSTCam/ApPipeDaytime.yaml" -APDB_CONFIG="s3://embargo@rubin-summit-users/apdb_config/cassandra/pp_apdb_lsstcam.yaml" -BUTLER_CONFIG="embargo" -INPUT_COLLECTIONS="LSSTCam/defaults,LSSTCam/templates,LSSTCam/runs/prompt-${DAY_OBS}" -RETAINED_TYPES="${AP_PIPE_DIR}/scripts/LSSTCam/retained_types.yaml" - -# Generate an explicit output run so the pre-built quantum graph -# shares a single name with the eventual BPS submission. -TIMESTAMP=$(date -u +"%Y%m%dT%H%M%SZ") -OUTPUT_RUN="${OUTPUT_COLLECTION}/${TIMESTAMP}" - -QGRAPH_DIR="$(pwd)/qgraphs/${DAY_OBS}/${TIMESTAMP}" -mkdir -p "$QGRAPH_DIR" -QGRAPH="${QGRAPH_DIR}/daytime.qg" - -DATA_QUERY="instrument='$INSTRUMENT' \ - AND skymap='lsst_cells_v2' \ - AND detector NOT IN $BAD_DETECTORS_SQL \ - AND day_obs=$DAY_OBS \ - AND exposure.science_program IN $BLOCKS_SQL" - -{ - set -e - - echo "[$(date)] Step 1/2: building quantum graph" - pipetask qgraph \ - -p "$PIPELINE_YAML" \ - -b "$BUTLER_CONFIG" \ - -i "$INPUT_COLLECTIONS" \ - --output "$OUTPUT_COLLECTION" \ - --output-run "$OUTPUT_RUN" \ - -d "$DATA_QUERY" \ - --skip-existing-in "LSSTCam/runs/prompt-${DAY_OBS}" \ - --retained-dataset-types "$RETAINED_TYPES" \ - --prune-unanchored-quanta getRegionTimeFromVisit:associateApdb \ - -c "parameters:release_id=1" \ - -c "parameters:apdb_config=${APDB_CONFIG}" \ - -c "associateApdb:doRunForcedMeasurement=False" \ - --dataset-query-constraint off \ - --qgraph-datastore-records \ - -q "$QGRAPH" - - echo "[$(date)] Step 2/2: submitting BPS workflow" - bps submit "${AP_PIPE_DIR}/bps/LSSTCam/bps_Daytime.yaml" \ - --qgraph "$QGRAPH" \ - --extra-run-quantum-options "--no-raise-on-partial-outputs" \ - --input "$INPUT_COLLECTIONS" \ - --output "$OUTPUT_COLLECTION" \ - --output-run "$OUTPUT_RUN" -} > "${LOG_FILE}" 2>&1 & -disown - -echo "Submission started for day_obs ${DAY_OBS}" -echo "APDB config: ${APDB_CONFIG}" -echo "Quantum graph: ${QGRAPH}" -echo "Submission output log written to ${LOG_FILE}" From 9a672edbbfc863fcafdebf601c95a97d3c9c62a0 Mon Sep 17 00:00:00 2001 From: Toby Jennings Date: Wed, 4 Nov 2026 15:23:02 -0600 Subject: [PATCH 4/5] fix(daytime): simplify day_obs expression --- .../LSSTCam/ap_daytime_cm_campaign_dev.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/scripts/LSSTCam/ap_daytime_cm_campaign_dev.yaml b/scripts/LSSTCam/ap_daytime_cm_campaign_dev.yaml index e7b1f10e..eb1caa75 100644 --- a/scripts/LSSTCam/ap_daytime_cm_campaign_dev.yaml +++ b/scripts/LSSTCam/ap_daytime_cm_campaign_dev.yaml @@ -26,8 +26,8 @@ apiVersion: io.lsst.cmservice/v1 kind: campaign metadata: - name: ap_daytime_dev_${{ yesterday | as_day_obs }} - day_obs: ${{ yesterday | as_day_obs }} + name: ap_daytime_dev_${{ day_obs }} + day_obs: ${{ day_obs }} spec: notification_labels: - default @@ -56,13 +56,13 @@ spec: campaign_input: - LSSTCam/defaults - LSSTCam/templates - - LSSTCam/runs/prompt-${{ yesterday | as_day_obs }} + - LSSTCam/runs/prompt-${{ day_obs }} # The `campaign_public_output` is the base collection name from which all other # collection names are derived, and it is also the name of the final chained collection # available at the completion of the campaign. - campaign_public_output: "u/lsstsvc1/runs/daytimeAP/${{ yesterday | as_day_obs | random_n(8, sep='/') }}" + campaign_public_output: "u/lsstsvc1/runs/daytimeAP/${{ day_obs | random_n(8, sep='/') }}" # TODO the following collection is redundant and deprecated in CM 0.9.1 - campaign_output: "u/lsstsvc1/runs/daytimeAP/${{ yesterday | as_day_obs | random_n(8, sep='/') }}" + campaign_output: "u/lsstsvc1/runs/daytimeAP/${{ day_obs | random_n(8, sep='/') }}" include_files: [] # The ${...} environment variables must be fulfilled externally. # See the `custom_group_payload` attribute of the step configuration for the @@ -72,7 +72,7 @@ spec: predicates: - instrument='LSSTCam' - skymap='lsst_cells_v2' - - day_obs=${{ yesterday | as_day_obs }} + - day_obs=${{ day_obs }} - detector not in (0,1,19,20,27,30,65,68,81,82,83,120,122,123,158,161,168,169,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204) - exposure.science_program in ('BLOCK-365','BLOCK-407','BLOCK-430','BLOCK-432','BLOCK-T698','BLOCK-T703','BLOCK-T704','BLOCK-T706') repo: embargo_future @@ -88,7 +88,7 @@ spec: # The {{...}} variable placeholders are fulfilled at runtime when CM builds # a configuration chain and renders artifacts during each campaign execution. extra_qgraph_options: - - "--skip-existing-in LSSTCam/runs/prompt-${{ yesterday | as_day_obs }}" + - "--skip-existing-in LSSTCam/runs/prompt-${{ day_obs }}" - "--retained-dataset-types ${AP_PIPE_DIR}/scripts/LSSTCam/retained_types.yaml" - "--prune-unanchored-quanta getRegionTimeFromVisit:associateApdb" - "-c parameters:release_id=1" @@ -187,10 +187,10 @@ spec: auto_start: false cron: "30 18 * * *" # These expressions are available to other manifests as `${{...}}` variables - # where the result of the Python expression is the object. These examples are - # both `datetime.datetime` objects. + # where the result of the Python expression is the object. expressions: yesterday: "datetime.now(timezone.utc) - timedelta(days=1)" + day_obs: "(datetime.now(timezone.utc) - timedelta(days=1)).strftime('%Y%m%d')" # This format string is used to identify the new campaign created by this # schedule as `_`. Because campaign names must be unique # in CM Service, this format string should more or less reflect the frequency From 09a136b4db221d543c34797994d586a86d28c3a9 Mon Sep 17 00:00:00 2001 From: Colin Slater Date: Fri, 6 Nov 2026 08:35:47 -0800 Subject: [PATCH 5/5] Bump AP Daytime to w45. --- scripts/LSSTCam/ap_daytime_cm_campaign.yaml | 6 +++--- scripts/LSSTCam/ap_daytime_cm_campaign_dev.yaml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/LSSTCam/ap_daytime_cm_campaign.yaml b/scripts/LSSTCam/ap_daytime_cm_campaign.yaml index b5e5cce2..da4d3bb3 100644 --- a/scripts/LSSTCam/ap_daytime_cm_campaign.yaml +++ b/scripts/LSSTCam/ap_daytime_cm_campaign.yaml @@ -110,7 +110,7 @@ spec: apiVersion: io.lsst.cmservice/v1 kind: lsst metadata: - name: slac_cvmfs_w44 + name: slac_cvmfs_w45 default: true spec: campaign: AP-daytime @@ -118,8 +118,8 @@ spec: environment: DAX_APDB_MONITOR_CONFIG: logging:lsst.dax.apdb.monitor MP_SKY_URL: http://usdf-mpsky.sdf.slac.stanford.edu/ephemerides/ - lsst_distrib_dir: /cvmfs/sw.lsst.eu/almalinux-x86_64/lsst_distrib/w_2026_44 - lsst_version: w_2026_44 + lsst_distrib_dir: /cvmfs/sw.lsst.eu/almalinux-x86_64/lsst_distrib/w_2026_45 + lsst_version: w_2026_45 project: ApPipe ticket: DM-XXXXX --- diff --git a/scripts/LSSTCam/ap_daytime_cm_campaign_dev.yaml b/scripts/LSSTCam/ap_daytime_cm_campaign_dev.yaml index eb1caa75..801b28be 100644 --- a/scripts/LSSTCam/ap_daytime_cm_campaign_dev.yaml +++ b/scripts/LSSTCam/ap_daytime_cm_campaign_dev.yaml @@ -113,7 +113,7 @@ spec: apiVersion: io.lsst.cmservice/v1 kind: lsst metadata: - name: slac_cvmfs_w44 + name: slac_cvmfs_w45 default: true spec: campaign: AP-daytime @@ -121,8 +121,8 @@ spec: environment: DAX_APDB_MONITOR_CONFIG: logging:lsst.dax.apdb.monitor MP_SKY_URL: http://usdf-mpsky.sdf.slac.stanford.edu/ephemerides/ - lsst_distrib_dir: /cvmfs/sw.lsst.eu/almalinux-x86_64/lsst_distrib/w_2026_44 - lsst_version: w_2026_44 + lsst_distrib_dir: /cvmfs/sw.lsst.eu/almalinux-x86_64/lsst_distrib/w_2026_45 + lsst_version: w_2026_45 project: ApPipe ticket: DM-XXXXX ---