From dd95f02420acecf50a8acd1d62e7e4a417abbf41 Mon Sep 17 00:00:00 2001
From: "Rob Knop (Nersc)" <raknop@lbl.gov>
Date: Sun, 20 Sep 2026 23:10:55 -0700
Subject: [PATCH 01/35] fixing ltcv ap

---
 src/ltcv.py               | 45 +++++++++++++++++++++++++++++----------
 src/webserver/baseview.py | 17 +++++++++++++++
 src/webserver/ltcvapp.py  | 18 +++++++++++-----
 3 files changed, 64 insertions(+), 16 deletions(-)

diff --git a/src/ltcv.py b/src/ltcv.py
index ff7bb19..d4adad1 100644
--- a/src/ltcv.py
+++ b/src/ltcv.py
@@ -1299,6 +1299,9 @@ def object_search( processing_version='default', just_objids=False, searchband=N
           nsn5: int
              Number of detections with S/N > 5
 
+          nfrc: int
+             Number of diaforcedsources
+
     Returns
     -------
       dict
@@ -1327,6 +1330,7 @@ def object_search( processing_version='default', just_objids=False, searchband=N
           nsn10             : int, number of detections with flux/fluxerr ≥ 10
           nsn7              : int, number of detections with flux/fluxerr ≥ 7
           nsn5              : int, number of detections with flux/fluxerr ≥ 5
+          nfrc              : int, number of forced-photometry points
 
     """
 
@@ -1356,6 +1360,7 @@ def object_search( processing_version='default', just_objids=False, searchband=N
         'nsn10':            { 'mult': False,  'sbustr': False, 'minmax': True, 'dtype': np.int16 },
         'nsn7':             { 'mult': False,  'sbustr': False, 'minmax': True, 'dtype': np.int16 },
         'nsn5':             { 'mult': False,  'sbustr': False, 'minmax': True, 'dtype': np.int16 },
+        'nfrc':             { 'mult': False,  'sbustr': False, 'minmax': True, 'dtype': np.int16 },
     }
 
     radius = None
@@ -1658,6 +1663,7 @@ def get_hot_ltcvs( processing_version, position_processing_version=None,
 
 
 def create_object_stats_materialized_view( procver ):
+    FDBLogger.info( f"Creating or refreshing object statistics materialized views for processing version {procver}" )
     with db.DBCon( dictcursor=True ) as dbcon:
         # Check to see if it already exists
         q = sql.SQL( "SELECT * FROM pg_class WHERE relname={viewname}" ).format( viewname=f'objstats_{procver}' )
@@ -1710,7 +1716,7 @@ def create_object_stats_materialized_view( procver ):
                         ).format( viewname=sql.Identifier( f'objstatscomb_{procver}' ) )
             dbcon.execute( q )
             dbcon.commit()
-            FDBLogger.info( f"Done refreshing materialized views for {procver}" )
+            FDBLogger.info( f"Done refreshing object statistics materialized views for {procver}" )
             return
 
         # If we get here, the materialized view does not exist
@@ -1741,7 +1747,8 @@ def create_object_stats_materialized_view( procver ):
                    CASE WHEN n21.ndets IS NULL THEN 0 ELSE n21.ndets END AS ndets21,
                    CASE WHEN sn10.ndets IS NULL THEN 0 ELSE sn10.ndets END AS nsn10,
                    CASE WHEN sn7.ndets IS NULL THEN 0 ELSE sn7.ndets END AS nsn7,
-                   CASE WHEN sn5.ndets IS NULL THEN 0 ELSE sn5.ndets END AS nsn5
+                   CASE WHEN sn5.ndets IS NULL THEN 0 ELSE sn5.ndets END AS nsn5,
+                   nf.nfrc
                FROM (
                  ( SELECT DISTINCT ON(o.rootid, s.band) o.rootid, r.ra, r.dec, s.band
                    FROM root_diaobject r
@@ -1751,11 +1758,11 @@ def create_object_stats_materialized_view( procver ):
                                                        AND j.procver_id={pvid}
                  )
                  UNION
-                 ( SELECT DISTINCT ON(o.rootid, s.band) o.rootid, r.ra, r.dec, s.band
+                 ( SELECT DISTINCT ON(o.rootid, f.band) o.rootid, r.ra, r.dec, f.band
                    FROM root_diaobject r
                    INNER JOIN diaobject o ON o.rootid=r.id
-                   INNER JOIN diaforcedsource s ON s.diaobjectid=o.diaobjectid
-                   INNER JOIN base_procver_of_procver j ON s.base_procver_id=j.base_procver_id
+                   INNER JOIN diaforcedsource f ON f.diaobjectid=o.diaobjectid
+                   INNER JOIN base_procver_of_procver j ON f.base_procver_id=j.base_procver_id
                                                        AND j.procver_id={pvid}
                  )
                ) r
@@ -1767,6 +1774,7 @@ def create_object_stats_materialized_view( procver ):
                     INNER JOIN diaobject o ON s.diaobjectid=o.diaobjectid
                     INNER JOIN base_procver_of_procver j ON s.base_procver_id=j.base_procver_id
                                                         AND j.procver_id={pvid}
+                    WHERE s.psfflux > 0
                     ORDER BY o.rootid, s.visit, j.priority DESC
                  ) subq
                  ORDER BY rootid, band, midpointmjdtai
@@ -1779,6 +1787,7 @@ def create_object_stats_materialized_view( procver ):
                     INNER JOIN diaobject o ON s.diaobjectid=o.diaobjectid
                     INNER JOIN base_procver_of_procver j ON s.base_procver_id=j.base_procver_id
                                                         AND j.procver_id={pvid}
+                    WHERE s.psfflux > 0
                     ORDER BY o.rootid, s.visit, j.priority DESC
                  ) subq
                  ORDER BY rootid, band, midpointmjdtai DESC
@@ -1791,6 +1800,7 @@ def create_object_stats_materialized_view( procver ):
                     INNER JOIN diaobject o ON s.diaobjectid=o.diaobjectid
                     INNER JOIN base_procver_of_procver j ON s.base_procver_id=j.base_procver_id
                                                         AND j.procver_id={pvid}
+                    WHERE s.psfflux > 0
                     ORDER BY o.rootid, s.visit, j.priority DESC
                  ) subq
                  ORDER BY rootid, band, psfflux DESC
@@ -1910,15 +1920,27 @@ def create_object_stats_materialized_view( procver ):
                   WHERE psfflux / psffluxerr >= 5
                   GROUP BY rootid, band
                ) sn5 ON r.rootid=sn5.rootid AND r.band=sn5.band
+               LEFT JOIN (
+                  SELECT rootid, band, COUNT(*) AS nfrc
+                  FROM (
+                     SELECT DISTINCT ON(o.rootid, f.visit) o.rootid, f.band
+                     FROM diaforcedsource f
+                     INNER JOIN diaobject o ON f.diaobjectid=o.diaobjectid
+                     INNER JOIN base_procver_of_procver j ON f.base_procver_id=j.base_procver_id
+                                                         AND j.procver_id={pvid}
+                     ORDER BY o.rootid, f.visit, j.priority DESC
+                  ) subq
+                  GROUP BY rootid, band
+               ) nf ON r.rootid=n.rootid AND r.band=n.band
             )
             """
         ) ).format( viewname=sql.Identifier( f'objstats_{procver}' ), pvid=pvid )
 
-        dbcon.execute_nofetch( q, explain=False )
+        dbcon.execute_nofetch( q, explain=True )
 
         indexcols = [ 'rootid', 'firstdet_mjd', 'lastdet_mjd', 'maxdet_mjd',
                      'firstdet_flux', 'lastdet_flux', 'maxdet_flux',
-                     'ndets', 'ndets24', 'ndets23', 'ndets22', 'ndets21', 'nsn10', 'nsn7', 'nsn5' ]
+                     'ndets', 'ndets24', 'ndets23', 'ndets22', 'ndets21', 'nsn10', 'nsn7', 'nsn5', 'nfrc' ]
         for col in indexcols:
             q = sql.SQL( 'CREATE INDEX {idxname} ON {viewname}({col})'
                         ).format( idxname=sql.Identifier( f'idx_obstats_{procver}_{col}' ),
@@ -1950,11 +1972,12 @@ def create_object_stats_materialized_view( procver ):
                        lf.flux AS lastforced_flux, lf.fluxerr AS lastforced_fluxerr,
                      s.ndets AS ndets, s.ndets24 AS ndets24, s.ndets23 AS ndets23,
                        s.ndets22 AS ndets22, s.ndets21 AS ndets21,
-                     s.nsn10 AS nsn10, s.nsn7 AS nsn7, s.nsn5 AS nsn5
+                     s.nsn10 AS nsn10, s.nsn7 AS nsn7, s.nsn5 AS nsn5,
+                     s.nfrc AS nfrc
               FROM (
                 SELECT rootid, ra, dec, SUM(ndets) AS ndets, SUM(ndets24) AS ndets24, SUM(ndets23) AS ndets23,
                        SUM(ndets22) AS ndets22, SUM(ndets21) AS ndets21, SUM(nsn10) AS nsn10,
-                       SUM(nsn7) AS nsn7, SUM(nsn5) AS nsn5
+                       SUM(nsn7) AS nsn7, SUM(nsn5) AS nsn5, SUM(nfrc) AS nfrc
                 FROM {viewname}
                 GROUP BY rootid, ra, dec
               ) s
@@ -1990,7 +2013,7 @@ def create_object_stats_materialized_view( procver ):
             )
             """ ) ).format( viewname=sql.Identifier(f'objstats_{procver}'),
                             combviewname=sql.Identifier(f'objstatscomb_{procver}') )
-        dbcon.execute( q, explain=False )
+        dbcon.execute( q, explain=True )
 
         for col in indexcols:
             q = sql.SQL( 'CREATE INDEX {idxname} ON {viewname}({col})'
@@ -2005,4 +2028,4 @@ def create_object_stats_materialized_view( procver ):
         dbcon.execute( q, explain=False )
 
         dbcon.commit()
-        FDBLogger.info( f"Done creating materialized view objstats_{procver}" )
+        FDBLogger.info( f"Done creating object statistics materialized views for {procver}" )
diff --git a/src/webserver/baseview.py b/src/webserver/baseview.py
index d431838..137f145 100644
--- a/src/webserver/baseview.py
+++ b/src/webserver/baseview.py
@@ -1,5 +1,7 @@
 from types import SimpleNamespace
 import simplejson
+import copy
+import re
 
 import flask
 import flask.views
@@ -46,6 +48,21 @@ class BaseView( flask.views.View ):
     def __init__( self, *args, **kwargs ):
         super().__init__( *args, **kwargs )
 
+    def argstr_to_args( self, argstr, initargs={} ):
+        """Parse argstr as a bunch of /kw=val to a dictionary, update with request body if it's json."""
+
+        args = copy.deepcopy( initargs )
+        if argstr is not None:
+            for arg in argstr.split("/"):
+                match = re.search( '^(?P<k>[^=]+)=(?P<v>.*)$', arg )
+                if match is None:
+                    FDBLogger( f"error parsing url argument {arg}, must be key=value" )
+                    raise FASTDBWebException( f'error parsing url argument {arg}, must be key=value' )
+                args[ match.group('k') ] = match.group('v')
+        if flask.request.is_json:
+            args.update( flask.request.json )
+        return args
+
     def check_auth( self ):
         self.username = flask.session['username'] if 'username' in flask.session else '(None)'
         self.displayname = flask.session['userdisplayname'] if 'userdisplayname' in flask.session else '(None)'
diff --git a/src/webserver/ltcvapp.py b/src/webserver/ltcvapp.py
index 5f08cb3..624b234 100644
--- a/src/webserver/ltcvapp.py
+++ b/src/webserver/ltcvapp.py
@@ -90,7 +90,7 @@ def get_ltcvs( self, procver, objids=None, dbcon=None ):
 
         if ( 'return_object_info' in kwargs ) and ( kwargs['return_object_info'] ):
             rval = { 'ltcvs': rval[0], 'objinfo': rval[1] }
-            FDBLogger.error( f"Returning: {rval}\n" )
+            FDBLogger.debug( f"Returning: {rval}\n" )
 
         return rval
 
@@ -144,11 +144,14 @@ def do_the_things( self, procver, objid=None ):
 
 class GetRandomLtcv( GetLtcv ):
     def do_the_things( self, procver="default" ):
+        FDBLogger.debug( "Getting random lightcurve." )
         with db.DBCon() as dbcon:
             pv = db.ProcessingVersion.procver_id( procver, dbcon=dbcon  )
 
             q = sql.SQL(
                 """
+                /*+ IndexScan(o idx_diaobject_procver)
+                */
                 SELECT diaobjectid
                 FROM (
                   SELECT DISTINCT ON (o.diaobjectid) o.diaobjectid
@@ -160,12 +163,17 @@ def do_the_things( self, procver="default" ):
                 ORDER BY random() LIMIT 1
                 """
             ).format( procver=pv )
-            rows, _cols = dbcon.execute( q )
+            rows, _cols = dbcon.execute( q, explain=True )
 
-            mess = self.get_ltcvs( pv, [ rows[0][0] ], dbcon=dbcon )
+            doid = rows[0][0]
+            FDBLogger.debug( f"Getting lightcurve for diaobjectid {doid}" )
+            rval = self.get_ltcvs( pv, [ doid ], dbcon=dbcon )
+        rval['ltcv'] = rval['ltcvs'][0]
+        del rval['ltcvs']
+        rval['objinfo'] = { k: v[0] for k, v in rval['objinfo'].items() }
 
-        key0 = list( mess.keys() )[0]
-        return mess[ key0 ]
+        FDBLogger.debug( f"Returning lightcurve for {rval['ltcv']['rootid']} ({doid})" )
+        return rval
 
 
 # ======================================================================

From 17739e94aec4a8f6adcf36049c198c71f02ecd73 Mon Sep 17 00:00:00 2001
From: "Rob Knop (Nersc)" <raknop@lbl.gov>
Date: Mon, 21 Sep 2026 17:14:14 -0700
Subject: [PATCH 02/35] BROKEN -- add a db migration to add indexes to some
 tables, not working

---
 db/2026-09-21a_procverintables.sql | 43 ++++++++++++++++++++++++++++++
 src/ltcv.py                        |  6 ++++-
 2 files changed, 48 insertions(+), 1 deletion(-)
 create mode 100644 db/2026-09-21a_procverintables.sql

diff --git a/db/2026-09-21a_procverintables.sql b/db/2026-09-21a_procverintables.sql
new file mode 100644
index 0000000..f035860
--- /dev/null
+++ b/db/2026-09-21a_procverintables.sql
@@ -0,0 +1,43 @@
+ALTER TABLE diaobject ADD COLUMN procver_id uuid;
+ALTER TABLE diaobject_position ADD COLUMN procver_id uuid;
+ALTER TABLE diasource ADD COLUMN procver_id uuid;
+ALTER TABLE diaforcedsource ADD COLUMN procver_id uuid;
+
+UPDATE diaobject o SET procver_id=q.procver_id
+FROM (
+ SELECT o.diaobjectid, j.procver_id
+ FROM diaobject o
+ INNER JOIN base_procver_of_procver j ON o.base_procver_id=j.base_procver_id
+) q
+WHERE q.diaobjectid=o.diaobjectid;
+
+UPDATE diaobject_position p SET procver_id=q.procver_id
+FROM (
+ SELECT p.diaobjectid, j.procver_id
+ FROM diaobject_position p
+ INNER JOIN base_procver_of_procver j ON p.base_procver_id=j.base_procver_id
+) q
+WHERE q.diaobjectid=o.diaobjectid;
+
+UPDATE diasource s SET procver_id=q.procver_id
+FROM (
+ SELECT s.diasourceid, j.procver_id
+ FROM diasource s
+ INNER JOIN base_procver_of_procver j ON s.base_procver_id=j.base_procver_id
+) q
+WHERE q.diasourceid=o.diasourceid;
+
+UPDATE diaforcedsource f SET procver_id=q.procver_id
+FROM (
+ SELECT f.diaforcedsourceid j.procver_id
+ FROM diaforcedsource f
+ INNER JOIN base_procver_of_procver j ON f.base_procver_id=j.base_procver_id
+) q
+WHERE q.diaforcedsourceid=o.diaforcedsourceid;
+
+ALTER INDEX idx_diaobject_base_procver_id RENAME TO idx_diaobject_position_base_procver_id;
+ALTER INDEX idx_diaobject_procver RENAME TO idx_diaobject_base_procver_id;
+CREATE INDEX idx_diaobject_procver_id ON diaobject (procver_id);
+CREATE INDEX idx_diasource_procver_id ON diasource (procver_id);
+CREATE INDEX idx_diaforcedsource_procver_id ON diaforcedsource (procver_id);
+CREATE INDEX Idx_diaobject_position_procver_id ON diaobject_position (procver_id);
diff --git a/src/ltcv.py b/src/ltcv.py
index d4adad1..88ec94c 100644
--- a/src/ltcv.py
+++ b/src/ltcv.py
@@ -1732,7 +1732,11 @@ def create_object_stats_materialized_view( procver ):
         pvid = db.ProcessingVersion.procver_id( procver, dbcon=dbcon )
 
         q = sql.SQL( textwrap.dedent(
-            """
+            """\
+            /*+ IndexScan (s idx_diasource_base_procver_id)
+                IndexScan (f idx_diaforcedsource_base_procver_id)
+                IndexScan (o idx_diaobject_diaobjectid idx_diaobject_procver)
+            */
             CREATE MATERIALIZED VIEW {viewname} AS (
                SELECT r.rootid, r.ra, r.dec, r.band,
                    d0.midpointmjdtai AS firstdet_mjd, d0.psfflux AS firstdet_flux, d0.psffluxerr AS firstdet_fluxerr,

From 08b11643dc6a18e59c96a86831e993cff2e2e55d Mon Sep 17 00:00:00 2001
From: Rob Knop <rknop@pobox.com>
Date: Mon, 21 Sep 2026 22:33:06 -0400
Subject: [PATCH 03/35] Add offset, limit, sort to object search, also in web
 ap.  Fix a horrible problem in materialized view generation.  Add a migration
 for adding procver_id to several tables, but disable it for now.

---
 ...26-09-21_001_procverintables.sql_disabled} |   8 +-
 docker/webserver/Dockerfile                   |   7 +-
 extern/rkwebutil                              |   2 +-
 src/admin/load_edp2_parquet.py                |   2 +-
 src/db.py                                     |  44 +++---
 src/ltcv.py                                   | 148 ++++++++++++------
 src/services/source_importer.py               |   2 +-
 src/webserver/static/fastdb.css               |   2 +
 src/webserver/static/objectinfo.js            |  12 +-
 src/webserver/static/objectlist.js            |  30 ++--
 src/webserver/static/objectsearch.js          |  82 +++++++++-
 tests/admin/test_load_edp2_parquet.py         |   9 +-
 12 files changed, 235 insertions(+), 113 deletions(-)
 rename db/{2026-09-21a_procverintables.sql => 2026-09-21_001_procverintables.sql_disabled} (90%)

diff --git a/db/2026-09-21a_procverintables.sql b/db/2026-09-21_001_procverintables.sql_disabled
similarity index 90%
rename from db/2026-09-21a_procverintables.sql
rename to db/2026-09-21_001_procverintables.sql_disabled
index f035860..f0b1e65 100644
--- a/db/2026-09-21a_procverintables.sql
+++ b/db/2026-09-21_001_procverintables.sql_disabled
@@ -17,7 +17,7 @@ FROM (
  FROM diaobject_position p
  INNER JOIN base_procver_of_procver j ON p.base_procver_id=j.base_procver_id
 ) q
-WHERE q.diaobjectid=o.diaobjectid;
+WHERE q.diaobjectid=p.diaobjectid;
 
 UPDATE diasource s SET procver_id=q.procver_id
 FROM (
@@ -25,15 +25,15 @@ FROM (
  FROM diasource s
  INNER JOIN base_procver_of_procver j ON s.base_procver_id=j.base_procver_id
 ) q
-WHERE q.diasourceid=o.diasourceid;
+WHERE q.diasourceid=s.diasourceid;
 
 UPDATE diaforcedsource f SET procver_id=q.procver_id
 FROM (
- SELECT f.diaforcedsourceid j.procver_id
+ SELECT f.diaforcedsourceid, j.procver_id
  FROM diaforcedsource f
  INNER JOIN base_procver_of_procver j ON f.base_procver_id=j.base_procver_id
 ) q
-WHERE q.diaforcedsourceid=o.diaforcedsourceid;
+WHERE q.diaforcedsourceid=f.diaforcedsourceid;
 
 ALTER INDEX idx_diaobject_base_procver_id RENAME TO idx_diaobject_position_base_procver_id;
 ALTER INDEX idx_diaobject_procver RENAME TO idx_diaobject_base_procver_id;
diff --git a/docker/webserver/Dockerfile b/docker/webserver/Dockerfile
index b043bc7..33dff67 100644
--- a/docker/webserver/Dockerfile
+++ b/docker/webserver/Dockerfile
@@ -151,9 +151,10 @@ RUN git clone https://github.com/mongodb/mongo-tools.git \
 
 # lsdb in PyPi requires older versions of pandas and nested_pandas than
 # we want.  This dev branch has updates in pyproject.toml
-RUN source /venv/bin/activate \
-   && pip --no-cache install "hats @ git+https://github.com/astronomy-commons/hats.git@pandas3" \
-                             "lsdb @ git+https://github.com/astronomy-commons/lsdb.git@pandas3"
+# CURRENTLY NOT WORKING
+# RUN source /venv/bin/activate \
+#    && pip --no-cache install "hats @ git+https://github.com/astronomy-commons/hats.git@pandas3" \
+ #                             "lsdb @ git+https://github.com/astronomy-commons/lsdb.git@pandas3"
 
 # ======================================================================
 # This is for interaction.  It runs something that will stay up that you can exec a shell on.
diff --git a/extern/rkwebutil b/extern/rkwebutil
index c5d4bcc..d0b2c6f 160000
--- a/extern/rkwebutil
+++ b/extern/rkwebutil
@@ -1 +1 @@
-Subproject commit c5d4bcc9e2a3f936420f4a45362adb9e53fe7f73
+Subproject commit d0b2c6f43c874048e7315f244072367d84d23846
diff --git a/src/admin/load_edp2_parquet.py b/src/admin/load_edp2_parquet.py
index ce687eb..9ebd49f 100644
--- a/src/admin/load_edp2_parquet.py
+++ b/src/admin/load_edp2_parquet.py
@@ -33,7 +33,7 @@ def __init__( self, processing_version, create_pv=False, test_only=False, slow_t
                 except Exception:
                     col = 'description'
                 if self._slow_test or ( not self._test_only ):
-                    pgdb.execute( "LOCK TABLE processing_version" )
+                    pgdb.execute( "LOCK TABLE processing_version", explain=False )
                 rows = pgdb.execute( sql.SQL( "SELECT * FROM processing_version WHERE {col}={pv}" )
                                      .format( col=sql.Identifier(col), pv=processing_version ) )
                 if len(rows) > 0:
diff --git a/src/db.py b/src/db.py
index 82903e0..01abdf3 100644
--- a/src/db.py
+++ b/src/db.py
@@ -47,7 +47,7 @@
 # explaining can slow down queries as sometimes it seems that
 # postgres really wants to think about what it's doing before giving you
 # a query plan (I don't know why; is it a pg_hint_plan thing?)
-_echoqueries = False
+_echoqueries = True
 _alwaysexplain = False
 _alwaysanalyze = False
 
@@ -506,16 +506,16 @@ def construct_pgsql_where_clause( searchspec, where="WHERE", **kwargs ):
             if util.isSequence( kwargs[field] ):
                 if not fieldinfo[ 'mult' ]:
                     raise ValueError( f"Field {field} can't be a list" )
-                q += sql.SQL( "{where} {field}=ANY(%({sfield})s)" ).format( where=sql.SQL(where),
-                                                                            field=sql.Identifier(field),
-                                                                            sfield=sql.SQL(field) )
+                q += sql.SQL( "{where} {field}=ANY(%({sfield})s)\n" ).format( where=sql.SQL(where),
+                                                                              field=sql.Identifier(field),
+                                                                              sfield=sql.SQL(field) )
                 subdict[field] = list( kwargs[field] )
             else:
-                q += sql.SQL( "{where} {field}=%({sfield})s" ).format( where=sql.SQL(where),
-                                                                       field=sql.Identifier(field),
-                                                                       sfield=sql.SQL(field) )
+                q += sql.SQL( "{where} {field}=%({sfield})s\n" ).format( where=sql.SQL(where),
+                                                                         field=sql.Identifier(field),
+                                                                         sfield=sql.SQL(field) )
                 subdict[field] = kwargs[field]
-            where = " AND"
+            where = "  AND"
             del kwargs[field]
 
         if f'{field}_contains' in kwargs:
@@ -534,14 +534,14 @@ def construct_pgsql_where_clause( searchspec, where="WHERE", **kwargs ):
                                     sfield=sql.SQL(field),
                                     i=sql.SQL(str(i)) ) )
                     subdict[f'{field}_contains_{i}'] = f'%{val}%'
-                q += sql.SQL( ")" )
-                where = " AND"
+                q += sql.SQL( ")\n" )
+                where = "  AND"
             else:
-                q += sql.SQL( "{where} {field} LIKE %({sfield}_contains)s" ).format( where=sql.SQL(where),
-                                                                                     field=sql.Identifier(field),
-                                                                                     sfield=sql.SQL(field) )
+                q += sql.SQL( "{where} {field} LIKE %({sfield}_contains)s\n" ).format( where=sql.SQL(where),
+                                                                                       field=sql.Identifier(field),
+                                                                                       sfield=sql.SQL(field) )
                 subdict[f'{field}_contains'] = f"%{kwargs[f'{field}_contains']}%"
-                where = " AND"
+                where = "  AND"
             del kwargs[f'{field}_contains']
 
         if f'{field}_min' in kwargs:
@@ -549,11 +549,11 @@ def construct_pgsql_where_clause( searchspec, where="WHERE", **kwargs ):
                 raise ValueError( f'Field {field} doesn\'t work with "min"' )
             if util.isSequence( kwargs[f'{field}_min'] ):
                 raise ValueError( f"{field}_max can't be a list" )
-            q += sql.SQL( "{where} {field}>=%({sfield}_min)s" ).format( where=sql.SQL(where),
-                                                                        field=sql.Identifier(field),
-                                                                        sfield=sql.SQL(field) )
+            q += sql.SQL( "{where} {field}>=%({sfield}_min)s\n" ).format( where=sql.SQL(where),
+                                                                          field=sql.Identifier(field),
+                                                                          sfield=sql.SQL(field) )
             subdict[f'{field}_min'] = kwargs[f'{field}_min']
-            where = " AND"
+            where = "  AND"
             del kwargs[f'{field}_min']
 
         if f'{field}_max' in kwargs:
@@ -561,11 +561,11 @@ def construct_pgsql_where_clause( searchspec, where="WHERE", **kwargs ):
                 raise ValueError( f'Field {field} doesn\'t work with "max"' )
             if util.isSequence( kwargs[f'{field}_max'] ):
                 raise ValueError( f"{field}_max can't be a list" )
-            q += sql.SQL( "{where} {field}<=%({sfield}_max)s" ).format( where=sql.SQL(where),
-                                                                        field=sql.Identifier(field),
-                                                                        sfield=sql.SQL(field) )
+            q += sql.SQL( "{where} {field}<=%({sfield}_max)s\n" ).format( where=sql.SQL(where),
+                                                                          field=sql.Identifier(field),
+                                                                          sfield=sql.SQL(field) )
             subdict[f'{field}_max'] = kwargs[f'{field}_max']
-            where = " AND"
+            where = "  AND"
             del kwargs[f'{field}_max']
 
     # Differences
diff --git a/src/ltcv.py b/src/ltcv.py
index 88ec94c..716275d 100644
--- a/src/ltcv.py
+++ b/src/ltcv.py
@@ -1181,7 +1181,8 @@ def debug_count_temp_table( con, table ):
 _object_search_timings_count = {}
 
 
-def object_search( processing_version='default', just_objids=False, searchband=None, dbcon=None, **kwargs ):
+def object_search( processing_version='default', just_objids=False, searchband=None, orderby="rootid",
+                   limit=None, offset=None, dbcon=None, **kwargs ):
     """Search for objects.
 
     This is a relatively fast search that only looks at object stats
@@ -1202,6 +1203,19 @@ def object_search( processing_version='default', just_objids=False, searchband=N
          list of bands is given, then the cuts will only consider
          photometry with this band or these bands.
 
+      orderby: str or list of str, default rootid
+         Sort the returned objects by these fields.  Fields can include
+         (almost) anything that's in the return.  Prepend "-" to the
+         name to make it a descending sort rather than an ascending
+         sort.
+
+      limit: int or None
+         If given, only return this many rows.
+
+      offset: int or None
+         If given, return limit rows starting at this offset.
+
+
       SEARCH FIELDS:
 
           For most of the following fields (i.e. where not indicated
@@ -1310,27 +1324,30 @@ def object_search( processing_version='default', just_objids=False, searchband=N
       column values.  It should be safe to stuff this directly into
       pandas.DataFrame().  Column names are:
 
-          rootid            : UUID, the rootid of the object
-          ra                : float
-          dec               : float
-          firstdet_mjd      : float, mjd of first detection (diasource)
-          firstdet_flux     : float, flux in nJy (zeropoint 31.4) of first detection
-          firstdet_fluxerr  : float
-          lastdet_mjd       : float, mjd of last detection (diasource)
-          lastdet_flux      : float, flux in nJy (zeropoint 31.4) of last detection
-          lastdet_fluxerr   : float
-          maxdet_mjd        : float, mjd of max (i.e. highest-flux) detection (diasource)
-          maxdet_flux       : float, flux in nJy (zeropoint 31.4) of max detection
-          maxdet_fluxerr    : float
-          ndets             : int, number of detections (diasources for this processing version and rootid)
-          ndets24           : int, number of detections with mag ≤ 24
-          ndets23           : int, number of detections with mag ≤ 23
-          ndets22           : int, number of detections with mag ≤ 22
-          ndets21           : int, number of detections with mag ≤ 21
-          nsn10             : int, number of detections with flux/fluxerr ≥ 10
-          nsn7              : int, number of detections with flux/fluxerr ≥ 7
-          nsn5              : int, number of detections with flux/fluxerr ≥ 5
-          nfrc              : int, number of forced-photometry points
+          rootid             : UUID, the rootid of the object
+          ra                 : float
+          dec                : float
+          firstdet_mjd       : float, mjd of first detection (diasource)
+          firstdet_flux      : float, flux in nJy (zeropoint 31.4) of first detection
+          firstdet_fluxerr   : float
+          lastdet_mjd        : float, mjd of last detection (diasource)
+          lastdet_flux       : float, flux in nJy (zeropoint 31.4) of last detection
+          lastdet_fluxerr    : float
+          maxdet_mjd         : float, mjd of max (i.e. highest-flux) detection (diasource)
+          maxdet_flux        : float, flux in nJy (zeropoint 31.4) of max detection
+          maxdet_fluxerr     : float
+          lastforced_mjd     : float, mjd of the last forced photometry measurement we have
+          lastforced_flux    : float, flux in nJy (zeropoint 31.4) of last forced photometry point
+          lastforced_fluxerr : float
+          ndets              : int, number of detections (diasources for this processing version and rootid)
+          ndets24            : int, number of detections with mag ≤ 24
+          ndets23            : int, number of detections with mag ≤ 23
+          ndets22            : int, number of detections with mag ≤ 22
+          ndets21            : int, number of detections with mag ≤ 21
+          nsn10              : int, number of detections with flux/fluxerr ≥ 10
+          nsn7               : int, number of detections with flux/fluxerr ≥ 7
+          nsn5               : int, number of detections with flux/fluxerr ≥ 5
+          nfrc               : int, number of forced-photometry points
 
     """
 
@@ -1340,27 +1357,30 @@ def object_search( processing_version='default', just_objids=False, searchband=N
     viewname = f'objstatscomb_{pvobj.description}' if len(searchband) == 0 else f'objstats_{pvobj.description}'
 
     searchspec = {
-        'rootid':           { 'mult': True,   'substr': False, 'minmax': False, 'dtype': np.dtype('O') },
-        'ra':               { 'mult': False,  'sbustr': False, 'minmax': True, 'dtype': np.float64 },
-        'dec':              { 'mult': False,  'sbustr': False, 'minmax': True, 'dtype': np.float64 },
-        'firstdet_mjd':     { 'mult': False,  'sbustr': False, 'minmax': True, 'dtype': np.float64 },
-        'firstdet_flux':    { 'mult': False,  'sbustr': False, 'minmax': True, 'dtype': np.float32 },
-        'firstdet_fluxerr': { 'mult': False,  'sbustr': False, 'minmax': True, 'dtype': np.float32 },
-        'lastdet_mjd':      { 'mult': False,  'sbustr': False, 'minmax': True, 'dtype': np.float64 },
-        'lastdet_flux':     { 'mult': False,  'sbustr': False, 'minmax': True, 'dtype': np.float32 },
-        'lastdet_fluxerr':  { 'mult': False,  'sbustr': False, 'minmax': True, 'dtype': np.float32 },
-        'maxdet_mjd':       { 'mult': False,  'sbustr': False, 'minmax': True, 'dtype': np.float64 },
-        'maxdet_flux':      { 'mult': False,  'sbustr': False, 'minmax': True, 'dtype': np.float32 },
-        'maxdet_fluxerr':   { 'mult': False,  'sbustr': False, 'minmax': True, 'dtype': np.int16 },
-        'ndets':            { 'mult': False,  'sbustr': False, 'minmax': True, 'dtype': np.int16 },
-        'ndets24':          { 'mult': False,  'sbustr': False, 'minmax': True, 'dtype': np.int16 },
-        'ndets23':          { 'mult': False,  'sbustr': False, 'minmax': True, 'dtype': np.int16 },
-        'ndets22':          { 'mult': False,  'sbustr': False, 'minmax': True, 'dtype': np.int16 },
-        'ndets21':          { 'mult': False,  'sbustr': False, 'minmax': True, 'dtype': np.int16 },
-        'nsn10':            { 'mult': False,  'sbustr': False, 'minmax': True, 'dtype': np.int16 },
-        'nsn7':             { 'mult': False,  'sbustr': False, 'minmax': True, 'dtype': np.int16 },
-        'nsn5':             { 'mult': False,  'sbustr': False, 'minmax': True, 'dtype': np.int16 },
-        'nfrc':             { 'mult': False,  'sbustr': False, 'minmax': True, 'dtype': np.int16 },
+        'rootid':             { 'mult': True,   'substr': False, 'minmax': False, 'dtype': np.dtype('O') },
+        'ra':                 { 'mult': False,  'substr': False, 'minmax': True, 'dtype': np.float64 },
+        'dec':                { 'mult': False,  'substr': False, 'minmax': True, 'dtype': np.float64 },
+        'firstdet_mjd':       { 'mult': False,  'substr': False, 'minmax': True, 'dtype': np.float64 },
+        'firstdet_flux':      { 'mult': False,  'substr': False, 'minmax': True, 'dtype': np.float32 },
+        'firstdet_fluxerr':   { 'mult': False,  'substr': False, 'minmax': True, 'dtype': np.float32 },
+        'lastdet_mjd':        { 'mult': False,  'substr': False, 'minmax': True, 'dtype': np.float64 },
+        'lastdet_flux':       { 'mult': False,  'substr': False, 'minmax': True, 'dtype': np.float32 },
+        'lastdet_fluxerr':    { 'mult': False,  'substr': False, 'minmax': True, 'dtype': np.float32 },
+        'maxdet_mjd':         { 'mult': False,  'substr': False, 'minmax': True, 'dtype': np.float64 },
+        'maxdet_flux':        { 'mult': False,  'substr': False, 'minmax': True, 'dtype': np.float32 },
+        'maxdet_fluxerr':     { 'mult': False,  'substr': False, 'minmax': True, 'dtype': np.float32 },
+        'lastforced_mjd':     { 'mult': False,  'substr': False, 'minmax': True, 'dtype': np.float32 },
+        'lastforced_flux':    { 'mult': False,  'substr': False, 'minmax': True, 'dtype': np.int16 },
+        'lastforced_fluxerr': { 'mult': False,  'substr': False, 'minmax': True, 'dtype': np.int16 },
+        'ndets':              { 'mult': False,  'substr': False, 'minmax': True, 'dtype': np.int16 },
+        'ndets24':            { 'mult': False,  'substr': False, 'minmax': True, 'dtype': np.int16 },
+        'ndets23':            { 'mult': False,  'substr': False, 'minmax': True, 'dtype': np.int16 },
+        'ndets22':            { 'mult': False,  'substr': False, 'minmax': True, 'dtype': np.int16 },
+        'ndets21':            { 'mult': False,  'substr': False, 'minmax': True, 'dtype': np.int16 },
+        'nsn10':              { 'mult': False,  'substr': False, 'minmax': True, 'dtype': np.int16 },
+        'nsn7':               { 'mult': False,  'substr': False, 'minmax': True, 'dtype': np.int16 },
+        'nsn5':               { 'mult': False,  'substr': False, 'minmax': True, 'dtype': np.int16 },
+        'nfrc':               { 'mult': False,  'substr': False, 'minmax': True, 'dtype': np.int16 },
     }
 
     radius = None
@@ -1380,14 +1400,14 @@ def object_search( processing_version='default', just_objids=False, searchband=N
         if len( rows ) == 0:
             raise RuntimeError( f"Can't do object search, materialized view {viewname} doesn't exist" )
 
-        q = sql.SQL( "SELECT * FROM {viewname} " ).format( viewname=sql.Identifier(viewname) )
+        q = sql.SQL( "SELECT * FROM {viewname}\n" ).format( viewname=sql.Identifier(viewname) )
         where = "WHERE"
         if len(searchband) > 0:
             if len(searchband) == 1:
-                q += sql.SQL( "WHERE band={band}" ).format( searchband )
+                q += sql.SQL( "WHERE band={band}\n" ).format( searchband )
             else:
-                q += sql.SQL( "WHERE band=ANY(ARRAY[{band}])" ).format( sql.SQL(",").join(searchband) )
-            where = " AND"
+                q += sql.SQL( "WHERE band=ANY(ARRAY[{band}])\n" ).format( sql.SQL(",").join(searchband) )
+            where = "  AND"
 
         qwhere, subdict, remainder, where = db.construct_pgsql_where_clause( searchspec, where=where, **kwargs )
         if len(remainder) > 0:
@@ -1396,13 +1416,38 @@ def object_search( processing_version='default', just_objids=False, searchband=N
         q += qwhere
 
         if radius is not None:
-            q += sql.SQL( "{where} q3c_radial_query(ra, dec, {ra}, {dec}, {radius})"
+            q += sql.SQL( "{where} q3c_radial_query(ra, dec, {ra}, {dec}, {radius})\n"
                          ).format( where=sql.SQL(where), ra=float(ra), dec=float(dec), radius=float(radius)/3600. )
 
+        if orderby is not None:
+            orderby = util.listify( orderby )
+            if len( orderby ) > 0:
+                q += sql.SQL( "ORDER BY" )
+                comma = " "
+                for sortcol in orderby:
+                    desc = sql.SQL( "" )
+                    if sortcol[0] == "-":
+                        desc = sql.SQL( " DESC" )
+                        sortcol = sortcol[1:]
+                    if sortcol not in searchspec.keys():
+                        # ... this is bobby tables protection
+                        raise ValueError( f"Unknown column to sort by: {sortcol}" )
+                    q += sql.SQL( "{comma}{field}{desc}" ).format( comma=sql.SQL(comma),
+                                                                   field=sql.Identifier(sortcol), desc=desc )
+                    comma = ", "
+                q += sql.SQL( "\n" )
+
+        if limit is not None:
+            limit = int( limit )
+            q += sql.SQL( "LIMIT {limit}\n" ).format( limit=sql.SQL(str(limit)) )
+        if offset is not None:
+            offset = int( offset )
+            q += sql.SQL( "OFFSET {offset}\n" ).format( offset=sql.SQL(str(offset)) )
+
         FDBLogger.debug( "Starting object search query..." )
         barf = "".join( random.choices( "abcdefghijklmnopqrstuvwxyz", k=6 ) )
         cursor = dbcon.execute_nofetch( q, subdict, cursorname=f'object_search_{barf}' )
-        cursor.itersize = 1000
+        cursor.itersize = 1000 if limit is None else min( 1000, limit )
         FDBLogger.debug( "...fetching results from postgres..." )
         cols = [ desc[0] for desc in cursor.description ]
 
@@ -1683,7 +1728,8 @@ def create_object_stats_materialized_view( procver ):
                              'lastdet_mjd', 'lastdet_flux', 'lastdet_fluxerr',
                              'maxdet_mjd', 'maxdet_flux', 'maxdet_fluxerr',
                              'lastforced_mjd', 'lastforced_flux', 'lastforced_fluxerr',
-                             'ndets', 'ndets24', 'ndets23', 'ndets22', 'ndets21', 'nsn10', 'nsn7', 'nsn5' }
+                             'ndets', 'ndets24', 'ndets23', 'ndets22', 'ndets21', 'nfrc',
+                             'nsn10', 'nsn7', 'nsn5' }
             if set( r['attname'] for r in rows ) != expectedcols:
                 raise RuntimeError( f"postgres view objstats_{procver} has the wrong set of columns" )
 
@@ -1935,7 +1981,7 @@ def create_object_stats_materialized_view( procver ):
                      ORDER BY o.rootid, f.visit, j.priority DESC
                   ) subq
                   GROUP BY rootid, band
-               ) nf ON r.rootid=n.rootid AND r.band=n.band
+               ) nf ON r.rootid=nf.rootid AND r.band=nf.band
             )
             """
         ) ).format( viewname=sql.Identifier( f'objstats_{procver}' ), pvid=pvid )
diff --git a/src/services/source_importer.py b/src/services/source_importer.py
index 4956149..a5b0a00 100644
--- a/src/services/source_importer.py
+++ b/src/services/source_importer.py
@@ -459,7 +459,7 @@ def import_objects( self, t0=None, t1=None, batchsize=10000, dbcon=None, commit=
                 #   root_diaobject forieng key in diaobject implicitly
                 #   grabs a ROW SHARE lock on root_diaobject.
                 FDBLogger.debug( "  ...starting root diaobject matching..." )
-                dbcon.execute( "LOCK TABLE root_diaobject IN SHARE ROW EXCLUSIVE MODE" )
+                dbcon.execute( "LOCK TABLE root_diaobject IN SHARE ROW EXCLUSIVE MODE", explain=False )
 
                 # Link new objects to existing root objects
                 # TODO : test this with multiple processing versions and multiple
diff --git a/src/webserver/static/fastdb.css b/src/webserver/static/fastdb.css
index f14f55d..d544fc8 100644
--- a/src/webserver/static/fastdb.css
+++ b/src/webserver/static/fastdb.css
@@ -204,6 +204,8 @@ a { color: var(--link-color); text-decoration: var(--link-decoration); cursor: p
 .hmargin { margin-left: 0.5ex; margin-right: 0.5ex }
 .xmargin { margin-left: 1ex; margin-right: 1ex }
 .mmargin { margin-left: 1em; margin-right: 1em }
+.xmarginleft { margin-left: 1ex }
+.mmarginleft { margin-left: 1em }
 .xmarginright { margin-right: 1ex }
 .mmarginright { margin-right: 1em }
 .xmargintop { margin-top: 1ex }
diff --git a/src/webserver/static/objectinfo.js b/src/webserver/static/objectinfo.js
index d9bfe4d..db22bcf 100644
--- a/src/webserver/static/objectinfo.js
+++ b/src/webserver/static/objectinfo.js
@@ -171,11 +171,11 @@ fastdbap.ObjectInfo = class
         tr = rkWebUtil.elemaker( "tr", table );
         td = rkWebUtil.elemaker( "td", tr, { "text": "RA:",
                                              "classes": [ "right", "xmarginright" ] } );
-        td = rkWebUtil.elemaker( "td", tr, { "text": this.data.objinfo.ra.toFixed(5) } );
+        td = rkWebUtil.elemaker( "td", tr, { "text": rkWebUtil.fixedOrEmpty(this.data.objinfo.ra, 5) } );
         tr = rkWebUtil.elemaker( "tr", table );
         td = rkWebUtil.elemaker( "td", tr, { "text": "Dec:",
                                              "classes": [ "right", "xmarginright" ] } );
-        td = rkWebUtil.elemaker( "td", tr, { "text": this.data.objinfo.dec.toFixed(5) } );
+        td = rkWebUtil.elemaker( "td", tr, { "text": rkWebUtil.fixedOrEmpty(this.data.objinfo.dec, 5) } );
         tr = rkWebUtil.elemaker( "tr", table );
         td = rkWebUtil.elemaker( "td", tr, { "text": "diaobjectid:",
                                              "classes": [ "right", "xmarginright" ] } );
@@ -194,11 +194,11 @@ fastdbap.ObjectInfo = class
             let dettext = "";
             if ( data.isdet[i] ) dettext = "Yes";
             let args = {
-                'mjd':        [ "td", tr, { "text": data.mjd[i].toFixed(4) } ],
+                'mjd':        [ "td", tr, { "text": rkWebUtil.fixedOrEmpty(data.mjd[i], 4) } ],
                 'band':       [ "td", tr, { "text": data.band[i] } ],
-                'flux':    [ "td", tr, { "text": data.flux[i].toExponential(4) } ],
-                'fluxerr': [ "td", tr, { "text": data.fluxerr[i].toExponential(4) } ],
-                's/n':        [ "td", tr, { "text": data['s/n'][i].toFixed(1) } ],
+                'flux':       [ "td", tr, { "text": rkWebUtil.exponentialOrEmpty(data.flux[i], 4) } ],
+                'fluxerr':    [ "td", tr, { "text": rkWebUtil.exponentialOrEmpty(data.fluxerr[i], 4) } ],
+                's/n':        [ "td", tr, { "text": rkWebUtil.fixedOrEmpty(data['s/n'][i], 1) } ],
                 'isdet':      [ "td", tr, { "text": dettext } ],
             };
             for ( let f of fields ) {
diff --git a/src/webserver/static/objectlist.js b/src/webserver/static/objectlist.js
index 5e9b95c..db32719 100644
--- a/src/webserver/static/objectlist.js
+++ b/src/webserver/static/objectlist.js
@@ -90,7 +90,7 @@ fastdbap.ObjectList = class
                        'magforcedlast',
                        'magforcedlasterr' ]
         let hdrs = { 'lastdet_mjd': 'mjd',
-                     'lastdet_fluxband': 'band',
+                     'lastdet_band': 'band',
                      'ndets': 'ndets',
                      'firstdet_mjd': 'mjd',
                      'firstdet_band': 'mjd',
@@ -121,29 +121,29 @@ fastdbap.ObjectList = class
                                                  self.show_object_info( data.rootid[i] );
                                                   }
                                                 } ],
-                'ra':               [ "td", tr, { "text": data.ra[i].toFixed(5) } ],
-                'dec':              [ "td", tr, { "text": data.dec[i].toFixed(5) } ],
+                'ra':               [ "td", tr, { "text": rkWebUtil.fixedOrEmpty(data.ra[i], 5) } ],
+                'dec':              [ "td", tr, { "text": rkWebUtil.fixedOrEmpty(data.dec[i], 5) } ],
                 'ndets':            [ "td", tr, { "text": data.ndets[i].toString() } ],
-                'firstdet_mjd':     [ "td", tr, { "text": data.firstdet_mjd[i].toFixed(2),
+                'firstdet_mjd':     [ "td", tr, { "text": rkWebUtil.fixedOrEmpty(data.firstdet_mjd[i], 2),
                                                   "classes": [ "borderleft" ] } ],
                 'firstdet_band':    [ "td", tr, { "text": data.lastdet_band[i] } ],
-                'magfirst':         [ "td", tr, { "text": data.maglast[i].toFixed(2) } ],
-                'magfirsterr':      [ "td", tr, { "text": data.maglasterr[i].toFixed(2) } ],
-                'lastdet_mjd':      [ "td", tr, { "text": data.lastdet_mjd[i].toFixed(2),
+                'magfirst':         [ "td", tr, { "text": rkWebUtil.fixedOrEmpty(data.maglast[i], 2) } ],
+                'magfirsterr':      [ "td", tr, { "text": rkWebUtil.fixedOrEmpty(data.maglasterr[i], 2) } ],
+                'lastdet_mjd':      [ "td", tr, { "text": rkWebUtil.fixedOrEmpty(data.lastdet_mjd[i], 2),
                                                   "classes": [ "borderleft" ] } ],
                 'lastdet_band':     [ "td", tr, { "text": data.lastdet_band[i] } ],
-                'maglast':          [ "td", tr, { "text": data.maglast[i].toFixed(2) } ],
-                'maglasterr':       [ "td", tr, { "text": data.maglasterr[i].toFixed(2) } ],
-                'maxdet_mjd':       [ "td", tr, { "text": data.maxdet_mjd[i].toFixed(2),
+                'maglast':          [ "td", tr, { "text": rkWebUtil.fixedOrEmpty(data.maglast[i], 2) } ],
+                'maglasterr':       [ "td", tr, { "text": rkWebUtil.fixedOrEmpty(data.maglasterr[i], 2) } ],
+                'maxdet_mjd':       [ "td", tr, { "text": rkWebUtil.fixedOrEmpty(data.maxdet_mjd[i], 2),
                                                   "classes": [ "borderleft" ] } ],
                 'maxdet_band':      [ "td", tr, { "text": data.maxdet_band[i] } ],
-                'magmax':           [ "td", tr, { "text": data.magmax[i].toFixed(2) } ],
-                'magmaxerr':        [ "td", tr, { "text": data.magmaxerr[i].toFixed(2) } ],
-                'lastforced_mjd':   [ "td", tr, { "text": data.lastforced_mjd[i].toFixed(2),
+                'magmax':           [ "td", tr, { "text": rkWebUtil.fixedOrEmpty(data.magmax[i], 2) } ],
+                'magmaxerr':        [ "td", tr, { "text": rkWebUtil.fixedOrEmpty(data.magmaxerr[i], 2) } ],
+                'lastforced_mjd':   [ "td", tr, { "text": rkWebUtil.fixedOrEmpty(data.lastforced_mjd[i], 2),
                                                   "classes": [ "borderleft" ] } ],
                 'lastforced_band':  [ "td", tr, { "text": data.lastforced_band[i] } ],
-                'magforcedlast':    [ "td", tr, { "text": data.magforcedlast[i].toFixed(2) } ],
-                'magforcedlasterr': [ "td", tr, { "text": data.magforcedlasterr[i].toFixed(2) } ]
+                'magforcedlast':    [ "td", tr, { "text": rkWebUtil.fixedOrEmpty(data.magforcedlast[i], 2) } ],
+                'magforcedlasterr': [ "td", tr, { "text": rkWebUtil.fixedOrEmpty(data.magforcedlasterr[i], 2) } ]
             }
             for ( let f of fields ) {
                 if ( ! args.hasOwnProperty( f ) ) {
diff --git a/src/webserver/static/objectsearch.js b/src/webserver/static/objectsearch.js
index b4e61ab..1579c78 100644
--- a/src/webserver/static/objectsearch.js
+++ b/src/webserver/static/objectsearch.js
@@ -17,7 +17,7 @@ fastdbap.ObjectSearch = class
     render_page()
     {
         let self = this;
-        let table, tr, td, div, hbox, vbox, p;
+        let table, tr, td, div, superdiv, subdiv,hbox, vbox, p;
 
         rkWebUtil.wipeDiv( this.topdiv );
 
@@ -32,15 +32,64 @@ fastdbap.ObjectSearch = class
         p = rkWebUtil.elemaker( "p", div );
         rkWebUtil.button( p, "Show Random Obj", (e) => { self.show_random_obj(); } );
 
+        // More general search
+
+        superdiv = rkWebUtil.elemaker( "div", this.topdiv, { "classes": [ "vbox" ] } );
+        div = rkWebUtil.elemaker( "div", superdiv, { "classes": [ "maxwcontent", "hbox" ] } );
+        subdiv = rkWebUtil.elemaker( "div", superdiv, { "classes": [ "maxwcontent", "hbox", "searchinner" ] } );
+        rkWebUtil.button( subdiv, "Search", (e) => { self.object_search() } );
+        rkWebUtil.elemaker( "span", subdiv, { "text": "Limit:", "classes": [ "mmarginleft" ] } );
+        this.searchlimit = rkWebUtil.elemaker( "input", subdiv, { "classes": [ "mmarginright" ],
+                                                                  "attributes": { "type": "number",
+                                                                                  "min": 10,
+                                                                                  "max": 1000,
+                                                                                  "value": 100,
+                                                                                  "sytle": "width: 4em" } } );
+        this.searchlimit.addEventListener( "blur", (e) => {
+            self.searchlimit.value = rkWebUtil.parseIntInRange( self.searchlimit.value, 10, 1000, 100 )
+        } );
+        rkWebUtil.elemaker( "text", subdiv, { "text": "Offset:" } );
+        this.searchoffset = rkWebUtil.elemaker( "input", subdiv, { "classes": [ "mmarginright" ],
+                                                                   "attributes": { "type": "number",
+                                                                                   "min": 0,
+                                                                                   "value": 0,
+                                                                                   "style": "width: 6em" } } );
+        rkWebUtil.elemaker( "text", subdiv, { "text": "Sort by:" } );
+        this.searchsort = rkWebUtil.elemaker( "select", subdiv );
+        let sorttext = { 'rootid': 'rootid',
+                         'ra/dec': 'SPECIAL-ra/dec',
+                         'dec/ra': 'SPECIAL-dec/ra',
+                         'mjd_firstdetection': 'firstdet_mjd',
+                         'mjd_lastdetection': 'lastdet_mjd',
+                         'mjd_maxdetection': 'maxdet_mjd',
+                         'mjd_lastforced': 'lastforced_mjd',
+                         'flux_firstdetection': 'firstdet_flux',
+                         'flux_lastdetection': 'lastdet_flux',
+                         'flux_maxdetection': 'maxdet_flux',
+                         'flux_lastforced': 'lastforced_flux',
+                         'n detections': 'ndets',
+                         'n forced phot points': 'nfrc',
+                         'n dets mag≤24': 'ndets24',
+                         'n dets mag≤23': 'ndets23',
+                         'n dets mag≤22': 'ndets22',
+                         'n dets mag≤21': 'ndets21',
+                         'n s/n ≥10': 'nsn10',
+                         'n s/n ≥7': 'nsn10',
+                         'n s/n ≥5': 'nsn10'
+                       }
+        for ( let sortdisplay in sorttext ) {
+            let sortkey = sorttext[ sortdisplay ];
+            let wid = rkWebUtil.elemaker( "option", this.searchsort, { "text": sortdisplay,
+                                                                       "attributes": { "value": sortkey } } );
+            if ( sortkey == "rootid" ) wid.setAttribute( "selected", 1 );
+        }
+
+
         // search by ra/dec
 
-        div = rkWebUtil.elemaker( "div", this.topdiv, { "classes": [ "maxwcontent", "hbox" ] } );
         vbox = rkWebUtil.elemaker( "div", div, { "classes": [ "vbox", "xmarginright", "searchinner" ] } );
         table = rkWebUtil.elemaker( "table", vbox, { "classes": [ "borderless" ] } );
         tr = rkWebUtil.elemaker( "tr", table );
-        td = rkWebUtil.elemaker( "td", tr );
-        rkWebUtil.button( td, "Search", (e) => { self.object_search() } );
-        tr = rkWebUtil.elemaker( "tr", table );
         td = rkWebUtil.elemaker( "td", tr, { "text": "RA:", "classes": [ "right" ] } );
         td = rkWebUtil.elemaker( "td", tr );
         this.ra_widget = rkWebUtil.elemaker( "input", td, { "attributes": { "size": 10 } } );
@@ -267,13 +316,13 @@ fastdbap.ObjectSearch = class
 
         for ( let maglim of [ 21, 22, 23, 24 ] ) {
             let val = this['mindetmaglt' + maglim + "_widget"].value.trim();
-            if ( val.length > 0 ) searchcriteria['ndets' + maglim] = val;
+            if ( val.length > 0 ) searchcriteria['ndets' + maglim + '_min'] = val;
         }
         for ( let snlim of [ 5, 7, 10 ] ) {
             let val = this['mindetsngt' + snlim + '_widget'].value.trim();
-            if ( val.length > 0 ) searchcriteria['nsn' + snlim] = val;
+            if ( val.length > 0 ) searchcriteria['nsn' + snlim + '_min'] = val;
         }
-        
+
         // ltcv.object_search doesn't have a search window
         // if ( this.window_mjd0_widget.value.trim().length > 0 )
         //     searchcriteria.window_t0 = this.window_mjd0_widget.value.trim();
@@ -281,7 +330,24 @@ fastdbap.ObjectSearch = class
         //     searchcriteria.window_t1 = this.window_mjd1_widget.value.trim();
         // if ( this.dets_in_window_widget.value.trim().length > 0 )
         //     searchcriteria.min_window_numdetections = this.dets_in_window_widget.value.trim();
+
+        // Sort, Limit, Offset
+
+        let sortby = this.searchsort.value;
+        if ( sortby == 'SPECIAL-ra/dec' ) {
+            sortby = [ 'ra', 'dec' ];
+        }
+        else if ( sortby == 'SPECIAL-/dec/ra' ) {
+            sortby = [ 'dec', 'ra' ];
+        }
+        if ( sortby != null ) {
+            searchcriteria['orderby'] = sortby;
+        }
+
+        searchcriteria['limit'] = this.searchlimit.value;
+        searchcriteria['offset'] = this.searchoffset.value;
         
+        // Do
         
         rkWebUtil.wipeDiv( this.context.objectlistdiv );
         this.context.maintabs.selectTab( "objectlist" );
diff --git a/tests/admin/test_load_edp2_parquet.py b/tests/admin/test_load_edp2_parquet.py
index ce59086..ee6b0cc 100644
--- a/tests/admin/test_load_edp2_parquet.py
+++ b/tests/admin/test_load_edp2_parquet.py
@@ -1,7 +1,14 @@
 import db
 
 
-def test_load_edp2_parquet( edp2_loaded_module ):
+# **********************************************************************
+# The test_user and edp2_materialized_view fixtures are not needed for
+#   the test, but because, like
+#   services/test_sourceimporter.py::test_full90days_fast, this is a
+#   convenient test for loading up a database for use developing the web
+#   ap.  See the developers documentation for FASTDB.
+#
+def test_load_edp2_parquet( edp2_loaded_module, test_user, edp2_materialized_view ):
     with db.DBCon( dictcursor=True ) as dbcon:
         rows = dbcon.execute( "SELECT rootid, diaobjectid FROM diaobject" )
         assert len(rows) == 100

From a206f4729d3ba56d54f4eba163c77190d04ee8cf Mon Sep 17 00:00:00 2001
From: Rob Knop <rknop@pobox.com>
Date: Mon, 21 Sep 2026 22:50:01 -0400
Subject: [PATCH 04/35] Fix a problem in objectinfo.js where it would crash if
 you tried to show an object that didn't have data in all the bands that the
 last object you showed did

---
 src/webserver/static/objectinfo.js | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/webserver/static/objectinfo.js b/src/webserver/static/objectinfo.js
index db22bcf..0f8c1f2 100644
--- a/src/webserver/static/objectinfo.js
+++ b/src/webserver/static/objectinfo.js
@@ -91,10 +91,15 @@ fastdbap.ObjectInfo = class
         this.allbands = this.allbands.concat( unknownbands );
         if ( fastdbap.ObjectInfo.shown_bands_cache == null ) {
             this.shownbands = [...this.allbands];
-            fastdbap.ObjectInfo.shown_bands_cache = [...this.shownbands];
         } else {
-            this.shownbands = [...fastdbap.ObjectInfo.shown_bands_cache];
+            this.shownbands = []
+            for ( let b of fastdbap.ObjectInfo.shown_bands_cache ) {
+                if ( this.allbands.includes( b ) ) {
+                    this.shownbands.push( b );
+                }
+            }
         }
+        fastdbap.ObjectInfo.shown_bands_cache = [...this.shownbands];
 
         this.datasets = {};
         for ( let b of this.allbands ) {

From 162a65097561794080985164c98529c0f2dc7776 Mon Sep 17 00:00:00 2001
From: Rob Knop <rknop@pobox.com>
Date: Tue, 22 Sep 2026 09:25:55 -0400
Subject: [PATCH 05/35] Add sort order (ascending vs. descending) to webap
 object search

---
 src/webserver/static/objectsearch.js | 44 +++++++++++++++++++---------
 1 file changed, 30 insertions(+), 14 deletions(-)

diff --git a/src/webserver/static/objectsearch.js b/src/webserver/static/objectsearch.js
index 1579c78..683ff0f 100644
--- a/src/webserver/static/objectsearch.js
+++ b/src/webserver/static/objectsearch.js
@@ -83,6 +83,10 @@ fastdbap.ObjectSearch = class
                                                                        "attributes": { "value": sortkey } } );
             if ( sortkey == "rootid" ) wid.setAttribute( "selected", 1 );
         }
+        this.sortorder = rkWebUtil.elemaker( "select", subdiv, { "classes": [ "xmarglinleft" ] } );
+        for ( let order of [ "ascending", "descending" ] ) {
+            rkWebUtil.elemaker( "option", this.sortorder, { "text":  order, "attributes": { "value": order } } );
+        }
 
 
         // search by ra/dec
@@ -137,10 +141,10 @@ fastdbap.ObjectSearch = class
                                                                "attributes": { "type": "checkbox",
                                                                                "checked": 1 } } );
         rkWebUtil.elemaker( "label", hbox, { "text": "Y ", "attributes": { "for": "y_statband_checkbox" } } );
-        
-        
+
+
         // search by first/max/last mjd/mag
-        
+
         vbox = rkWebUtil.elemaker( "div", div, { "classes": [ "vbox", "xmarginright", "searchinner" ] } );
         table = rkWebUtil.elemaker( "table", vbox, { "classes": [ "borderless"] } );
         tr = rkWebUtil.elemaker( "tr", table );
@@ -150,7 +154,7 @@ fastdbap.ObjectSearch = class
         td = rkWebUtil.elemaker( "th", tr );
         td = rkWebUtil.elemaker( "th", tr, { "text": "min" } );
         td = rkWebUtil.elemaker( "th", tr, { "text": "max" } );
-        
+
         tr = rkWebUtil.elemaker( "tr", table );
         td = rkWebUtil.elemaker( "td", tr, { "text": "First detection mjd", "classes": [ "right" ] } );
         td = rkWebUtil.elemaker( "td", tr );
@@ -228,8 +232,8 @@ fastdbap.ObjectSearch = class
         tr = rkWebUtil.elemaker( "tr", table );
         td = rkWebUtil.elemaker( "td", tr, { "text": "s/n ≥ 10", "classes": [ "right" ] } );
         this.mindetsngt10_widget = rkWebUtil.elemaker( "input", td, { "attributes": { "size": 4 } } );
-        
-        
+
+
         // Window... not currently supported by ltcv.py::object_search
 
         // vbox = rkWebUtil.elemaker( "div", div, { "classes": [ "vbox", "xmarginright", "searchinner" ] } );
@@ -271,7 +275,7 @@ fastdbap.ObjectSearch = class
         // If all are checked, don't include it as a criterion
         if ( statbands.length < 6 )
             searchcriteria.searchband = statbands;
-        
+
         if ( this.firstdetminmjd_widget.value.trim().length > 0 )
             searchcriteria.firstdet_mjd_min = this.firstdetminmjd_widget.value.trim();
         if ( this.firstdetmaxmjd_widget.value.trim().length > 0 )
@@ -282,7 +286,7 @@ fastdbap.ObjectSearch = class
         if ( this.firstdetmaxmag_widget.value.trim().length > 0 )
             searchcriteria.firstdet_flux_min = (
                 10 ** ( ( 31.4 - parseFloat(this.firstdetmaxmag_widget.value.trim()) ) / 2.5 ) );
-            
+
         if ( this.lastdetminmjd_widget.value.trim().length > 0 )
             searchcriteria.lastdet_mjd_min = this.lastdetminmjd_widget.value.trim();
         if ( this.lastdetmaxmjd_widget.value.trim().length > 0 )
@@ -293,7 +297,7 @@ fastdbap.ObjectSearch = class
         if ( this.lastdetmaxmag_widget.value.trim().length > 0 )
              searchcriteria.lastdet_flux_min = (
                 10 ** ( ( 31.4 - parseFloat(this.lastdetmaxmag_widget.value.trim()) ) / 2.5 ) );
-            
+
         if ( this.maxdetminmjd_widget.value.trim().length > 0 )
             searchcriteria.maxdet_mjd_min = this.maxdetminmjd_widget.value.trim();
         if ( this.maxdetmaxmjd_widget.value.trim().length > 0 )
@@ -333,22 +337,34 @@ fastdbap.ObjectSearch = class
 
         // Sort, Limit, Offset
 
+        let descending = false
+        if ( this.sortorder.value == 'descending' ) descending = true;
+
         let sortby = this.searchsort.value;
         if ( sortby == 'SPECIAL-ra/dec' ) {
-            sortby = [ 'ra', 'dec' ];
+            if ( descending )
+                sortby = [ '-ra', '-dec' ];
+            else
+                sortby = [ 'ra', 'dec' ];
         }
         else if ( sortby == 'SPECIAL-/dec/ra' ) {
-            sortby = [ 'dec', 'ra' ];
+            if ( descending )
+                sortby = [ '-dec', '-ra' ];
+            else
+                sortby = [ 'dec', 'ra' ];
         }
         if ( sortby != null ) {
-            searchcriteria['orderby'] = sortby;
+            if ( descending )
+                searchcriteria['orderby'] = "-" + sortby;
+            else
+                searchcriteria['orderby'] = sortby;
         }
 
         searchcriteria['limit'] = this.searchlimit.value;
         searchcriteria['offset'] = this.searchoffset.value;
-        
+
         // Do
-        
+
         rkWebUtil.wipeDiv( this.context.objectlistdiv );
         this.context.maintabs.selectTab( "objectlist" );
         rkWebUtil.elemaker( "p", this.context.objectlistdiv, { "text": "Searching for objects...",

From 313b1eb263b6189fc395a1bb296c6200ad26f051 Mon Sep 17 00:00:00 2001
From: Rob Knop <rknop@pobox.com>
Date: Tue, 22 Sep 2026 09:27:54 -0400
Subject: [PATCH 06/35] spelling counts

---
 src/webserver/static/objectsearch.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/webserver/static/objectsearch.js b/src/webserver/static/objectsearch.js
index 683ff0f..690ee1f 100644
--- a/src/webserver/static/objectsearch.js
+++ b/src/webserver/static/objectsearch.js
@@ -83,7 +83,7 @@ fastdbap.ObjectSearch = class
                                                                        "attributes": { "value": sortkey } } );
             if ( sortkey == "rootid" ) wid.setAttribute( "selected", 1 );
         }
-        this.sortorder = rkWebUtil.elemaker( "select", subdiv, { "classes": [ "xmarglinleft" ] } );
+        this.sortorder = rkWebUtil.elemaker( "select", subdiv, { "classes": [ "xmarginleft" ] } );
         for ( let order of [ "ascending", "descending" ] ) {
             rkWebUtil.elemaker( "option", this.sortorder, { "text":  order, "attributes": { "value": order } } );
         }

From 98d9472f4f06662547105b4e4432cfdd9710cabc Mon Sep 17 00:00:00 2001
From: Rob Knop <rknop@pobox.com>
Date: Tue, 22 Sep 2026 09:46:35 -0400
Subject: [PATCH 07/35] =?UTF-8?q?Add=20both=20lower=20and=20upper=20limits?=
 =?UTF-8?q?=20in=20webap=20object=20search=20for=20mag=E2=89=A4=20and=20sn?=
 =?UTF-8?q?=E2=89=A5?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 src/webserver/static/objectsearch.js | 29 ++++++++++++++++++++++++++--
 1 file changed, 27 insertions(+), 2 deletions(-)

diff --git a/src/webserver/static/objectsearch.js b/src/webserver/static/objectsearch.js
index 690ee1f..811581a 100644
--- a/src/webserver/static/objectsearch.js
+++ b/src/webserver/static/objectsearch.js
@@ -17,7 +17,7 @@ fastdbap.ObjectSearch = class
     render_page()
     {
         let self = this;
-        let table, tr, td, div, superdiv, subdiv,hbox, vbox, p;
+        let table, tr, td, div, superdiv, subdiv,hbox, vbox, p, span;
 
         rkWebUtil.wipeDiv( this.topdiv );
 
@@ -206,32 +206,53 @@ fastdbap.ObjectSearch = class
 
         vbox = rkWebUtil.elemaker( "div", div, { "classes": [ "vbox", "xmarginright", "searchinner" ] } );
         hbox = rkWebUtil.elemaker( "div", vbox, { "classes": [ "hbox", "bold" ],
-                                                  "text": "Min # detections with:" } );
+                                                  "text": "Detections with:" } );
         hbox = rkWebUtil.elemaker( "div", vbox, { "classes": [ "hbox" ] } );
         table = rkWebUtil.elemaker( "table", hbox, { "classes": [ "borderless", "mmarginright" ] } );
         tr = rkWebUtil.elemaker( "tr", table );
         td = rkWebUtil.elemaker( "td", tr, { "text": "mag ≤ 24", "classes": [ "right" ] } );
+        td = rkWebUtil.elemaker( "td", tr, { "text": "≥" } );
         this.mindetmaglt24_widget = rkWebUtil.elemaker( "input", td, { "attributes": { "size": 4 } } );
+        td = rkWebUtil.elemaker( "td", tr, { "text": "≤" } );
+        this.maxdetmaglt24_widget = rkWebUtil.elemaker( "input", td, { "attributes": { "size": 4 } } );
         tr = rkWebUtil.elemaker( "tr", table );
         td = rkWebUtil.elemaker( "td", tr, { "text": "mag ≤ 23", "classes": [ "right" ] } );
+        td = rkWebUtil.elemaker( "td", tr, { "text": "≥" } );
         this.mindetmaglt23_widget = rkWebUtil.elemaker( "input", td, { "attributes": { "size": 4 } } );
+        td = rkWebUtil.elemaker( "td", tr, { "text": "≤" } );
+        this.maxdetmaglt23_widget = rkWebUtil.elemaker( "input", td, { "attributes": { "size": 4 } } );
         tr = rkWebUtil.elemaker( "tr", table );
         td = rkWebUtil.elemaker( "td", tr, { "text": "mag ≤ 22", "classes": [ "right" ] } );
+        td = rkWebUtil.elemaker( "td", tr, { "text": "≥" } );
         this.mindetmaglt22_widget = rkWebUtil.elemaker( "input", td, { "attributes": { "size": 4 } } );
+        td = rkWebUtil.elemaker( "td", tr, { "text": "≤" } );
+        this.maxdetmaglt22_widget = rkWebUtil.elemaker( "input", td, { "attributes": { "size": 4 } } );
         tr = rkWebUtil.elemaker( "tr", table );
         td = rkWebUtil.elemaker( "td", tr, { "text": "mag ≤ 21", "classes": [ "right" ] } );
+        td = rkWebUtil.elemaker( "td", tr, { "text": "≥" } );
         this.mindetmaglt21_widget = rkWebUtil.elemaker( "input", td, { "attributes": { "size": 4 } } );
+        td = rkWebUtil.elemaker( "td", tr, { "text": "≤" } );
+        this.maxdetmaglt21_widget = rkWebUtil.elemaker( "input", td, { "attributes": { "size": 4 } } );
 
         table = rkWebUtil.elemaker( "table", hbox, { "classes": [ "borderless" ] } );
         tr = rkWebUtil.elemaker( "tr", table );
         td = rkWebUtil.elemaker( "td", tr, { "text": "s/n ≥ 5", "classes": [ "right" ] } );
+        td = rkWebUtil.elemaker( "td", tr, { "text": "≥" } );
         this.mindetsngt5_widget = rkWebUtil.elemaker( "input", td, { "attributes": { "size": 4 } } );
+        td = rkWebUtil.elemaker( "td", tr, { "text": "≤" } );
+        this.maxdetsngt5_widget = rkWebUtil.elemaker( "input", td, { "attributes": { "size": 4 } } );
         tr = rkWebUtil.elemaker( "tr", table );
         td = rkWebUtil.elemaker( "td", tr, { "text": "s/n ≥ 7", "classes": [ "right" ] } );
+        td = rkWebUtil.elemaker( "td", tr, { "text": "≥" } )
         this.mindetsngt7_widget = rkWebUtil.elemaker( "input", td, { "attributes": { "size": 4 } } );
+        td = rkWebUtil.elemaker( "td", tr, { "text": "≤" } )
+        this.maxdetsngt7_widget = rkWebUtil.elemaker( "input", td, { "attributes": { "size": 4 } } );
         tr = rkWebUtil.elemaker( "tr", table );
         td = rkWebUtil.elemaker( "td", tr, { "text": "s/n ≥ 10", "classes": [ "right" ] } );
+        td = rkWebUtil.elemaker( "td", tr, { "text": "≥" } )
         this.mindetsngt10_widget = rkWebUtil.elemaker( "input", td, { "attributes": { "size": 4 } } );
+        td = rkWebUtil.elemaker( "td", tr, { "text": "≤" } )
+        this.maxdetsngt10_widget = rkWebUtil.elemaker( "input", td, { "attributes": { "size": 4 } } );
 
 
         // Window... not currently supported by ltcv.py::object_search
@@ -321,10 +342,14 @@ fastdbap.ObjectSearch = class
         for ( let maglim of [ 21, 22, 23, 24 ] ) {
             let val = this['mindetmaglt' + maglim + "_widget"].value.trim();
             if ( val.length > 0 ) searchcriteria['ndets' + maglim + '_min'] = val;
+            val = this['maxdetmaglt' + maglim + "_widget"].value.trim()
+            if ( val.length > 0 ) searchcriteria['ndets' + maglim + '_max'] = val;
         }
         for ( let snlim of [ 5, 7, 10 ] ) {
             let val = this['mindetsngt' + snlim + '_widget'].value.trim();
             if ( val.length > 0 ) searchcriteria['nsn' + snlim + '_min'] = val;
+            val = this['maxdetsngt' + snlim + '_widget'].value.trim();
+            if ( val.length > 0 ) searchcriteria['nsn' + snlim + '_max'] = val;
         }
 
         // ltcv.object_search doesn't have a search window

From d8c74dad5decd2074799c0277ba569f67a8d3da0 Mon Sep 17 00:00:00 2001
From: "Rob Knop (Nersc)" <raknop@lbl.gov>
Date: Tue, 22 Sep 2026 07:34:29 -0700
Subject: [PATCH 08/35] Update server-side count things to use the objstatscomb
 materialized view

---
 extern/rkwebutil               |  2 +-
 src/webserver/baseview.py      |  3 +-
 src/webserver/server.py        | 80 +++++++++++-----------------------
 src/webserver/static/fastdb.js | 11 +++--
 4 files changed, 36 insertions(+), 60 deletions(-)

diff --git a/extern/rkwebutil b/extern/rkwebutil
index d0b2c6f..58000db 160000
--- a/extern/rkwebutil
+++ b/extern/rkwebutil
@@ -1 +1 @@
-Subproject commit d0b2c6f43c874048e7315f244072367d84d23846
+Subproject commit 58000db6072eda68e758f7e00a87e254cf3cde0d
diff --git a/src/webserver/baseview.py b/src/webserver/baseview.py
index 137f145..5edc0ae 100644
--- a/src/webserver/baseview.py
+++ b/src/webserver/baseview.py
@@ -123,8 +123,9 @@ def dispatch_request( self, *args, **kwargs ):
                     #   into doubles... thereby destroying 64-bit integers.  The fastdb
                     #   javascript code sets the Fastdb-Stringifyints header to tell us
                     #   to send integers back as strings so they won't get destroyed.
-                    FDBLogger.warning( "Stringifying integers" )
+                    FDBLogger.warning( "Stringifying integers..." )
                     retval = util.stringify_integers( retval )
+                    FDBLogger.warning( "...done stringifying integers." )
                 # Can't just use the default JSON handling, because it
                 #   writes out NaN which is not standard JSON and which
                 #   the javascript JSON parser chokes on.  simplejson
diff --git a/src/webserver/server.py b/src/webserver/server.py
index 9616177..c8aaaeb 100644
--- a/src/webserver/server.py
+++ b/src/webserver/server.py
@@ -1,6 +1,6 @@
+import re
 import copy
 import logging
-import textwrap
 
 from psycopg import sql
 import flask
@@ -125,7 +125,7 @@ class CountThings( BaseView ):
     def do_the_things( self, which, procver='default' ):
         global app
 
-        synonyms = { 'rootid': [ 'rootid', 'rootobject', 'rootdiaobject' ],
+        synonyms = { 'rootid': [ 'root', 'rootid', 'rootobject', 'rootdiaobject' ],
                      'diaobject': [ 'object', 'diaobject'],
                      'diasource': [ 'source', 'diasource' ],
                      'diaforcedsource': [ 'forced', 'forcedsource' ,'diaforcedsource' ]
@@ -138,71 +138,43 @@ def do_the_things( self, which, procver='default' ):
                 break
         if thingtocount is None:
             return f"Unknown thing to count: {which}", 422
+        if thingtocount == 'diaobject':
+            return "Counting diaobject not supported, count rootid", 422
+        if not re.search( r'^[A-Za-z0-9_\-]+$', procver ):
+            # Let's really make sure not to bobby tables
+            return f"Invalid processing version {procver}, can only contain A-Z, a-z, 0-9, _, and -", 422
 
         estimate = False
         if flask.request.is_json:
             data = flask.request.json
             estimate = ( 'estimate' in data ) and ( data['estimate'] )
+            if estimate:
+                return "Estimate not supported", 422
 
         with db.DBCon() as dbcon:
             try:
-                pvid = db.ProcessingVersion.procver_id( procver )
+                # Just make sure this is a know processing version, even though
+                #   we aren't actually going to use it.
+                # This is a necessary but not sufficient condition; the
+                #   objstatscomb materialized view for this processing
+                #   version must also exist.  TODO: think about whether
+                #   I want to interpret aliases for finding the
+                #   objstatscomb table.  Temptation: yes.
+                _pvid = db.ProcessingVersion.procver_id( procver )
             except Exception as ex:
                 raise FASTDBWebException( str(ex) )
 
-            if thingtocount in ( 'rootid', 'diaobject' ):
-                distinct = 'diaobjectid' if thingtocount=='diaobject' else 'rootid'
-                indexes = [ 'o idx_diaobject_procver' ]
-                baseq = sql.SQL( textwrap.dedent(
-                    """\
-                    SELECT DISTINCT ON({distinct}) {distinct} FROM diaobject o
-                    INNER JOIN base_procver_of_procver pv ON o.base_procver_id=pv.base_procver_id
-                                                         AND pv.procver_id={pvid}
-                    ORDER BY {distinct}, pv.priority DESC
-                    """
-                ) ).format( distinct=sql.Identifier("o", distinct), pvid=pvid )
-            else:
-                indexes = [ f's idx_{thingtocount}_base_procver_id', f'o idx_{thingtocount}_diaobjectid' ]
-                baseq = sql.SQL( textwrap.dedent(
-                    """\
-                    SELECT DISTINCT ON(o.rootid, s.visit) {idfield} FROM {table} s
-                    INNER JOIN base_procver_of_procver pv ON s.base_procver_id=pv.base_procver_id
-                                                         AND pv.procver_id={pvid}
-                    INNER JOIN diaobject o ON o.diaobjectid=s.diaobjectid
-                    ORDER BY o.rootid, s.visit, pv.priority DESC
-                    """
-                ) ).format( table=sql.Identifier(thingtocount),
-                            idfield=sql.Identifier("s", f"{thingtocount}id" ),
-                            pvid=pvid )
-
-            if estimate:
-                # THIS DOES A REALLY TERRIBLE JOB.
-                # cf: https://wiki.postgresql.org/wiki/Count_estimate
-                # TODO : figure out how accurate this count estimate really is.  I have
-                #    a suspicion that it's not very good when there are multiple
-                #    different processing versions.
-                FDBLogger.debug( f"Getting estimate of count of {which} for {pvid}" )
-                q = sql.SQL( "EXPLAIN (FORMAT JSON) {baseq}" ).format( baseq=baseq )
-                rows, _  = dbcon.execute( q, explain=False )
-                FDBLogger.debug( f"rows is {rows}" )
-                count = rows[0][0][0]['Plan']['Plan Rows']
+            table = sql.Identifier( f"objstatscomb_{procver}" )
 
+            if thingtocount == 'rootid':
+                rows, _cols = dbcon.execute( sql.SQL("SELECT COUNT(rootid) FROM {table}").format( table=table ) )
+            elif thingtocount == 'diasource':
+                rows, _cols = dbcon.execute( sql.SQL("SELECT SUM(ndets) FROM {table}").format( table=table ) )
+            elif thingtocount == 'diaforcedsource':
+                rows, _cols = dbcon.execute( sql.SQL("SELECT SUM(nfrc) FROM {table}").format( table=table ) )
             else:
-                # Gah.  I'm thrashing about a lot with these query
-                #   optimization thingies that I'm doing.  After bumping
-                #   the memory postgres had for buffers, the queries got
-                #   *slower*, for reasons I don't understand.  One thing
-                #   it did was assign fewer workers; the postgres query
-                #   optimizer is so strange.  So, I'm forcing parallel
-                #   workers, to make it faster.  For this count query,
-                #   this is probably not going to be a problem, but it
-                #   is of course scary.
-                q = sql.SQL( "/*+ " )
-                q += sql.SQL(" ").join( sql.SQL( f"IndexScan({i})" ) for i in indexes )
-                q += sql.SQL( " Parallel(t 4) */\n" )
-                q += sql.SQL( "SELECT COUNT(*) FROM (\n{baseq}\n) subq" ).format( baseq=baseq )
-                rows, _ = dbcon.execute( q )
-                count = rows[0][0]
+                return "This should never happen", 422
+            count = rows[0][0]
 
             return { 'status': 'ok',
                      'table': thingtocount,
diff --git a/src/webserver/static/fastdb.js b/src/webserver/static/fastdb.js
index b1f2b6c..d936205 100644
--- a/src/webserver/static/fastdb.js
+++ b/src/webserver/static/fastdb.js
@@ -133,20 +133,23 @@ fastdbap.Context = class
         this.sources_span.innerHTML = "(loading...) sources";
         this.forced_span.innerHTML = "(loading...) forced";
 
-        this.connector.sendHttpRequest( "count/object/" + encodeURIComponent( pv ),
+        this.connector.sendHttpRequest( "count/root/" + encodeURIComponent( pv ),
                                         { 'estimate': false },
                                         (data) => {
-                                            self.objects_span.innerHTML = data.count.toString() + " objects";
+                                            self.objects_span.innerHTML =
+                                                rkWebUtil.commafyInt(data.count) + " objects";
                                         } );
         this.connector.sendHttpRequest( "count/source/" + encodeURIComponent( pv ),
                                         { 'estimate': false },
                                         (data) => {
-                                            self.sources_span.innerHTML = data.count.toString() + " sources";
+                                            self.sources_span.innerHTML =
+                                                rkWebUtil.commafyInt(data.count) + " sources";
                                         } );
         this.connector.sendHttpRequest( "count/forced/" + encodeURIComponent( pv ),
                                         { 'estimate': false },
                                         (data) => {
-                                            self.forced_span.innerHTML = data.count.toString() + " forced";
+                                            self.forced_span.innerHTML =
+                                                rkWebUtil.commafyInt(data.count) + " forced";
                                         } );
     }
 

From 7f6ab59b05ae24eee813c46394976026d717ef76 Mon Sep 17 00:00:00 2001
From: Rob Knop <raknop@lbl.gov>
Date: Tue, 22 Sep 2026 10:49:09 -0700
Subject: [PATCH 09/35] Update postgresql.conf to reduce random_page_cost; add
 lsdb back to Dockerfile (can now just pip install it); make teh test_user and
 fastdb_client test fixtures module scope, not session, as session scope was
 breaking db tests.

---
 docker/postgres/postgresql.conf |  5 +++++
 docker/webserver/Dockerfile     |  1 +
 tests/conftest.py               |  4 ++--
 tests/db/test_authuser.py       | 12 ++++++------
 4 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/docker/postgres/postgresql.conf b/docker/postgres/postgresql.conf
index 793f6a4..7cd843b 100644
--- a/docker/postgres/postgresql.conf
+++ b/docker/postgres/postgresql.conf
@@ -436,6 +436,11 @@ min_wal_size = 80MB
 
 #seq_page_cost = 1.0			# measured on an arbitrary scale
 #random_page_cost = 4.0			# same scale as above
+# RKNOP 2026-09-22 reducing this to 1.1  Backend drives
+#   should be SDDs, which do way better with random access
+#   than HDDs did.  We don't want this factor to overly
+#   de-emphasize use of indexes.
+random_page_cost = 1.1
 #cpu_tuple_cost = 0.01			# same scale as above
 #cpu_index_tuple_cost = 0.005		# same scale as above
 #cpu_operator_cost = 0.0025		# same scale as above
diff --git a/docker/webserver/Dockerfile b/docker/webserver/Dockerfile
index 33dff67..1ea82e5 100644
--- a/docker/webserver/Dockerfile
+++ b/docker/webserver/Dockerfile
@@ -106,6 +106,7 @@ RUN source /venv/bin/activate && \
        "fastavro>=1.10.0,<2.0.0" \
        "fink-client>=10.1,<11.0" \
        "healpy>=1.20.0,<2.0.0" \
+       "lsdb>=0.11.0,<1.0.0" \
        "nested-pandas>=0.7.1,<0.8.0" \
        "numpy>=2.5.1,<3.0.0" \
        "pandas>=3.0.5,<4.0.0" \
diff --git a/tests/conftest.py b/tests/conftest.py
index 08a6edf..35c51da 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -869,7 +869,7 @@ def pos_offset( bpvkey, mjd ):
             con.commit()
 
 
-@pytest.fixture( scope='session' )
+@pytest.fixture( scope='module' )
 def test_user():
     # Test user with password 'test_password'
     user = AuthUser( id=asUUID('788e391e-ca63-4057-8788-25cc8647e722'),
@@ -902,7 +902,7 @@ def test_user():
     user.delete_from_db()
 
 
-@pytest.fixture( scope='session' )
+@pytest.fixture( scope='module' )
 def fastdb_client( test_user ):
     return FASTDBClient( 'http://webap:8080', username="test", password="test_password", verify=False, debug=True,
                          retrysleep=0.1, retries=2 )
diff --git a/tests/db/test_authuser.py b/tests/db/test_authuser.py
index fc5691e..af42d08 100644
--- a/tests/db/test_authuser.py
+++ b/tests/db/test_authuser.py
@@ -15,22 +15,22 @@ def basetest_setup( self ):
         self.safe_to_modify = [ 'displayname', 'email', 'pubkey', 'privkey' ]
         self.uniques = [ 'username' ]
         self.obj1 = AuthUser( id=uuid.uuid4(),
-                              username='test',
-                              displayname='test user',
+                              username='test_authuser',
+                              displayname='test authuser',
                               email='test@nowhere.org',
                               pubkey='',
                               privkey={} )
         self.dict1 = { k: getattr( self.obj1, k ) for k in self.columns }
         self.obj2 = AuthUser( id=uuid.uuid4(),
-                              username='test2',
-                              displayname='test user 2',
+                              username='test_authuser2',
+                              displayname='test authseruser 2',
                               email='test2@nowhere.org',
                               pubkey='',
                               privkey={} )
         self.dict2 = { k: getattr( self.obj2, k ) for k in self.columns }
         self.dict3 = { 'id': uuid.uuid4(),
-                       'username': 'test3',
-                       'displayname': 'test user 3',
+                       'username': 'test_authser3',
+                       'displayname': 'test authuseruser 3',
                        'email': 'test3@nowhere.org',
                        'pubkey': 'blah',
                        'privkey': { 'blah': 'blah' } }

From 2a1ac4cc365f8f721b94ad9a63abfd2083ee92d5 Mon Sep 17 00:00:00 2001
From: Rob Knop <raknop@lbl.gov>
Date: Tue, 22 Sep 2026 11:38:48 -0700
Subject: [PATCH 10/35] Some query formatting for log file readability.

---
 src/db.py          | 12 ++++++------
 tests/conftest.py  |  2 ++
 tests/test_db.py   | 26 +++++++++++++-------------
 tests/test_ltcv.py |  4 ++--
 4 files changed, 23 insertions(+), 21 deletions(-)

diff --git a/src/db.py b/src/db.py
index 01abdf3..4d4c62d 100644
--- a/src/db.py
+++ b/src/db.py
@@ -594,13 +594,13 @@ def construct_pgsql_where_clause( searchspec, where="WHERE", **kwargs ):
 
         lege = ">=" if minmax == "min" else "<="
 
-        q += sql.SQL( "{where} {field}-{other}{op}%({kw})s" ).format( where=sql.SQL(where),
-                                                                      field=sql.Identifier(field),
-                                                                      other=sql.Identifier(other),
-                                                                      op=sql.SQL(lege),
-                                                                      kw=sql.SQL(kw) )
+        q += sql.SQL( "{where} {field}-{other}{op}%({kw})s\n" ).format( where=sql.SQL(where),
+                                                                        field=sql.Identifier(field),
+                                                                        other=sql.Identifier(other),
+                                                                        op=sql.SQL(lege),
+                                                                        kw=sql.SQL(kw) )
         subdict[kw] = kwargs[kw]
-        where = " AND"
+        where = "  AND"
         yanks.add( kw )
 
     for yank in yanks:
diff --git a/tests/conftest.py b/tests/conftest.py
index 35c51da..4137842 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1499,6 +1499,7 @@ def do_the_things( procver, band=None ):
                 if band is None:
                     thisexp[ f'{n}_band'] = [ None, None, None ]
             thisexp[ 'ndets' ] = [ 0, None, None ]
+            thisexp[ 'nfrc' ] = [ 0, None, None ]
 
             for mag in [ 21, 22, 23, 24 ]:
                 thisexp[ f'ndets{mag}' ] = [ 0, None, None ]
@@ -1572,6 +1573,7 @@ def do_the_things( procver, band=None ):
                         # already have a higher prio bpv
                         continue
                     seenvisits.add( visit )
+                    thisexp['nfrc'][0] += 1
                     if frc.midpointmjdtai > thisexp['lastforced_mjd'][0]:
                         thisexp['lastforced_mjd'][0] = frc.midpointmjdtai
                         if band is None:
diff --git a/tests/test_db.py b/tests/test_db.py
index 6d46f02..6899fda 100644
--- a/tests/test_db.py
+++ b/tests/test_db.py
@@ -13,64 +13,64 @@ def test_construct_sql_where_clause():
 
     q, subdict, missing, _where = construct_pgsql_where_clause( searchspec, just="A" )
     assert missing == set()
-    assert q.as_string() == 'WHERE "just"=%(just)s'
+    assert q.as_string() == 'WHERE "just"=%(just)s\n'
     assert subdict == { 'just': 'A' }
 
     q, subdict, missing, _where = construct_pgsql_where_clause( searchspec, just="A", mult="B", does_not_exist=42 )
     assert missing == set( [ 'does_not_exist' ] )
-    assert q.as_string() == 'WHERE "just"=%(just)s AND "mult"=%(mult)s'
+    assert q.as_string() == 'WHERE "just"=%(just)s\n  AND "mult"=%(mult)s\n'
     assert subdict == { 'just': 'A', 'mult': 'B' }
 
     q, subdict, missing, _where = construct_pgsql_where_clause( searchspec, just="A", mult=[ "B", "C" ] )
     assert missing == set()
-    assert q.as_string() == 'WHERE "just"=%(just)s AND "mult"=ANY(%(mult)s)'
+    assert q.as_string() == 'WHERE "just"=%(just)s\n  AND "mult"=ANY(%(mult)s)\n'
     assert subdict == { 'just': 'A', 'mult': [ 'B', 'C' ] }
 
     q, subdict, missing, _where = construct_pgsql_where_clause( searchspec, just="A", mult=( "B", "C" ) )
     assert missing == set()
-    assert q.as_string() == 'WHERE "just"=%(just)s AND "mult"=ANY(%(mult)s)'
+    assert q.as_string() == 'WHERE "just"=%(just)s\n  AND "mult"=ANY(%(mult)s)\n'
     assert subdict == { 'just': 'A', 'mult': [ 'B', 'C' ] }
 
     q, subdict, missing, _where = construct_pgsql_where_clause( searchspec, substr="B", substr_contains="C" )
     assert missing == set()
-    assert q.as_string() == 'WHERE "substr"=%(substr)s AND "substr" LIKE %(substr_contains)s'
+    assert q.as_string() == 'WHERE "substr"=%(substr)s\n  AND "substr" LIKE %(substr_contains)s\n'
     assert subdict == { 'substr': 'B', 'substr_contains': '%C%' }
 
     q, subdict, missing, _where = construct_pgsql_where_clause( searchspec, multsubstr="C", multsubstr_contains="D" )
     assert missing == set()
-    assert q.as_string() == 'WHERE "multsubstr"=%(multsubstr)s AND "multsubstr" LIKE %(multsubstr_contains)s'
+    assert q.as_string() == 'WHERE "multsubstr"=%(multsubstr)s\n  AND "multsubstr" LIKE %(multsubstr_contains)s\n'
     assert subdict == { 'multsubstr': "C", 'multsubstr_contains': '%D%' }
 
     q, subdict, missing, _where = construct_pgsql_where_clause( searchspec, multsubstr="C",
                                                                 multsubstr_contains=[ "D", "E" ] )
     assert missing == set()
-    assert q.as_string() == ( 'WHERE "multsubstr"=%(multsubstr)s AND '
+    assert q.as_string() == ( 'WHERE "multsubstr"=%(multsubstr)s\n  AND '
                               '("multsubstr" LIKE %(multsubstr_contains_0)s OR '
-                              '"multsubstr" LIKE %(multsubstr_contains_1)s)' )
+                              '"multsubstr" LIKE %(multsubstr_contains_1)s)\n' )
     assert subdict == { 'multsubstr': "C", 'multsubstr_contains_0': '%D%', 'multsubstr_contains_1': "%E%" }
 
 
     q, subdict, missing, _where = construct_pgsql_where_clause( searchspec, minmax_min=5 )
     assert missing == set()
-    assert q.as_string() == 'WHERE "minmax">=%(minmax_min)s'
+    assert q.as_string() == 'WHERE "minmax">=%(minmax_min)s\n'
     assert subdict == { 'minmax_min': 5 }
 
     q, subdict, missing, _where = construct_pgsql_where_clause( searchspec, minmax_min=5, minmax_max=10 )
     assert missing == set()
-    assert q.as_string() == 'WHERE "minmax">=%(minmax_min)s AND "minmax"<=%(minmax_max)s'
+    assert q.as_string() == 'WHERE "minmax">=%(minmax_min)s\n  AND "minmax"<=%(minmax_max)s\n'
     assert subdict == { 'minmax_min': 5, 'minmax_max': 10 }
 
 
     q, subdict, missing, _where = construct_pgsql_where_clause( searchspec, minmax_minus_minmax2_min=1 )
     assert missing == set()
-    assert q.as_string() == 'WHERE "minmax"-"minmax2">=%(minmax_minus_minmax2_min)s'
+    assert q.as_string() == 'WHERE "minmax"-"minmax2">=%(minmax_minus_minmax2_min)s\n'
     assert subdict == { 'minmax_minus_minmax2_min': 1 }
 
     q, subdict, missing, _where = construct_pgsql_where_clause( searchspec, minmax_minus_minmax2_min=1,
                                                                 minmax_minus_minmax2_max=2 )
     assert missing == set()
-    assert q.as_string() == ( 'WHERE "minmax"-"minmax2">=%(minmax_minus_minmax2_min)s '
-                              'AND "minmax"-"minmax2"<=%(minmax_minus_minmax2_max)s' )
+    assert q.as_string() == ( 'WHERE "minmax"-"minmax2">=%(minmax_minus_minmax2_min)s\n  '
+                              'AND "minmax"-"minmax2"<=%(minmax_minus_minmax2_max)s\n' )
     assert subdict == { 'minmax_minus_minmax2_min': 1, 'minmax_minus_minmax2_max': 2 }
 
 
diff --git a/tests/test_ltcv.py b/tests/test_ltcv.py
index 08fa80b..a75984f 100644
--- a/tests/test_ltcv.py
+++ b/tests/test_ltcv.py
@@ -18,7 +18,7 @@ def objstats_cols():
              'lastdet_mjd', 'lastdet_flux', 'lastdet_fluxerr',
              'maxdet_mjd', 'maxdet_flux', 'maxdet_fluxerr',
              'lastforced_mjd', 'lastforced_flux', 'lastforced_fluxerr',
-             'ndets', 'ndets24', 'ndets23', 'ndets22', 'ndets21',
+             'ndets', 'ndets24', 'ndets23', 'ndets22', 'ndets21', 'nfrc',
              'nsn10', 'nsn7', 'nsn5' }
 
 
@@ -285,7 +285,7 @@ def test_get_object_infos( set_of_lightcurves, procver_collection ):
                         for col in [ 'diaobjectid', 'obj_base_procver', 'pos_base_procver',
                                      'diaobject_ra', 'diaobject_dec', 'diaobject_raerr', 'diaobject_decerr',
                                      'diaobject_ra_dec_cov' ] )
-            assert info.loc[rid, 'diaobjectid'] == expobj
+            assert set( info.loc[rid, 'diaobjectid'] ) == set( expobj )
             assert set( info.loc[rid, 'obj_base_procver'] ) == { bpvs['realtime_diaobject'].id,
                                                                  bpvs['bpv2_diaobject'].id }
 

From 6b63a6d5f943b8feffa988024a0cb1837d508734 Mon Sep 17 00:00:00 2001
From: "Rob Knop (Nersc)" <raknop@lbl.gov>
Date: Tue, 22 Sep 2026 11:54:17 -0700
Subject: [PATCH 11/35] Fix migration that adds procver_id to tables, start
 (but don't finish) futzing with materialized view query.

---
 admin/spin/production/postgres.yaml              |  2 +-
 ...sabled => 2026-09-21_001_procverintables.sql} | 16 ++++++++--------
 src/ltcv.py                                      | 10 ++--------
 3 files changed, 11 insertions(+), 17 deletions(-)
 rename db/{2026-09-21_001_procverintables.sql_disabled => 2026-09-21_001_procverintables.sql} (70%)

diff --git a/admin/spin/production/postgres.yaml b/admin/spin/production/postgres.yaml
index c6de6ce..477e917 100644
--- a/admin/spin/production/postgres.yaml
+++ b/admin/spin/production/postgres.yaml
@@ -43,7 +43,7 @@ spec:
       - env:
         - name: POSTGRES_DATA_DIR
           value: /var/lib/postgresql/data
-        image: registry.nersc.gov/m1727/raknop/fastdb-postgres:20260331a
+        image: registry.nersc.gov/m1727/raknop/fastdb-postgres:20260922a
         imagePullPolicy: IfNotPresent
         name: postgres
         resources: {}
diff --git a/db/2026-09-21_001_procverintables.sql_disabled b/db/2026-09-21_001_procverintables.sql
similarity index 70%
rename from db/2026-09-21_001_procverintables.sql_disabled
rename to db/2026-09-21_001_procverintables.sql
index f0b1e65..cb5b323 100644
--- a/db/2026-09-21_001_procverintables.sql_disabled
+++ b/db/2026-09-21_001_procverintables.sql
@@ -5,35 +5,35 @@ ALTER TABLE diaforcedsource ADD COLUMN procver_id uuid;
 
 UPDATE diaobject o SET procver_id=q.procver_id
 FROM (
- SELECT o.diaobjectid, j.procver_id
+ SELECT o.diaobjectid, o.base_procver_id, j.procver_id
  FROM diaobject o
  INNER JOIN base_procver_of_procver j ON o.base_procver_id=j.base_procver_id
 ) q
-WHERE q.diaobjectid=o.diaobjectid;
+WHERE q.diaobjectid=o.diaobjectid AND q.base_procver_id=o.base_procver_id;
 
 UPDATE diaobject_position p SET procver_id=q.procver_id
 FROM (
- SELECT p.diaobjectid, j.procver_id
+ SELECT p.base_procver_id, p.diaobjectid, j.procver_id
  FROM diaobject_position p
  INNER JOIN base_procver_of_procver j ON p.base_procver_id=j.base_procver_id
 ) q
-WHERE q.diaobjectid=p.diaobjectid;
+WHERE q.base_procver_id=p.base_procver_id AND q.diaobjectid=p.diaobjectid;
 
 UPDATE diasource s SET procver_id=q.procver_id
 FROM (
- SELECT s.diasourceid, j.procver_id
+ SELECT s.base_procver_id, s.diasourceid, j.procver_id
  FROM diasource s
  INNER JOIN base_procver_of_procver j ON s.base_procver_id=j.base_procver_id
 ) q
-WHERE q.diasourceid=s.diasourceid;
+WHERE  q.base_procver_id=s.base_procver_id AND q.diasourceid=s.diasourceid;
 
 UPDATE diaforcedsource f SET procver_id=q.procver_id
 FROM (
- SELECT f.diaforcedsourceid, j.procver_id
+ SELECT f.base_procver_id, f.diaobjectid, f.visit, j.procver_id
  FROM diaforcedsource f
  INNER JOIN base_procver_of_procver j ON f.base_procver_id=j.base_procver_id
 ) q
-WHERE q.diaforcedsourceid=f.diaforcedsourceid;
+WHERE q.base_procver_id=f.base_procver_id AND q.diaobjectid=f.diaobjectid AND q.visit=f.visit;
 
 ALTER INDEX idx_diaobject_base_procver_id RENAME TO idx_diaobject_position_base_procver_id;
 ALTER INDEX idx_diaobject_procver RENAME TO idx_diaobject_base_procver_id;
diff --git a/src/ltcv.py b/src/ltcv.py
index 716275d..07d3e49 100644
--- a/src/ltcv.py
+++ b/src/ltcv.py
@@ -1779,10 +1779,6 @@ def create_object_stats_materialized_view( procver ):
 
         q = sql.SQL( textwrap.dedent(
             """\
-            /*+ IndexScan (s idx_diasource_base_procver_id)
-                IndexScan (f idx_diaforcedsource_base_procver_id)
-                IndexScan (o idx_diaobject_diaobjectid idx_diaobject_procver)
-            */
             CREATE MATERIALIZED VIEW {viewname} AS (
                SELECT r.rootid, r.ra, r.dec, r.band,
                    d0.midpointmjdtai AS firstdet_mjd, d0.psfflux AS firstdet_flux, d0.psffluxerr AS firstdet_fluxerr,
@@ -1804,16 +1800,14 @@ def create_object_stats_materialized_view( procver ):
                    FROM root_diaobject r
                    INNER JOIN diaobject o ON o.rootid=r.id
                    INNER JOIN diasource s ON s.diaobjectid=o.diaobjectid
-                   INNER JOIN base_procver_of_procver j ON s.base_procver_id=j.base_procver_id
-                                                       AND j.procver_id={pvid}
+                   WHERE s.procver_id={pvid}
                  )
                  UNION
                  ( SELECT DISTINCT ON(o.rootid, f.band) o.rootid, r.ra, r.dec, f.band
                    FROM root_diaobject r
                    INNER JOIN diaobject o ON o.rootid=r.id
                    INNER JOIN diaforcedsource f ON f.diaobjectid=o.diaobjectid
-                   INNER JOIN base_procver_of_procver j ON f.base_procver_id=j.base_procver_id
-                                                       AND j.procver_id={pvid}
+                   WHERE f.procver_id={pvid}
                  )
                ) r
                LEFT JOIN (

From c0eb131123c65bdfe77eb22d47312d924397ef8f Mon Sep 17 00:00:00 2001
From: Rob Knop <raknop@lbl.gov>
Date: Tue, 22 Sep 2026 12:03:48 -0700
Subject: [PATCH 12/35] Remove temp file size limit from postgresql.conf

---
 docker/postgres/postgresql.conf | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker/postgres/postgresql.conf b/docker/postgres/postgresql.conf
index 7cd843b..5ffa8c0 100644
--- a/docker/postgres/postgresql.conf
+++ b/docker/postgres/postgresql.conf
@@ -182,8 +182,8 @@ dynamic_shared_memory_type = posix	# the default is usually the first option
 
 # - Disk -
 
-# Added RKNOP 2022-08-22
-temp_file_limit = 1073741824            # 1 TB temp file limit
+# Added RKNOP 2022-08-22 ... removed 2026-09-22
+# temp_file_limit = 1073741824            # 1 TB temp file limit
 #temp_file_limit = -1			# limits per-process temp file space
 					# in kilobytes, or -1 for no limit
 

From 3e59df20a0df3cbd1fe5054eaf464a8978225c55 Mon Sep 17 00:00:00 2001
From: "Rob Knop (Nersc)" <raknop@lbl.gov>
Date: Tue, 22 Sep 2026 12:04:18 -0700
Subject: [PATCH 13/35] ...re-disable the query adding procver_id to tables
 (realized I wasn't going to be able to use it enough to make things more
 efficient to make it anywhere near worth the database normalization
 violation), bump postgres docker image version.

---
 admin/spin/production/postgres.yaml                             | 2 +-
 ...intables.sql => 2026-09-21_001_procverintables.sql_disabled} | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename db/{2026-09-21_001_procverintables.sql => 2026-09-21_001_procverintables.sql_disabled} (100%)

diff --git a/admin/spin/production/postgres.yaml b/admin/spin/production/postgres.yaml
index 477e917..cd7603f 100644
--- a/admin/spin/production/postgres.yaml
+++ b/admin/spin/production/postgres.yaml
@@ -43,7 +43,7 @@ spec:
       - env:
         - name: POSTGRES_DATA_DIR
           value: /var/lib/postgresql/data
-        image: registry.nersc.gov/m1727/raknop/fastdb-postgres:20260922a
+        image: registry.nersc.gov/m1727/raknop/fastdb-postgres:20260922b
         imagePullPolicy: IfNotPresent
         name: postgres
         resources: {}
diff --git a/db/2026-09-21_001_procverintables.sql b/db/2026-09-21_001_procverintables.sql_disabled
similarity index 100%
rename from db/2026-09-21_001_procverintables.sql
rename to db/2026-09-21_001_procverintables.sql_disabled

From 2c28543e146407bd0d06b5c5a33d561d61f81069 Mon Sep 17 00:00:00 2001
From: Rob Knop <rknop@pobox.com>
Date: Tue, 22 Sep 2026 16:22:44 -0400
Subject: [PATCH 14/35] futzing with materalized view query

---
 src/ltcv.py | 64 +++++++++++++++++++++++++++--------------------------
 1 file changed, 33 insertions(+), 31 deletions(-)

diff --git a/src/ltcv.py b/src/ltcv.py
index 07d3e49..0e0c234 100644
--- a/src/ltcv.py
+++ b/src/ltcv.py
@@ -1800,14 +1800,16 @@ def create_object_stats_materialized_view( procver ):
                    FROM root_diaobject r
                    INNER JOIN diaobject o ON o.rootid=r.id
                    INNER JOIN diasource s ON s.diaobjectid=o.diaobjectid
-                   WHERE s.procver_id={pvid}
+                   INNER JOIN base_procver_of_procver j ON s.base_procver_id=j.base_procver_id
+                                                       AND j.procver_id={pvid}
                  )
                  UNION
                  ( SELECT DISTINCT ON(o.rootid, f.band) o.rootid, r.ra, r.dec, f.band
                    FROM root_diaobject r
                    INNER JOIN diaobject o ON o.rootid=r.id
                    INNER JOIN diaforcedsource f ON f.diaobjectid=o.diaobjectid
-                   WHERE f.procver_id={pvid}
+                   INNER JOIN base_procver_of_procver j ON f.base_procver_id=j.base_procver_id
+                                                       AND j.procver_id={pvid}
                  )
                ) r
                LEFT JOIN (
@@ -1862,9 +1864,9 @@ def create_object_stats_materialized_view( procver ):
                  ORDER BY rootid, band, midpointmjdtai DESC
                ) fn ON fn.rootid=r.rootid AND fn.band=r.band
                LEFT JOIN (
-                  SELECT rootid, band, COUNT(diasourceid) AS ndets
+                  SELECT rootid, band, COUNT(*) AS ndets
                   FROM (
-                     SELECT DISTINCT ON(o.rootid, s.visit) o.rootid, s.band, s.diasourceid
+                     SELECT DISTINCT ON(o.rootid, s.visit) o.rootid, s.band
                      FROM diasource s
                      INNER JOIN diaobject o ON s.diaobjectid=o.diaobjectid
                      INNER JOIN base_procver_of_procver j ON s.base_procver_id=j.base_procver_id
@@ -1874,9 +1876,9 @@ def create_object_stats_materialized_view( procver ):
                   GROUP BY rootid, band
                ) n ON r.rootid=n.rootid AND r.band=n.band
                LEFT JOIN (
-                  SELECT rootid, band, COUNT(diasourceid) AS ndets
+                  SELECT rootid, band, COUNT(*) AS ndets
                   FROM (
-                     SELECT DISTINCT ON(o.rootid, s.visit) o.rootid, s.band, s.diasourceid, s.psfflux
+                     SELECT DISTINCT ON(o.rootid, s.visit) o.rootid, s.band, s.psfflux
                      FROM diasource s
                      INNER JOIN diaobject o ON s.diaobjectid=o.diaobjectid
                      INNER JOIN base_procver_of_procver j ON s.base_procver_id=j.base_procver_id
@@ -1887,9 +1889,9 @@ def create_object_stats_materialized_view( procver ):
                   GROUP BY rootid, band
                ) n24 ON r.rootid=n24.rootid AND r.band=n24.band
                LEFT JOIN (
-                  SELECT rootid, band, COUNT(diasourceid) AS ndets
+                  SELECT rootid, band, COUNT(*) AS ndets
                   FROM (
-                     SELECT DISTINCT ON(o.rootid, s.visit) o.rootid, s.band, s.diasourceid, s.psfflux
+                     SELECT DISTINCT ON(o.rootid, s.visit) o.rootid, s.band, s.psfflux
                      FROM diasource s
                      INNER JOIN diaobject o ON s.diaobjectid=o.diaobjectid
                      INNER JOIN base_procver_of_procver j ON s.base_procver_id=j.base_procver_id
@@ -1900,9 +1902,9 @@ def create_object_stats_materialized_view( procver ):
                   GROUP BY rootid, band
                ) n23 ON r.rootid=n23.rootid AND r.band=n23.band
                LEFT JOIN (
-                  SELECT rootid, band, COUNT(diasourceid) AS ndets
+                  SELECT rootid, band, COUNT(*) AS ndets
                   FROM (
-                     SELECT DISTINCT ON(o.rootid, s.visit) o.rootid, s.band, s.diasourceid, s.psfflux
+                     SELECT DISTINCT ON(o.rootid, s.visit) o.rootid, s.band, s.psfflux
                      FROM diasource s
                      INNER JOIN diaobject o ON s.diaobjectid=o.diaobjectid
                      INNER JOIN base_procver_of_procver j ON s.base_procver_id=j.base_procver_id
@@ -1913,9 +1915,9 @@ def create_object_stats_materialized_view( procver ):
                   GROUP BY rootid, band
                ) n22 ON r.rootid=n22.rootid AND r.band=n22.band
                LEFT JOIN (
-                  SELECT rootid, band, COUNT(diasourceid) AS ndets
+                  SELECT rootid, band, COUNT(*) AS ndets
                   FROM (
-                     SELECT DISTINCT ON(o.rootid, s.visit) o.rootid, s.band, s.diasourceid, s.psfflux
+                     SELECT DISTINCT ON(o.rootid, s.visit) o.rootid, s.band, s.psfflux
                      FROM diasource s
                      INNER JOIN diaobject o ON s.diaobjectid=o.diaobjectid
                      INNER JOIN base_procver_of_procver j ON s.base_procver_id=j.base_procver_id
@@ -1926,9 +1928,21 @@ def create_object_stats_materialized_view( procver ):
                   GROUP BY rootid, band
                ) n21 ON r.rootid=n21.rootid AND r.band=n21.band
                LEFT JOIN (
-                  SELECT rootid, band, COUNT(diasourceid) AS ndets
+                  SELECT rootid, band, COUNT(*) AS nfrc
                   FROM (
-                     SELECT DISTINCT ON(o.rootid, s.visit) o.rootid, s.band, s.diasourceid, s.psfflux, s.psffluxerr
+                     SELECT DISTINCT ON(o.rootid, f.visit) o.rootid, f.band
+                     FROM diaforcedsource f
+                     INNER JOIN diaobject o ON f.diaobjectid=o.diaobjectid
+                     INNER JOIN base_procver_of_procver j ON f.base_procver_id=j.base_procver_id
+                                                         AND j.procver_id={pvid}
+                     ORDER BY o.rootid, f.visit, j.priority DESC
+                  ) subq
+                  GROUP BY rootid, band
+               ) nf ON r.rootid=nf.rootid AND r.band=nf.band
+               LEFT JOIN (
+                  SELECT rootid, band, COUNT(*) AS ndets
+                  FROM (
+                     SELECT DISTINCT ON(o.rootid, s.visit) o.rootid, s.band, s.psfflux, s.psffluxerr
                      FROM diasource s
                      INNER JOIN diaobject o ON s.diaobjectid=o.diaobjectid
                      INNER JOIN base_procver_of_procver j ON s.base_procver_id=j.base_procver_id
@@ -1939,9 +1953,9 @@ def create_object_stats_materialized_view( procver ):
                   GROUP BY rootid, band
                ) sn10 ON r.rootid=sn10.rootid AND r.band=sn10.band
                LEFT JOIN (
-                  SELECT rootid, band, COUNT(diasourceid) AS ndets
+                  SELECT rootid, band, COUNT(*) AS ndets
                   FROM (
-                     SELECT DISTINCT ON(o.rootid, s.visit) o.rootid, s.band, s.diasourceid, s.psfflux, s.psffluxerr
+                     SELECT DISTINCT ON(o.rootid, s.visit) o.rootid, s.band, s.psfflux, s.psffluxerr
                      FROM diasource s
                      INNER JOIN diaobject o ON s.diaobjectid=o.diaobjectid
                      INNER JOIN base_procver_of_procver j ON s.base_procver_id=j.base_procver_id
@@ -1952,9 +1966,9 @@ def create_object_stats_materialized_view( procver ):
                   GROUP BY rootid, band
                ) sn7 ON r.rootid=sn7.rootid AND r.band=sn7.band
                LEFT JOIN (
-                  SELECT rootid, band, COUNT(diasourceid) AS ndets
+                  SELECT rootid, band, COUNT(*) AS ndets
                   FROM (
-                     SELECT DISTINCT ON(o.rootid, s.visit) o.rootid, s.band, s.diasourceid, s.psfflux, s.psffluxerr
+                     SELECT DISTINCT ON(o.rootid, s.visit) o.rootid, s.band, s.psfflux, s.psffluxerr
                      FROM diasource s
                      INNER JOIN diaobject o ON s.diaobjectid=o.diaobjectid
                      INNER JOIN base_procver_of_procver j ON s.base_procver_id=j.base_procver_id
@@ -1964,18 +1978,6 @@ def create_object_stats_materialized_view( procver ):
                   WHERE psfflux / psffluxerr >= 5
                   GROUP BY rootid, band
                ) sn5 ON r.rootid=sn5.rootid AND r.band=sn5.band
-               LEFT JOIN (
-                  SELECT rootid, band, COUNT(*) AS nfrc
-                  FROM (
-                     SELECT DISTINCT ON(o.rootid, f.visit) o.rootid, f.band
-                     FROM diaforcedsource f
-                     INNER JOIN diaobject o ON f.diaobjectid=o.diaobjectid
-                     INNER JOIN base_procver_of_procver j ON f.base_procver_id=j.base_procver_id
-                                                         AND j.procver_id={pvid}
-                     ORDER BY o.rootid, f.visit, j.priority DESC
-                  ) subq
-                  GROUP BY rootid, band
-               ) nf ON r.rootid=nf.rootid AND r.band=nf.band
             )
             """
         ) ).format( viewname=sql.Identifier( f'objstats_{procver}' ), pvid=pvid )
@@ -2003,7 +2005,7 @@ def create_object_stats_materialized_view( procver ):
 
         # Now create the view that combines all the bands together
         q = sql.SQL( textwrap.dedent(
-            """
+            """\
             CREATE MATERIALIZED VIEW {combviewname} AS (
               SELECT s.rootid, s.ra, s.dec,
                      fd.mjd AS firstdet_mjd, fd.band AS firstdet_band,

From f875d123d3873d7583ca30d3b16c6b93205d6717 Mon Sep 17 00:00:00 2001
From: Rob Knop <rknop@pobox.com>
Date: Wed, 23 Sep 2026 17:29:57 -0400
Subject: [PATCH 15/35] In the middle of adding reliability statistics to the
 materialized objects stats view, it's not working, rage.

---
 src/ltcv.py                          | 208 +++++++++++++--------------
 src/webserver/static/objectsearch.js |  39 ++++-
 tests/conftest.py                    |  30 +++-
 tests/test_ltcv.py                   |   6 +-
 4 files changed, 167 insertions(+), 116 deletions(-)

diff --git a/src/ltcv.py b/src/ltcv.py
index 0e0c234..3c44ac3 100644
--- a/src/ltcv.py
+++ b/src/ltcv.py
@@ -1292,6 +1292,9 @@ def object_search( processing_version='default', just_objids=False, searchband=N
           ndets: int
              Number of detections
 
+          nfrc: int
+             Number of diaforcedsources
+
           ndets24: int
              Number of detections with mag≤24
 
@@ -1313,9 +1316,6 @@ def object_search( processing_version='default', just_objids=False, searchband=N
           nsn5: int
              Number of detections with S/N > 5
 
-          nfrc: int
-             Number of diaforcedsources
-
     Returns
     -------
       dict
@@ -1340,6 +1340,7 @@ def object_search( processing_version='default', just_objids=False, searchband=N
           lastforced_flux    : float, flux in nJy (zeropoint 31.4) of last forced photometry point
           lastforced_fluxerr : float
           ndets              : int, number of detections (diasources for this processing version and rootid)
+          nfrc               : int, number of forced-photometry points
           ndets24            : int, number of detections with mag ≤ 24
           ndets23            : int, number of detections with mag ≤ 23
           ndets22            : int, number of detections with mag ≤ 22
@@ -1347,7 +1348,12 @@ def object_search( processing_version='default', just_objids=False, searchband=N
           nsn10              : int, number of detections with flux/fluxerr ≥ 10
           nsn7               : int, number of detections with flux/fluxerr ≥ 7
           nsn5               : int, number of detections with flux/fluxerr ≥ 5
-          nfrc               : int, number of forced-photometry points
+          nrelp5             : int, number of detections with reliability > 0.5
+          nrelp6             : int, number of detections with reliability > 0.6
+          nrelp7             : int, number of detections with reliability > 0.7
+          nrelp8             : int, number of detections with reliability > 0.8
+          nrelp9             : int, number of detections with reliability > 0.9
+          nrelp95            : int, number of detections with reliability > 0.95
 
     """
 
@@ -1357,30 +1363,36 @@ def object_search( processing_version='default', just_objids=False, searchband=N
     viewname = f'objstatscomb_{pvobj.description}' if len(searchband) == 0 else f'objstats_{pvobj.description}'
 
     searchspec = {
-        'rootid':             { 'mult': True,   'substr': False, 'minmax': False, 'dtype': np.dtype('O') },
-        'ra':                 { 'mult': False,  'substr': False, 'minmax': True, 'dtype': np.float64 },
-        'dec':                { 'mult': False,  'substr': False, 'minmax': True, 'dtype': np.float64 },
-        'firstdet_mjd':       { 'mult': False,  'substr': False, 'minmax': True, 'dtype': np.float64 },
-        'firstdet_flux':      { 'mult': False,  'substr': False, 'minmax': True, 'dtype': np.float32 },
-        'firstdet_fluxerr':   { 'mult': False,  'substr': False, 'minmax': True, 'dtype': np.float32 },
-        'lastdet_mjd':        { 'mult': False,  'substr': False, 'minmax': True, 'dtype': np.float64 },
-        'lastdet_flux':       { 'mult': False,  'substr': False, 'minmax': True, 'dtype': np.float32 },
-        'lastdet_fluxerr':    { 'mult': False,  'substr': False, 'minmax': True, 'dtype': np.float32 },
-        'maxdet_mjd':         { 'mult': False,  'substr': False, 'minmax': True, 'dtype': np.float64 },
-        'maxdet_flux':        { 'mult': False,  'substr': False, 'minmax': True, 'dtype': np.float32 },
-        'maxdet_fluxerr':     { 'mult': False,  'substr': False, 'minmax': True, 'dtype': np.float32 },
-        'lastforced_mjd':     { 'mult': False,  'substr': False, 'minmax': True, 'dtype': np.float32 },
-        'lastforced_flux':    { 'mult': False,  'substr': False, 'minmax': True, 'dtype': np.int16 },
-        'lastforced_fluxerr': { 'mult': False,  'substr': False, 'minmax': True, 'dtype': np.int16 },
-        'ndets':              { 'mult': False,  'substr': False, 'minmax': True, 'dtype': np.int16 },
-        'ndets24':            { 'mult': False,  'substr': False, 'minmax': True, 'dtype': np.int16 },
-        'ndets23':            { 'mult': False,  'substr': False, 'minmax': True, 'dtype': np.int16 },
-        'ndets22':            { 'mult': False,  'substr': False, 'minmax': True, 'dtype': np.int16 },
-        'ndets21':            { 'mult': False,  'substr': False, 'minmax': True, 'dtype': np.int16 },
-        'nsn10':              { 'mult': False,  'substr': False, 'minmax': True, 'dtype': np.int16 },
-        'nsn7':               { 'mult': False,  'substr': False, 'minmax': True, 'dtype': np.int16 },
-        'nsn5':               { 'mult': False,  'substr': False, 'minmax': True, 'dtype': np.int16 },
-        'nfrc':               { 'mult': False,  'substr': False, 'minmax': True, 'dtype': np.int16 },
+        'rootid':             { 'mult': True,  'substr': False, 'minmax': False, 'dtype': np.dtype('O') },
+        'ra':                 { 'mult': False, 'substr': False, 'minmax': True, 'dtype': np.float64 },
+        'dec':                { 'mult': False, 'substr': False, 'minmax': True, 'dtype': np.float64 },
+        'firstdet_mjd':       { 'mult': False, 'substr': False, 'minmax': True, 'dtype': np.float64 },
+        'firstdet_flux':      { 'mult': False, 'substr': False, 'minmax': True, 'dtype': np.float32 },
+        'firstdet_fluxerr':   { 'mult': False, 'substr': False, 'minmax': True, 'dtype': np.float32 },
+        'lastdet_mjd':        { 'mult': False, 'substr': False, 'minmax': True, 'dtype': np.float64 },
+        'lastdet_flux':       { 'mult': False, 'substr': False, 'minmax': True, 'dtype': np.float32 },
+        'lastdet_fluxerr':    { 'mult': False, 'substr': False, 'minmax': True, 'dtype': np.float32 },
+        'maxdet_mjd':         { 'mult': False, 'substr': False, 'minmax': True, 'dtype': np.float64 },
+        'maxdet_flux':        { 'mult': False, 'substr': False, 'minmax': True, 'dtype': np.float32 },
+        'maxdet_fluxerr':     { 'mult': False, 'substr': False, 'minmax': True, 'dtype': np.float32 },
+        'lastforced_mjd':     { 'mult': False, 'substr': False, 'minmax': True, 'dtype': np.float32 },
+        'lastforced_flux':    { 'mult': False, 'substr': False, 'minmax': True, 'dtype': np.int16 },
+        'lastforced_fluxerr': { 'mult': False, 'substr': False, 'minmax': True, 'dtype': np.int16 },
+        'ndets':              { 'mult': False, 'substr': False, 'minmax': True, 'dtype': np.int16 },
+        'nfrc':               { 'mult': False, 'substr': False, 'minmax': True, 'dtype': np.int16 },
+        'ndets24':            { 'mult': False, 'substr': False, 'minmax': True, 'dtype': np.int16 },
+        'ndets23':            { 'mult': False, 'substr': False, 'minmax': True, 'dtype': np.int16 },
+        'ndets22':            { 'mult': False, 'substr': False, 'minmax': True, 'dtype': np.int16 },
+        'ndets21':            { 'mult': False, 'substr': False, 'minmax': True, 'dtype': np.int16 },
+        'nsn10':              { 'mult': False, 'substr': False, 'minmax': True, 'dtype': np.int16 },
+        'nsn7':               { 'mult': False, 'substr': False, 'minmax': True, 'dtype': np.int16 },
+        'nsn5':               { 'mult': False, 'substr': False, 'minmax': True, 'dtype': np.int16 },
+        'nrelgtp50':          { 'mult': False, 'substr': False, 'minmax': True, 'dtype': np.int16 },
+        'nrelgtp60':          { 'mult': False, 'substr': False, 'minmax': True, 'dtype': np.int16 },
+        'nrelgtp70':          { 'mult': False, 'substr': False, 'minmax': True, 'dtype': np.int16 },
+        'nrelgtp80':          { 'mult': False, 'substr': False, 'minmax': True, 'dtype': np.int16 },
+        'nrelgtp90':          { 'mult': False, 'substr': False, 'minmax': True, 'dtype': np.int16 },
+        'nrelgtp95':          { 'mult': False, 'substr': False, 'minmax': True, 'dtype': np.int16 },
     }
 
     radius = None
@@ -1728,8 +1740,9 @@ def create_object_stats_materialized_view( procver ):
                              'lastdet_mjd', 'lastdet_flux', 'lastdet_fluxerr',
                              'maxdet_mjd', 'maxdet_flux', 'maxdet_fluxerr',
                              'lastforced_mjd', 'lastforced_flux', 'lastforced_fluxerr',
-                             'ndets', 'ndets24', 'ndets23', 'ndets22', 'ndets21', 'nfrc',
-                             'nsn10', 'nsn7', 'nsn5' }
+                             'ndets', 'nfrc' ,'ndets24', 'ndets23', 'ndets22', 'ndets21',
+                             'nsn10', 'nsn7', 'nsn5', 'nrelgtp50', 'nrelgtp60',
+                             'nrelgtp70', 'nrelgtp80', 'nrelgtp90', 'nrelgtp95' }
             if set( r['attname'] for r in rows ) != expectedcols:
                 raise RuntimeError( f"postgres view objstats_{procver} has the wrong set of columns" )
 
@@ -1786,7 +1799,7 @@ def create_object_stats_materialized_view( procver ):
                    dx.midpointmjdtai AS maxdet_mjd, dx.psfflux AS maxdet_flux, dx.psffluxerr AS maxdet_fluxerr,
                    fn.midpointmjdtai AS lastforced_mjd, fn.psfflux AS lastforced_flux,
                      fn.psffluxerr AS lastforced_fluxerr,
-                   n.ndets,
+                   n.ndets, nf.nfrc,
                    CASE WHEN n24.ndets IS NULL THEN 0 ELSE n24.ndets END as ndets24,
                    CASE WHEN n23.ndets IS NULL THEN 0 ELSE n23.ndets END AS ndets23,
                    CASE WHEN n22.ndets IS NULL THEN 0 ELSE n22.ndets END AS ndets22,
@@ -1794,7 +1807,12 @@ def create_object_stats_materialized_view( procver ):
                    CASE WHEN sn10.ndets IS NULL THEN 0 ELSE sn10.ndets END AS nsn10,
                    CASE WHEN sn7.ndets IS NULL THEN 0 ELSE sn7.ndets END AS nsn7,
                    CASE WHEN sn5.ndets IS NULL THEN 0 ELSE sn5.ndets END AS nsn5,
-                   nf.nfrc
+                   CASE WHEN relp5.ngood IS NULL THEN 0 ELSE relp5.ngood END AS nrelgtp50,
+                   CASE WHEN relp6.ngood IS NULL THEN 0 ELSE relp6.ngood END AS nrelgtp60,
+                   CASE WHEN relp7.ngood IS NULL THEN 0 ELSE relp7.ngood END AS nrelgtp70,
+                   CASE WHEN relp8.ngood IS NULL THEN 0 ELSE relp8.ngood END AS nrelgtp80,
+                   CASE WHEN relp9.ngood IS NULL THEN 0 ELSE relp9.ngood END AS nrelgtp90,
+                   CASE WHEN relp95.ngood IS NULL THEN 0 ELSE relp95.ngood END AS nrelgtp95
                FROM (
                  ( SELECT DISTINCT ON(o.rootid, s.band) o.rootid, r.ra, r.dec, s.band
                    FROM root_diaobject r
@@ -1875,58 +1893,6 @@ def create_object_stats_materialized_view( procver ):
                   ) subq
                   GROUP BY rootid, band
                ) n ON r.rootid=n.rootid AND r.band=n.band
-               LEFT JOIN (
-                  SELECT rootid, band, COUNT(*) AS ndets
-                  FROM (
-                     SELECT DISTINCT ON(o.rootid, s.visit) o.rootid, s.band, s.psfflux
-                     FROM diasource s
-                     INNER JOIN diaobject o ON s.diaobjectid=o.diaobjectid
-                     INNER JOIN base_procver_of_procver j ON s.base_procver_id=j.base_procver_id
-                                                         AND j.procver_id={pvid}
-                     ORDER BY o.rootid, s.visit, j.priority DESC
-                  ) subq
-                  WHERE psfflux >= 912
-                  GROUP BY rootid, band
-               ) n24 ON r.rootid=n24.rootid AND r.band=n24.band
-               LEFT JOIN (
-                  SELECT rootid, band, COUNT(*) AS ndets
-                  FROM (
-                     SELECT DISTINCT ON(o.rootid, s.visit) o.rootid, s.band, s.psfflux
-                     FROM diasource s
-                     INNER JOIN diaobject o ON s.diaobjectid=o.diaobjectid
-                     INNER JOIN base_procver_of_procver j ON s.base_procver_id=j.base_procver_id
-                                                         AND j.procver_id={pvid}
-                     ORDER BY o.rootid, s.visit, j.priority DESC
-                  ) subq
-                  WHERE psfflux >= 2291
-                  GROUP BY rootid, band
-               ) n23 ON r.rootid=n23.rootid AND r.band=n23.band
-               LEFT JOIN (
-                  SELECT rootid, band, COUNT(*) AS ndets
-                  FROM (
-                     SELECT DISTINCT ON(o.rootid, s.visit) o.rootid, s.band, s.psfflux
-                     FROM diasource s
-                     INNER JOIN diaobject o ON s.diaobjectid=o.diaobjectid
-                     INNER JOIN base_procver_of_procver j ON s.base_procver_id=j.base_procver_id
-                                                         AND j.procver_id={pvid}
-                     ORDER BY o.rootid, s.visit, j.priority DESC
-                  ) subq
-                  WHERE psfflux >= 5754
-                  GROUP BY rootid, band
-               ) n22 ON r.rootid=n22.rootid AND r.band=n22.band
-               LEFT JOIN (
-                  SELECT rootid, band, COUNT(*) AS ndets
-                  FROM (
-                     SELECT DISTINCT ON(o.rootid, s.visit) o.rootid, s.band, s.psfflux
-                     FROM diasource s
-                     INNER JOIN diaobject o ON s.diaobjectid=o.diaobjectid
-                     INNER JOIN base_procver_of_procver j ON s.base_procver_id=j.base_procver_id
-                                                         AND j.procver_id={pvid}
-                     ORDER BY o.rootid, s.visit, j.priority DESC
-                  ) subq
-                  WHERE psfflux >= 14454
-                  GROUP BY rootid, band
-               ) n21 ON r.rootid=n21.rootid AND r.band=n21.band
                LEFT JOIN (
                   SELECT rootid, band, COUNT(*) AS nfrc
                   FROM (
@@ -1939,19 +1905,30 @@ def create_object_stats_materialized_view( procver ):
                   ) subq
                   GROUP BY rootid, band
                ) nf ON r.rootid=nf.rootid AND r.band=nf.band
+            """
+        ) ).format( viewname=sql.Identifier( f'objstats_{procver}' ), pvid=pvid )
+        for alias, flux in zip( [ 'n24', 'n23', 'n22', 'n21' ],
+                                [ 912, 2291, 5754, 14454 ] ):
+            q += sql.SQL( textwrap.indent( textwrap.dedent(
+                """\
                LEFT JOIN (
                   SELECT rootid, band, COUNT(*) AS ndets
                   FROM (
-                     SELECT DISTINCT ON(o.rootid, s.visit) o.rootid, s.band, s.psfflux, s.psffluxerr
+                     SELECT DISTINCT ON(o.rootid, s.visit) o.rootid, s.band, s.psfflux
                      FROM diasource s
                      INNER JOIN diaobject o ON s.diaobjectid=o.diaobjectid
                      INNER JOIN base_procver_of_procver j ON s.base_procver_id=j.base_procver_id
                                                          AND j.procver_id={pvid}
                      ORDER BY o.rootid, s.visit, j.priority DESC
                   ) subq
-                  WHERE psfflux / psffluxerr >= 10
+                  WHERE psfflux >= {flux}
                   GROUP BY rootid, band
-               ) sn10 ON r.rootid=sn10.rootid AND r.band=sn10.band
+               ) {alias} ON r.rootid={alias}.rootid AND r.band={alias}.band
+                """
+                ), '   ' ) ).format( pvid=pvid, flux=sql.SQL(str(flux)), alias=sql.Identifier(alias) )
+        for alias, sn in zip( [ 'sn10', 'sn7', 'sn5' ], [ 10, 7, 5 ] ):
+            q += sql.SQL( textwrap.indent( textwrap.dedent(
+                """\
                LEFT JOIN (
                   SELECT rootid, band, COUNT(*) AS ndets
                   FROM (
@@ -1962,22 +1939,33 @@ def create_object_stats_materialized_view( procver ):
                                                          AND j.procver_id={pvid}
                      ORDER BY o.rootid, s.visit, j.priority DESC
                   ) subq
-                  WHERE psfflux / psffluxerr >= 7
+                  WHERE psfflux / psffluxerr >= {sn}
                   GROUP BY rootid, band
-               ) sn7 ON r.rootid=sn7.rootid AND r.band=sn7.band
+               ) {alias} ON r.rootid={alias}.rootid AND r.band={alias}.band
+                """
+                ), '   ' ) ).format( pvid=pvid, sn=sql.SQL(str(sn)), alias=sql.Identifier(alias) )
+        for alias, rel in zip( [ 'relp5', 'relp6', 'relp7', 'relp8', 'relp9', 'relp95' ],
+                               [ 0.5, 0.6, 0.7, 0.8, 0.9, 0.95 ] ):
+            q += sql.SQL( textwrap.indent( textwrap.dedent(
+                """\
                LEFT JOIN (
-                  SELECT rootid, band, COUNT(*) AS ndets
+                  SELECT rootid, band, COUNT(*) as ngood
                   FROM (
-                     SELECT DISTINCT ON(o.rootid, s.visit) o.rootid, s.band, s.psfflux, s.psffluxerr
-                     FROM diasource s
-                     INNER JOIN diaobject o ON s.diaobjectid=o.diaobjectid
-                     INNER JOIN base_procver_of_procver j ON s.base_procver_id=j.base_procver_id
-                                                         AND j.procver_id={pvid}
-                     ORDER BY o.rootid, s.visit, j.priority DESC
-                  ) subq
-                  WHERE psfflux / psffluxerr >= 5
-                  GROUP BY rootid, band
-               ) sn5 ON r.rootid=sn5.rootid AND r.band=sn5.band
+                    SELECT DISTINCT ON(o.rootid, s.visit) o.rootid, s.band, se.reliability
+                    FROM diasource s
+                    INNER JOIN diaobject o ON s.diaobjectid=o.diaobjectid
+                    INNER JOIN diasource_extra se ON s.diasourceid=se.diasourceid
+                    INNER JOIN base_procver_of_procver j ON s.base_procver_id=j.base_procver_id
+                                                     AND j.procver_id={pvid}
+                    ORDER BY o.rootid, s.visit, j.priority DESC
+                 ) subq
+                 WHERE reliability > {rel}
+                 GROUP BY rootid, band
+               ) {alias} ON r.rootid={alias}.rootid AND r.band={alias}.band
+                """
+                ), '   ' ) ).format( pvid=pvid, rel=sql.SQL(f'{rel:.1f}'), alias=sql.Identifier(alias) )
+        q += sql.SQL( textwrap.dedent(
+            """\
             )
             """
         ) ).format( viewname=sql.Identifier( f'objstats_{procver}' ), pvid=pvid )
@@ -1986,7 +1974,9 @@ def create_object_stats_materialized_view( procver ):
 
         indexcols = [ 'rootid', 'firstdet_mjd', 'lastdet_mjd', 'maxdet_mjd',
                      'firstdet_flux', 'lastdet_flux', 'maxdet_flux',
-                     'ndets', 'ndets24', 'ndets23', 'ndets22', 'ndets21', 'nsn10', 'nsn7', 'nsn5', 'nfrc' ]
+                     'ndets', 'nfrc',
+                     'ndets24', 'ndets23', 'ndets22', 'ndets21', 'nsn10', 'nsn7', 'nsn5',
+                     'nrelgtp50', 'nrelgtp60', 'nrelgtp70', 'nrelgtp80', 'nrelgtp90', 'nrelgtp95' ]
         for col in indexcols:
             q = sql.SQL( 'CREATE INDEX {idxname} ON {viewname}({col})'
                         ).format( idxname=sql.Identifier( f'idx_obstats_{procver}_{col}' ),
@@ -2016,14 +2006,18 @@ def create_object_stats_materialized_view( procver ):
                        xd.flux AS maxdet_flux, xd.fluxerr AS maxdet_fluxerr,
                      lf.mjd AS lastforced_mjd, lf.band AS lastforced_band,
                        lf.flux AS lastforced_flux, lf.fluxerr AS lastforced_fluxerr,
-                     s.ndets AS ndets, s.ndets24 AS ndets24, s.ndets23 AS ndets23,
-                       s.ndets22 AS ndets22, s.ndets21 AS ndets21,
+                     s.ndets AS ndets, s.nfrc AS nfrc, s.ndets24 AS ndets24,
+                       s.ndets23 AS ndets23, s.ndets22 AS ndets22, s.ndets21 AS ndets21,
                      s.nsn10 AS nsn10, s.nsn7 AS nsn7, s.nsn5 AS nsn5,
-                     s.nfrc AS nfrc
+                     s.nrelgtp50 AS nrelgtp50, s.nrelgtp60 AS nrelgtp60, s.nrelgtp70 AS nrelgtp70,
+                       s.nrelgtp80 AS nrelgtp80, s.nrelgtp90 AS nrelgtp90, s.nrelgtp95 AS nrelgtp95
               FROM (
-                SELECT rootid, ra, dec, SUM(ndets) AS ndets, SUM(ndets24) AS ndets24, SUM(ndets23) AS ndets23,
-                       SUM(ndets22) AS ndets22, SUM(ndets21) AS ndets21, SUM(nsn10) AS nsn10,
-                       SUM(nsn7) AS nsn7, SUM(nsn5) AS nsn5, SUM(nfrc) AS nfrc
+                SELECT rootid, ra, dec, SUM(ndets) AS ndets, SUM(nfrc) AS nfrc,
+                       SUM(ndets24) AS ndets24, SUM(ndets23) AS ndets23,
+                       SUM(ndets22) AS ndets22, SUM(ndets21) AS ndets21,
+                       SUM(nsn10) AS nsn10, SUM(nsn7) AS nsn7, SUM(nsn5) AS nsn5,
+                       SUM(nrelgtp50) as nrelgtp50, SUM(nrelgtp60) AS nrelgtp60, SUM(nrelgtp70) AS nrelgtp70,
+                       SUM(nrelgtp80) as nrelgtp80, SUM(nrelgtp90) AS nrelgtp90, SUM(nrelgtp95) AS nrelgtp95
                 FROM {viewname}
                 GROUP BY rootid, ra, dec
               ) s
diff --git a/src/webserver/static/objectsearch.js b/src/webserver/static/objectsearch.js
index 811581a..b3eb573 100644
--- a/src/webserver/static/objectsearch.js
+++ b/src/webserver/static/objectsearch.js
@@ -206,7 +206,7 @@ fastdbap.ObjectSearch = class
 
         vbox = rkWebUtil.elemaker( "div", div, { "classes": [ "vbox", "xmarginright", "searchinner" ] } );
         hbox = rkWebUtil.elemaker( "div", vbox, { "classes": [ "hbox", "bold" ],
-                                                  "text": "Detections with:" } );
+                                                  "text": "Number of Detections with:" } );
         hbox = rkWebUtil.elemaker( "div", vbox, { "classes": [ "hbox" ] } );
         table = rkWebUtil.elemaker( "table", hbox, { "classes": [ "borderless", "mmarginright" ] } );
         tr = rkWebUtil.elemaker( "tr", table );
@@ -254,7 +254,44 @@ fastdbap.ObjectSearch = class
         td = rkWebUtil.elemaker( "td", tr, { "text": "≤" } )
         this.maxdetsngt10_widget = rkWebUtil.elemaker( "input", td, { "attributes": { "size": 4 } } );
 
+        table = rkWebUtil.elemaker( "table", vbox );
+        tr = rkWebUtil.elemaker( "tr", table );
+        td = rkWebUtil.elemaker( "td", tr, { "text": "reliability > 0.5", "classes": [ "right" ] } );
+        td = rkWebUtil.elemaker( "td", tr, { "text": "≥" } );
+        this.minreliabilitygtp5_widget = rkWebUtil.elemaker( "input", td, { "attributes": { "size": 4 } } );
+        td = rkWebUtil.elemaker( "td", tr, { "text": "≤" } );
+        this.maxreliabilitygtp5_widget = rkWebUtil.elemaker( "input", td, { "attributes": { "size": 4 } } );
+        td = rkWebUtil.elemaker( "td", tr, { "text": "reliability > 0.8", "classes": [ "right" ] } );
+        td = rkWebUtil.elemaker( "td", tr, { "text": "≥" } );
+        this.minreliabilitygtp8_widget = rkWebUtil.elemaker( "input", td, { "attributes": { "size": 4 } } );
+        td = rkWebUtil.elemaker( "td", tr, { "text": "≤" } );
+        this.maxreliabilitygtp8_widget = rkWebUtil.elemaker( "input", td, { "attributes": { "size": 4 } } );
+
+        tr = rkWebUtil.elemaker( "tr", table );
+        td = rkWebUtil.elemaker( "td", tr, { "text": "reliability > 0.6", "classes": [ "right" ] } );
+        td = rkWebUtil.elemaker( "td", tr, { "text": "≥" } );
+        this.minreliabilitygtp6_widget = rkWebUtil.elemaker( "input", td, { "attributes": { "size": 4 } } );
+        td = rkWebUtil.elemaker( "td", tr, { "text": "≤" } );
+        this.maxreliabilitygtp6_widget = rkWebUtil.elemaker( "input", td, { "attributes": { "size": 4 } } );
+        td = rkWebUtil.elemaker( "td", tr, { "text": "reliability > 0.9", "classes": [ "right" ] } );
+        td = rkWebUtil.elemaker( "td", tr, { "text": "≥" } );
+        this.minreliabilitygtp9_widget = rkWebUtil.elemaker( "input", td, { "attributes": { "size": 4 } } );
+        td = rkWebUtil.elemaker( "td", tr, { "text": "≤" } );
+        this.maxreliabilitygtp9_widget = rkWebUtil.elemaker( "input", td, { "attributes": { "size": 4 } } );
+
+        tr = rkWebUtil.elemaker( "tr", table );
+        td = rkWebUtil.elemaker( "td", tr, { "text": "reliability > 0.7", "classes": [ "right" ] } );
+        td = rkWebUtil.elemaker( "td", tr, { "text": "≥" } );
+        this.minreliabilitygtp7_widget = rkWebUtil.elemaker( "input", td, { "attributes": { "size": 4 } } );
+        td = rkWebUtil.elemaker( "td", tr, { "text": "≤" } );
+        this.maxreliabilitygtp7_widget = rkWebUtil.elemaker( "input", td, { "attributes": { "size": 4 } } );
+        td = rkWebUtil.elemaker( "td", tr, { "text": "reliability > 0.95", "classes": [ "right" ] } );
+        td = rkWebUtil.elemaker( "td", tr, { "text": "≥" } );
+        this.minreliabilitygtp95_widget = rkWebUtil.elemaker( "input", td, { "attributes": { "size": 4 } } );
+        td = rkWebUtil.elemaker( "td", tr, { "text": "≤" } );
+        this.maxreliabilitygtp95_widget = rkWebUtil.elemaker( "input", td, { "attributes": { "size": 4 } } );
 
+        
         // Window... not currently supported by ltcv.py::object_search
 
         // vbox = rkWebUtil.elemaker( "div", div, { "classes": [ "vbox", "xmarginright", "searchinner" ] } );
diff --git a/tests/conftest.py b/tests/conftest.py
index 4137842..1db4f03 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -383,6 +383,9 @@ def pos_offset( bpvkey, mjd ):
         lastmag = [ 25.9, 25.1, 25.6, 26. ]
         zeromag = 32.
 
+        # Also random-but-deterministic reliabilities
+        reliability_rng = np.random.default_rng( 137 )
+
         visit = 0
         for i, rootobj in enumerate(rootobjinfo):
             robj = RootDiaObject( id=uuid.uuid4(), ra=rootobjinfo[i]['ra'], dec=rootobjinfo[i]['dec'] )
@@ -502,7 +505,9 @@ def pos_offset( bpvkey, mjd ):
                                                 base_procver_id=src.base_procver_id,
                                                 detector=0,
                                                 x=1,
-                                                y=1 )
+                                                y=1,
+                                                reliability=reliability_rng.random()
+                                               )
                         srcexobjs.append( srcex )
                         rootdict['srcex']['realtime_diasource'].append( srcex )
                     else:
@@ -584,11 +589,12 @@ def pos_offset( bpvkey, mjd ):
                                                 base_procver_id=src.base_procver_id,
                                                 detector=0,
                                                 x=1,
-                                                y=1 )
+                                                y=1,
+                                                reliability=reliability_rng.random() )
                         srcexobjs.append( srcex )
                         rootdict['srcex'][f'{bpv}_diasource'].append( srcex )
                     else:
-                        rootdict['srcex'][f'{bpv}_diasource'].append( srcex )
+                        rootdict['srcex'][f'{bpv}_diasource'].append( None )
 
                     if ( bpv != 'bpv2a' ) or ( sourcemjd <= 60025 ):
                         frc = DiaForcedSource( diaforcedsourceid=objtouse.diaobjectid * 1000000 + int(sourcemjd),
@@ -657,7 +663,8 @@ def pos_offset( bpvkey, mjd ):
                                                     base_procver_id=src.base_procver_id,
                                                     detector=0,
                                                     x=1,
-                                                    y=1 )
+                                                    y=1,
+                                                    reliability=reliability_rng.random() )
                             srcexobjs.append( srcex )
                             rootdict['srcex'][f'{bpv}_diasource'].append( srcex )
                         else:
@@ -1505,6 +1512,8 @@ def do_the_things( procver, band=None ):
                 thisexp[ f'ndets{mag}' ] = [ 0, None, None ]
             for sn in [ 5, 7, 10 ]:
                 thisexp[ f'nsn{sn}' ] = [ 0, None, None ]
+            for rcut in [ 50, 60, 70, 80, 90, 95 ]:
+                thisexp[ f'nrelgtp{rcut}' ] = [ 0, None, None ]
 
             # There are multiple base processing versions that have data.  We need to
             #  extract the highest priority for each.  To do this, rewrangle
@@ -1520,13 +1529,19 @@ def do_the_things( procver, band=None ):
             frc_bpvkeys = [ p[2] for p in pvrow['diaforcedsource'] ]
             reindexed_srces = { k: { s.visit: s for s in root['src'][k] if cond(s) }
                                 for k in src_bpvkeys if k in root['src'].keys() }
+            reindexed_srcextras = { k: { s.visit: se for s, se in zip(root['src'][k], root['srcex'][k]) if cond(s) }
+                                    for k in src_bpvkeys if k in root['src'].keys() }
             reindexed_frced = { k: { f.visit: f for f in root['frc'][k] if cond(f) }
                                 for k in frc_bpvkeys if k in root['frc'].keys() }
 
             # Extract the expected source values for this root object
             seenvisits = set()
-            for bpv, srces in reindexed_srces.items():
-                for visit, src in srces.items():
+            for bpv in reindexed_srces.keys():
+                srces = reindexed_srces[ bpv ]
+                srcextras = reindexed_srcextras[ bpv ]
+                for visit in srces.keys():
+                    src = srces[ visit ]
+                    srcex = srcextras[ visit ]
                     if visit in seenvisits:
                         # already have a higher prio bpv
                         continue
@@ -1564,6 +1579,9 @@ def do_the_things( procver, band=None ):
                     for sn in [ 5, 7, 10 ]:
                         if ( src.psfflux / src.psffluxerr ) >= sn:
                             thisexp[ f'nsn{sn}' ][0] += 1
+                    for rel in [ 50, 60, 70, 80, 90, 95 ]:
+                        if ( srcex is not None ) and ( srcex.reliability > rel / 100. ):
+                            thisexp[ f'nrelgtp{rel}' ][0] += 1
 
             # Extract the expected forced source values for this root object
             seenvisits = set()
diff --git a/tests/test_ltcv.py b/tests/test_ltcv.py
index a75984f..f2457d5 100644
--- a/tests/test_ltcv.py
+++ b/tests/test_ltcv.py
@@ -18,8 +18,10 @@ def objstats_cols():
              'lastdet_mjd', 'lastdet_flux', 'lastdet_fluxerr',
              'maxdet_mjd', 'maxdet_flux', 'maxdet_fluxerr',
              'lastforced_mjd', 'lastforced_flux', 'lastforced_fluxerr',
-             'ndets', 'ndets24', 'ndets23', 'ndets22', 'ndets21', 'nfrc',
-             'nsn10', 'nsn7', 'nsn5' }
+             'ndets', 'nfrc',
+             'ndets24', 'ndets23', 'ndets22', 'ndets21',
+             'nsn10', 'nsn7', 'nsn5',
+             'nrelgtp50', 'nrelgtp60', 'nrelgtp70', 'nrelgtp80', 'nrelgtp90', 'nrelgtp95' }
 
 
 # THIS TEST MUST COME FIRST, because it depends on both the

From d3b01d00cf92a2efd83ea3cc9dbcfa4d24717b1b Mon Sep 17 00:00:00 2001
From: Rob Knop <rknop@pobox.com>
Date: Thu, 24 Sep 2026 11:04:26 -0400
Subject: [PATCH 16/35] Fix error in making materialized view with reliability
 stats.  Comments in tests.

---
 src/ltcv.py       |   1 +
 tests/conftest.py | 201 ++++++++++++++++++++++++++++------------------
 2 files changed, 123 insertions(+), 79 deletions(-)

diff --git a/src/ltcv.py b/src/ltcv.py
index 3c44ac3..aeea2fa 100644
--- a/src/ltcv.py
+++ b/src/ltcv.py
@@ -1955,6 +1955,7 @@ def create_object_stats_materialized_view( procver ):
                     FROM diasource s
                     INNER JOIN diaobject o ON s.diaobjectid=o.diaobjectid
                     INNER JOIN diasource_extra se ON s.diasourceid=se.diasourceid
+                                                 AND s.base_procver_id=se.base_procver_id
                     INNER JOIN base_procver_of_procver j ON s.base_procver_id=j.base_procver_id
                                                      AND j.procver_id={pvid}
                     ORDER BY o.rootid, s.visit, j.priority DESC
diff --git a/tests/conftest.py b/tests/conftest.py
index 1db4f03..8b20dc8 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -57,28 +57,62 @@ def procver_bases():
 
 @pytest.fixture( scope='module' )
 def procver_collection( procver_postimes, procver_bases ):
-    # A set of processing versions loaded into the database, including:
-    #
-    #   A set of base procesing versions pvc_bpv1, pvc_bpv1a, pvc_bpv1b, pvc_bpv2, pvc_bpv2a, pvc_bpv3, realtime
-    #   For each of these, there is a base processing version for tables diaobject, diaobject_position,
-    #     diasource, and diaforcedsource.
-    #
-    #   A set of processing versions with priorities (high to low)
-    #     pvc_pv1 : pvc_bpv1b, pvc_bpv1a, pvc_bpv1
-    #     pvc_pv2 : pvc_bpv2a, pvc_bpv2
-    #     pvc_pv3 : pvc_bpv3
-    #     realtime : realtime
-    #   A processing version alias default pointing at pvc_pv2
-    #
-    # (The fallbacks of pvc_pv2 and pvc_pv3 to the earlier bpvs are for Object handing in set_of_lightcurves)
-    #
-    # fixture value is a tuple of two dictionaries of str → BaseProcessingVersion, str → ProcessingVersion
-    #
-    # Keys for the processing versions are pv1, pv2, pv3, and realtime
-    # Keys for the base processing versions are {name} or {name}_{table},
-    #   where {name} is one of bpv1, bpv1a, bpv1b, bpv2, bpv2a, bpv3, realtime
-    #   and {table} is one of diaobject, diasource, diaforcedsource
-    # Base processing versions also have {name}_diaoject_position_{mjd} for diaobject_position
+    """A set of processing versions loaded into the database.
+
+    What gets loaded into the database:
+
+    * set of base procesing versions pvc_bpv1, pvc_bpv1a, pvc_bpv1b,
+      pvc_bpv2, pvc_bpv2a, pvc_bpv3, realtime.
+
+      For each of these, there is a base processing version for tables
+      diaobject, diasource, and diaforcedsource, and there are several
+      base processing versions for diaobject_position.
+
+    * A set of processing versions with priorities (high to low)
+        pvc_pv1 : pvc_bpv1b, pvc_bpv1a, pvc_bpv1
+        pvc_pv2 : pvc_bpv2a, pvc_bpv2
+        pvc_pv3 : pvc_bpv3
+        realtime : realtime
+
+    * A processing version alias default pointing at pvc_pv2
+
+    (The fallbacks of pvc_pv2 and pvc_pv3 to the earlier bpvs are for
+    Object handing in set_of_lightcurves) (...just read this months
+    later and I'm not sure it means.)
+
+    Yields
+    ------
+
+    ( bpvs, pvs, pvinfo )
+
+    * bpvs : dictionary of key : BaseProcessingVersion
+         Keys are {name} or {name}_{table}, where {name} is one of
+         [bpv1, bpv1a, bpv1b, bpv2, bpv2a, bpv3, realtime] (that is the
+         list returned by the procver_bases fiture) and {table} is one
+         of diaobject, diasource, diaforcedsource, or
+         diaobject_position_{mjd} (for a bunch of different mjds).  The
+         description of BaseProcessingVersion is the same as key with
+         "pvc_" prepended, except for realtime, where it's just
+         realtime.  (...yes that's annoying.  At some point in the past
+         I thought that was a good idea, who knows why.)
+
+    * pvs : dictionary of key : ProcessingVersion
+         Keys are (CHECK THIS) pv1, pv2, pv3, and realtime.  The
+         corresponding descriptions in the ProcessingVersion objects are
+         pvc_pv1, pvc_pv2, pvc_pv3, and realtime.
+
+    * pvinfo : list of dictionaries, each element of the list has:
+           { 'procver': ProcessingVersion
+             'diaobject': [ list of ( BaseProcessingVersion, priority, bpvkey ) ],
+             'diaobject_position': [ list of ( BaseProcessingVersion, priority, bpvkey ) ],
+             'diasource': [ list of ( BaseProcessingVersion, priority, bpvkey ) ],
+             'diaforcedsource': [ list of ( BaseProcessingVersion, priority, bpvkey ) ],
+           }
+         where bpvkey is is the key to use in the bpvs dictionary.  Each
+         list is sorted from higher priority to lower priority base
+         processing version.
+
+    """
 
     bpvs = {}
     pvs = {}
@@ -267,62 +301,71 @@ def procver_collection( procver_postimes, procver_bases ):
 
 @pytest.fixture( scope='module' )
 def set_of_lightcurves( procver_bases, procver_postimes, procver_collection ):
-    # Define four root objects:
-    #   0-1: 2 within 15" of each other
-    #   2: 1 20" away
-    #   3: 1 much further away
-    #
-    # Objects 0 through 2 are in realtime as [ 0, 1, 2 ]
-    # Object 0 is in bpv1 as 100
-    # All objects are in bpv2 as [ 200, 201, 202, 203 ]
-    #   BUT object 1 is also in bpv2 as 2011
-    #
-    # Positions are stored at procver_postimes *if* the object still has a detection by the time
-    #   (diaobjects 203 and 2011 have no positions stored)
-    #
-    # Objects have lightcurves:
-    #     object 0 : first detection mjd 60000, last detection mjd 60030, peak 60010, mag 24
-    #     object 1 : first detection mjd 60020, last detection mjd 60060, peak 60035, mag 22
-    #     object 2 : first detection mjd 60040, last detection mjd 60080, peak 60050, mag 23
-    #     object 3 : first detection mjd 60050, last detection mjd 60060, peak 60055, mag 23.5
-    #
-    # All lightcurves have a candence of 2.5 days.  (I'm counting on
-    #   floats being perfectly representable in these ranges at
-    #   increments of 2.5, which I believe is true.)
-    #
-    # Full lightcurves are in bpv2 and bpv3
-    # Times [ 60020, 60030 ] have forcedsources in bpv2a
-    # Times [ 60020, 60025 ] have sources in bpv2a
-    # FOR OBJECT 1 IN bpv2 ONLY:
-    #    if mjd = floor(mjd), diasource and diaforcedsource are associated with object 201
-    #       otherwise diasource and diaforcedsource are associated with object 2011
-    #
-    # Photometry through 60060 and forced through 60055 is in realtime for objects 0 through 2
-    #
-    # Object 0:
-    #    photometry through 60015 and forced through 60010 in bpv1a
-    #    photometry through 60030 and forced through 60025 in bpv1
-    #
-    # For all diasource and diaforcedsource:
-    #   in bpv0 they point to the object in bpv0
-    #   in bpv1 they point to the object in bpv1
-    #   in bpv2 or bpv3, they point to the object in bpv2
-    #
-    # RETURN STRUCTURE:
-    # [
-    #    { 'root':  RootDiaObject,
-    #      'obj':   { diaobjectid: DiaObject for all relevant object base processing versions },
-    #      'pos':   { (diaobjectid, bpv): DiaObjectPosition for all relevant position base processing versions },
-    #      'src':   { bpv: [ list of DiaSource ] },
-    #      'srcex': { bpv: [ list of DiaSourceExtra ] },
-    #      'frc':   { bpv: [ list of DiaForcedSource ] },
-    #      'frcex'  { bpv: [ list of DiaForcedSourceExtra ] }
-    #    }
-    # ]
-    #
-    # Ordered for object 0, 1, 2, 3
-    #
-    # all bpv keys are keys into the bpvs dictionary of the procver_collection fixture
+    """Load the database with lightcurves for four root objects.
+
+    Define four root objects:
+      0-1: 2 within 15" of each other
+      2: 1 20" away
+      3: 1 much further away
+
+    Objects 0 through 2 are in realtime as [ 0, 1, 2 ]
+    Object 0 is in bpv1 as 100
+    All objects are in bpv2 as [ 200, 201, 202, 203 ]
+      BUT object 1 is also in bpv2 as 2011
+
+    Positions are stored at procver_postimes *if* the object still has a detection by the time
+      (diaobjects 203 and 2011 have no positions stored)
+
+    Objects have lightcurves:
+        object 0 : first detection mjd 60000, last detection mjd 60030, peak 60010, mag 24
+        object 1 : first detection mjd 60020, last detection mjd 60060, peak 60035, mag 22
+        object 2 : first detection mjd 60040, last detection mjd 60080, peak 60050, mag 23
+        object 3 : first detection mjd 60050, last detection mjd 60060, peak 60055, mag 23.5
+
+    All lightcurves have a candence of 2.5 days.  (I'm counting on
+      floats being perfectly representable in these ranges at
+      increments of 2.5, which I believe is true.)
+
+    Full lightcurves are in bpv2 and bpv3
+    Times [ 60020, 60030 ] have forcedsources in bpv2a
+    Times [ 60020, 60025 ] have sources in bpv2a
+    FOR OBJECT 1 IN bpv2 ONLY:
+       if mjd = floor(mjd), diasource and diaforcedsource are associated with object 201
+          otherwise diasource and diaforcedsource are associated with object 2011
+
+    Photometry through 60060 and forced through 60055 is in realtime for objects 0 through 2
+
+    Object 0:
+       photometry through 60015 and forced through 60010 in bpv1a
+       photometry through 60030 and forced through 60025 in bpv1
+
+    For all diasource and diaforcedsource:
+      in bpv0 they point to the object in bpv0
+      in bpv1 they point to the object in bpv1
+      in bpv2 or bpv3, they point to the object in bpv2
+
+    Yields
+    ------
+      [
+         { 'root':  RootDiaObject,
+           'obj':   { diaobjectid: DiaObject for all relevant object base processing versions },
+           'pos':   { (diaobjectid, bpv): DiaObjectPosition for all relevant position base processing versions },
+           'src':   { bpv: [ list of DiaSource ] },
+           'srcex': { bpv: [ list of DiaSourceExtra ] },
+           'frc':   { bpv: [ list of DiaForcedSource ] },
+           'frcex'  { bpv: [ list of DiaForcedSourceExtra ] }
+         }
+      ]
+
+      Ordered for object 0, 1, 2, 3
+
+      all bpv keys are keys into the bpvs dictionary of the procver_collection fixture
+
+      The list of src and srcex for a given root object and bpv should
+      have the same length, BUT some of the srcex entries may be None.
+      Likewise for frc/frcex.
+
+    """
 
     roots = []
     rootobjs = []
@@ -1517,7 +1560,7 @@ def do_the_things( procver, band=None ):
 
             # There are multiple base processing versions that have data.  We need to
             #  extract the highest priority for each.  To do this, rewrangle
-            #  the set_of_lightcurves data so that it's indexed by bsae_procver,
+            #  the set_of_lightcurves data so that it's indexed by base_procver,
             #  in descending priority order
 
             if band is not None:

From 2e42953b1b6db03934785ebd44e42dc778d4f47b Mon Sep 17 00:00:00 2001
From: Rob Knop <rknop@pobox.com>
Date: Thu, 24 Sep 2026 15:05:53 -0400
Subject: [PATCH 17/35] Deal with fallout of changing how object counts work

---
 src/admin/fastdb_loader.py     |   4 +
 src/admin/load_edp2_parquet.py |   4 +
 src/ltcv.py                    |  85 ++++++++++++---------
 src/webserver/server.py        |  18 ++---
 tests/conftest.py              |  24 ++++++
 tests/webserver/test_server.py | 132 +++++++++++++++++----------------
 6 files changed, 158 insertions(+), 109 deletions(-)

diff --git a/src/admin/fastdb_loader.py b/src/admin/fastdb_loader.py
index 1c5302f..d40cf6a 100644
--- a/src/admin/fastdb_loader.py
+++ b/src/admin/fastdb_loader.py
@@ -31,6 +31,10 @@ class FastDBLoader:
     """
 
     def __init__( self, processing_version=None ):
+        if ( processing_version is not None ) and ( not re.search( '^[a-z0-9_]+$', processing_version ) ):
+            raise ValueError( f"Invalid processing version name {processing_version}, must only include "
+                              f"lowercase letters, numbers, and _" )
+
         self._all_tables = db.all_table_names.copy()
         self._all_tables.remove( "authuser" )
         self._all_tables.remove( "passwordlink" )
diff --git a/src/admin/load_edp2_parquet.py b/src/admin/load_edp2_parquet.py
index 9ebd49f..9cc08b1 100644
--- a/src/admin/load_edp2_parquet.py
+++ b/src/admin/load_edp2_parquet.py
@@ -4,6 +4,7 @@
 import textwrap
 import functools
 import multiprocessing
+import re
 from concurrent.futures import ProcessPoolExecutor
 
 
@@ -19,6 +20,9 @@
 
 class EDP2Loader:
     def __init__( self, processing_version, create_pv=False, test_only=False, slow_test=False, stop_after_n_files=0 ):
+        if not re.search( r'^[a-z0-9_]+$', processing_version ):
+            raise ValueError( "Invalid processing_version, must only include a-z, 0-9, and _" )
+
         self.processing_version_passed = processing_version
         self._test_only = test_only or slow_test
         self._slow_test = slow_test
diff --git a/src/ltcv.py b/src/ltcv.py
index aeea2fa..0bf7f5e 100644
--- a/src/ltcv.py
+++ b/src/ltcv.py
@@ -4,6 +4,7 @@
 import numbers
 import textwrap
 import random
+import re
 import json   # noqa: F401
 
 from psycopg import sql
@@ -1720,20 +1721,30 @@ def get_hot_ltcvs( processing_version, position_processing_version=None,
 
 
 def create_object_stats_materialized_view( procver ):
-    FDBLogger.info( f"Creating or refreshing object statistics materialized views for processing version {procver}" )
     with db.DBCon( dictcursor=True ) as dbcon:
+        # Get the processing version; this will raise an exception if it's not found
+        pvobj = db.ProcessingVersion.get_procver( procver, dbcon=dbcon )
+
+        FDBLogger.info( f"Creating or refreshing object statistics materialized "
+                        f"views for processing version {pvobj.description}" )
+
+        # Do a quick bobby tables check
+        if not re.search( r'^[a-z0-9_]+$', pvobj.description ):
+            raise ValueError( f"Name {pvobj.description} for processing version {pvobj.id} is invalid; "
+                              f"Should only include lowercase letters, numbers, and underscore." )
+
         # Check to see if it already exists
-        q = sql.SQL( "SELECT * FROM pg_class WHERE relname={viewname}" ).format( viewname=f'objstats_{procver}' )
-        rows = dbcon.execute( q )
+        rows = dbcon.execute( sql.SQL( "SELECT * FROM pg_class WHERE relname={viewname}" )
+                              .format( viewname=f'objstats_{pvobj.description}' ) )
         if len(rows) > 0:
             row = rows[0]
             if row['relkind'] != 'm':
-                raise RuntimeError( f"postgres class objstats_{procver} exists, but is not a materialized "
+                raise RuntimeError( f"postgres class objstats_{pvobj.description} exists, but is not a materialized "
                                     f"view!  (It is a \"{row['relkind']}\")" )
             q = sql.SQL( "SELECT a.attname FROM pg_catalog.pg_attribute a "
                          "INNER JOIN pg_class c ON c.oid=a.attrelid "
                          "WHERE c.relname={viewname} AND a.attnum>0"
-                        ).format( viewname=f'objstats_{procver}' )
+                        ).format( viewname=f'objstats_{pvobj.description}' )
             rows = dbcon.execute( q )
             expectedcols = { 'rootid', 'ra', 'dec', 'band',
                              'firstdet_mjd', 'firstdet_flux', 'firstdet_fluxerr',
@@ -1744,38 +1755,39 @@ def create_object_stats_materialized_view( procver ):
                              'nsn10', 'nsn7', 'nsn5', 'nrelgtp50', 'nrelgtp60',
                              'nrelgtp70', 'nrelgtp80', 'nrelgtp90', 'nrelgtp95' }
             if set( r['attname'] for r in rows ) != expectedcols:
-                raise RuntimeError( f"postgres view objstats_{procver} has the wrong set of columns" )
+                raise RuntimeError( f"postgres view objstats_{pvobj.description} has the wrong set of columns" )
 
             q = sql.SQL( "SELECT * FROM pg_class WHERE relname={viewname}"
-                        ).format( viewname=f'objstatscomb_{procver}' )
+                        ).format( viewname=f'objstatscomb_{pvobj.description}' )
             rows = dbcon.execute( q )
             if len(rows) == 0:
-                raise RuntimeError( f"view objstats_{procver} exists, but objstatscomb_{procver} does not" )
+                raise RuntimeError( f"view objstats_{pvobj.description} exists, "
+                                    f"but objstatscomb_{pvobj.description} does not" )
             row = rows[0]
             if row['relkind'] != 'm':
-                raise RuntimeError( f"postgres class objstatscomb_{procver} exists, but is not a materialized "
-                                    f"view!  (It is a \"{row['relkind']}\")" )
+                raise RuntimeError( f"postgres class objstatscomb_{pvobj.description} exists, but is not a "
+                                    f"materialized view!  (It is a \"{row['relkind']}\")" )
             q = sql.SQL( "SELECT a.attname FROM pg_catalog.pg_attribute a "
                          "INNER JOIN pg_class c ON c.oid=a.attrelid "
                          "WHERE c.relname={viewname} AND a.attnum>0"
-                        ).format( viewname=f'objstatscomb_{procver}' )
+                        ).format( viewname=f'objstatscomb_{pvobj.description}' )
             rows = dbcon.execute( q )
             if set( r['attname'] for r in rows ) != ( expectedcols.union( { 'firstdet_band', 'lastdet_band',
                                                                             'maxdet_band', 'lastforced_band' } )
                                                       - { 'band' } ):
-                raise RuntimeError( f"postgrew view objstatscomb_{procver} has the wrong set of columns" )
+                raise RuntimeError( f"postgrew view objstatscomb_{pvobj.description} has the wrong set of columns" )
 
 
-            FDBLogger.info( f"Refreshing materizalized view objstats_{procver}" )
+            FDBLogger.info( f"Refreshing materizalized view objstats_{pvobj.description}" )
             q = sql.SQL( "REFRESH MATERIALIZED VIEW {viewname}"
-                        ).format( viewname=sql.Identifier( f'objstats_{procver}' ) )
+                        ).format( viewname=sql.Identifier( f'objstats_{pvobj.description}' ) )
             dbcon.execute( q )
-            FDBLogger.info( f"Refreshing materizalized view objstatscomb_{procver}" )
+            FDBLogger.info( f"Refreshing materizalized view objstatscomb_{pvobj.description}" )
             q = sql.SQL( "REFRESH MATERIALIZED VIEW {viewname}"
-                        ).format( viewname=sql.Identifier( f'objstatscomb_{procver}' ) )
+                        ).format( viewname=sql.Identifier( f'objstatscomb_{pvobj.description}' ) )
             dbcon.execute( q )
             dbcon.commit()
-            FDBLogger.info( f"Done refreshing object statistics materialized views for {procver}" )
+            FDBLogger.info( f"Done refreshing object statistics materialized views for {pvobj.description}" )
             return
 
         # If we get here, the materialized view does not exist
@@ -1787,8 +1799,7 @@ def create_object_stats_materialized_view( procver ):
         #     m = 22 : f =  5754
         #     m = 21 : f = 14454
 
-        FDBLogger.info( f"Creating materialized view objstats_{procver}" )
-        pvid = db.ProcessingVersion.procver_id( procver, dbcon=dbcon )
+        FDBLogger.info( f"Creating materialized view objstats_{pvobj.description}" )
 
         q = sql.SQL( textwrap.dedent(
             """\
@@ -1906,7 +1917,7 @@ def create_object_stats_materialized_view( procver ):
                   GROUP BY rootid, band
                ) nf ON r.rootid=nf.rootid AND r.band=nf.band
             """
-        ) ).format( viewname=sql.Identifier( f'objstats_{procver}' ), pvid=pvid )
+        ) ).format( viewname=sql.Identifier( f'objstats_{pvobj.description}' ), pvid=pvobj.id )
         for alias, flux in zip( [ 'n24', 'n23', 'n22', 'n21' ],
                                 [ 912, 2291, 5754, 14454 ] ):
             q += sql.SQL( textwrap.indent( textwrap.dedent(
@@ -1925,7 +1936,7 @@ def create_object_stats_materialized_view( procver ):
                   GROUP BY rootid, band
                ) {alias} ON r.rootid={alias}.rootid AND r.band={alias}.band
                 """
-                ), '   ' ) ).format( pvid=pvid, flux=sql.SQL(str(flux)), alias=sql.Identifier(alias) )
+                ), '   ' ) ).format( pvid=pvobj.id, flux=sql.SQL(str(flux)), alias=sql.Identifier(alias) )
         for alias, sn in zip( [ 'sn10', 'sn7', 'sn5' ], [ 10, 7, 5 ] ):
             q += sql.SQL( textwrap.indent( textwrap.dedent(
                 """\
@@ -1943,7 +1954,7 @@ def create_object_stats_materialized_view( procver ):
                   GROUP BY rootid, band
                ) {alias} ON r.rootid={alias}.rootid AND r.band={alias}.band
                 """
-                ), '   ' ) ).format( pvid=pvid, sn=sql.SQL(str(sn)), alias=sql.Identifier(alias) )
+                ), '   ' ) ).format( pvid=pvobj.id, sn=sql.SQL(str(sn)), alias=sql.Identifier(alias) )
         for alias, rel in zip( [ 'relp5', 'relp6', 'relp7', 'relp8', 'relp9', 'relp95' ],
                                [ 0.5, 0.6, 0.7, 0.8, 0.9, 0.95 ] ):
             q += sql.SQL( textwrap.indent( textwrap.dedent(
@@ -1964,12 +1975,12 @@ def create_object_stats_materialized_view( procver ):
                  GROUP BY rootid, band
                ) {alias} ON r.rootid={alias}.rootid AND r.band={alias}.band
                 """
-                ), '   ' ) ).format( pvid=pvid, rel=sql.SQL(f'{rel:.1f}'), alias=sql.Identifier(alias) )
+                ), '   ' ) ).format( pvid=pvobj.id, rel=sql.SQL(f'{rel:.1f}'), alias=sql.Identifier(alias) )
         q += sql.SQL( textwrap.dedent(
             """\
             )
             """
-        ) ).format( viewname=sql.Identifier( f'objstats_{procver}' ), pvid=pvid )
+        ) ).format( viewname=sql.Identifier( f'objstats_{pvobj.description}' ), pvid=pvobj.id )
 
         dbcon.execute_nofetch( q, explain=True )
 
@@ -1980,18 +1991,18 @@ def create_object_stats_materialized_view( procver ):
                      'nrelgtp50', 'nrelgtp60', 'nrelgtp70', 'nrelgtp80', 'nrelgtp90', 'nrelgtp95' ]
         for col in indexcols:
             q = sql.SQL( 'CREATE INDEX {idxname} ON {viewname}({col})'
-                        ).format( idxname=sql.Identifier( f'idx_obstats_{procver}_{col}' ),
-                                  viewname=sql.Identifier( f'objstats_{procver}' ),
+                        ).format( idxname=sql.Identifier( f'idx_obstats_{pvobj.description}_{col}' ),
+                                  viewname=sql.Identifier( f'objstats_{pvobj.description}' ),
                                   col=sql.Identifier( col ) )
             dbcon.execute( q, explain=False )
 
         q = sql.SQL( 'CREATE INDEX {idxname} ON {viewname}(band)',
-                    ).format( idxname=sql.Identifier( f'idx_obstats_{procver}_band' ),
-                              viewname=sql.Identifier( f'objstats_{procver}' ) )
+                    ).format( idxname=sql.Identifier( f'idx_obstats_{pvobj.description}_band' ),
+                              viewname=sql.Identifier( f'objstats_{pvobj.description}' ) )
 
         q = sql.SQL( 'CREATE INDEX {idxname} ON {viewname}(q3c_ang2ipix(ra, dec))'
-                    ).format( idxname=sql.Identifier( f'idx_objstats_{procver}_q3c' ),
-                              viewname=sql.Identifier( f'objstats_{procver}' ) )
+                    ).format( idxname=sql.Identifier( f'idx_objstats_{pvobj.description}_q3c' ),
+                              viewname=sql.Identifier( f'objstats_{pvobj.description}' ) )
         dbcon.execute( q, explain=False )
 
         # Now create the view that combines all the bands together
@@ -2052,21 +2063,21 @@ def create_object_stats_materialized_view( procver ):
               ) lf ON s.rootid=lf.rootid
               WHERE fd.mjd IS NOT NULL OR lf.mjd IS NOT NULL
             )
-            """ ) ).format( viewname=sql.Identifier(f'objstats_{procver}'),
-                            combviewname=sql.Identifier(f'objstatscomb_{procver}') )
+            """ ) ).format( viewname=sql.Identifier(f'objstats_{pvobj.description}'),
+                            combviewname=sql.Identifier(f'objstatscomb_{pvobj.description}') )
         dbcon.execute( q, explain=True )
 
         for col in indexcols:
             q = sql.SQL( 'CREATE INDEX {idxname} ON {viewname}({col})'
-                        ).format( idxname=sql.Identifier( f'idx_objstatscomb_{procver}_{col}' ),
-                                  viewname=sql.Identifier( f'objstatscomb_{procver}' ),
+                        ).format( idxname=sql.Identifier( f'idx_objstatscomb_{pvobj.description}_{col}' ),
+                                  viewname=sql.Identifier( f'objstatscomb_{pvobj.description}' ),
                                   col=sql.Identifier( col ) )
             dbcon.execute( q, explain=False )
 
         q = sql.SQL( 'CREATE INDEX {idxname} ON {viewname}(q3c_ang2ipix(ra, dec))'
-                    ).format( idxname=sql.Identifier( f'idx_objstatscomb_{procver}_q3c' ),
-                              viewname=sql.Identifier( f'objstatscomb_{procver}' ) )
+                    ).format( idxname=sql.Identifier( f'idx_objstatscomb_{pvobj.description}_q3c' ),
+                              viewname=sql.Identifier( f'objstatscomb_{pvobj.description}' ) )
         dbcon.execute( q, explain=False )
 
         dbcon.commit()
-        FDBLogger.info( f"Done creating object statistics materialized views for {procver}" )
+        FDBLogger.info( f"Done creating object statistics materialized views for {pvobj.description}" )
diff --git a/src/webserver/server.py b/src/webserver/server.py
index c8aaaeb..def92f9 100644
--- a/src/webserver/server.py
+++ b/src/webserver/server.py
@@ -153,18 +153,18 @@ def do_the_things( self, which, procver='default' ):
 
         with db.DBCon() as dbcon:
             try:
-                # Just make sure this is a know processing version, even though
-                #   we aren't actually going to use it.
-                # This is a necessary but not sufficient condition; the
-                #   objstatscomb materialized view for this processing
-                #   version must also exist.  TODO: think about whether
-                #   I want to interpret aliases for finding the
-                #   objstatscomb table.  Temptation: yes.
-                _pvid = db.ProcessingVersion.procver_id( procver )
+                # Get the processing version
+                procver = db.ProcessingVersion.get_procver( procver )
+                # Bobby table check.  One that's not valid should never have passed in the first place,
+                #  but be paranoid.
+                if not re.search( '^[a-z0-9_]+$', procver.description ):
+                    raise ValueError( f"Invalid processing version name \"{procver.description}\" for "
+                                      f"processing version {procver.id}; should only include "
+                                      f"a-z, 0-9, and _" )
             except Exception as ex:
                 raise FASTDBWebException( str(ex) )
 
-            table = sql.Identifier( f"objstatscomb_{procver}" )
+            table = sql.Identifier( f"objstatscomb_{procver.description}" )
 
             if thingtocount == 'rootid':
                 rows, _cols = dbcon.execute( sql.SQL("SELECT COUNT(rootid) FROM {table}").format( table=table ) )
diff --git a/tests/conftest.py b/tests/conftest.py
index 8b20dc8..5370792 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1519,6 +1519,30 @@ def objstats_realtime_view( procver_collection ):
             con.commit()
 
 
+@pytest.fixture( scope="module" )
+def objstats_pvc_pv2_view( procver_collection, set_of_lightcurves ):
+    try:
+        ltcv.create_object_stats_materialized_view( 'pvc_pv2' )
+        yield True
+    finally:
+        with DBCon() as con:
+            con.execute_nofetch( "DROP MATERIALIZED VIEW IF EXISTS objstatscomb_pvc_pv2" )
+            con.execute_nofetch( "DROP MATERIALIZED VIEW IF EXISTS objstats_pvc_pv2" )
+            con.commit()
+
+
+@pytest.fixture( scope="module" )
+def objstats_pvc_pv3_view( procver_collection, set_of_lightcurves ):
+    try:
+        ltcv.create_object_stats_materialized_view( 'pvc_pv3' )
+        yield True
+    finally:
+        with DBCon() as con:
+            con.execute_nofetch( "DROP MATERIALIZED VIEW IF EXISTS objstatscomb_pvc_pv3" )
+            con.execute_nofetch( "DROP MATERIALIZED VIEW IF EXISTS objstats_pvc_pv3" )
+            con.commit()
+
+
 @pytest.fixture( scope='module' )
 def accumulate_expected_stats( set_of_lightcurves, procver_collection ):
     def do_the_things( procver, band=None ):
diff --git a/tests/webserver/test_server.py b/tests/webserver/test_server.py
index eea126d..a47d8cb 100644
--- a/tests/webserver/test_server.py
+++ b/tests/webserver/test_server.py
@@ -106,69 +106,6 @@ def test_base_procver( procver_collection, test_user, fastdb_client ):
                 assert res['procvers'] == [ pv ]
 
 
-def test_countthings( set_of_lightcurves, test_user, fastdb_client ):
-    for pv in ( '', 'pvc_pv2', 'pvc_pv3', 'default' ):
-        for table in ( 'rootobject', 'rootdiaobject', 'rootid' ):
-            suffix = table if pv == '' else f'{table}/{pv}'
-            res = fastdb_client.post( f'/count/{suffix}' )
-            assert res['status'] == 'ok'
-            assert res['table'] == 'rootid'
-            assert res['count'] == ( 0 if pv == 'pvc_pv3' else 4 )
-
-        for table in ( 'diaobject', 'object' ):
-            suffix = table if pv == '' else f'{table}/{pv}'
-            res = fastdb_client.post( f'/count/{suffix}' )
-            assert res['status'] == 'ok'
-            assert res['table'] == 'diaobject'
-            assert res['count'] == ( 0 if pv == 'pvc_pv3' else 5 )
-
-        for table in ( 'diasource', 'source' ):
-            suffix = table if pv == '' else f'{table}/{pv}'
-            res = fastdb_client.post( f'/count/{suffix}' )
-            assert res['status'] == 'ok'
-            assert res['table'] == 'diasource'
-            assert res['count'] == 52
-
-        for table in ( 'diaforcedsource', 'forced' ):
-            suffix = table if pv == '' else f'{table}/{pv}'
-            res = fastdb_client.post( f'/count/{suffix}' )
-            assert res['status'] == 'ok'
-            assert res['table'] == 'diaforcedsource'
-            assert res['count'] == 100
-
-
-    for table in ( 'diaobject', 'object', 'rootobject', 'rootdiaobject', 'rootid' ):
-        res = fastdb_client.post( f'/count/{table}/realtime' )
-        assert res['status'] == 'ok'
-        assert res['table'] == ( 'diaobject' if table in ( 'diaobject', 'object' ) else 'rootid' )
-        assert res['count'] == 3
-
-    for table in ( 'diasource', 'source' ):
-        res = fastdb_client.post( f'/count/{table}/realtime' )
-        assert res['status'] == 'ok'
-        assert res['table'] == 'diasource'
-        assert res['count'] == 39
-
-    for table in ( 'diaforcedsource', 'forced' ):
-        res = fastdb_client.post( f'/count/{table}/realtime' )
-        assert res['status'] == 'ok'
-        assert res['table'] == 'diaforcedsource'
-        assert res['count'] == 55
-
-    # Temporarily reduce retries to 0 so these will fail fast
-    orig_retries = fastdb_client.retries
-    try:
-        fastdb_client.retries = 0
-        with pytest.raises( RuntimeError, match=( 'Error response from server, status 422: Unknown '
-                                                  'processing version this_processing_version_does_not_exist' ) ):
-            res = fastdb_client.post( '/count/diaobject/this_processing_version_does_not_exist' )
-        with pytest.raises( RuntimeError, match=( 'Error response from server, status 422: '
-                                                  'Unknown thing to count: this_table_does_not_exist' ) ):
-            res = fastdb_client.post( '/count/this_table_does_not_exist' )
-    finally:
-        fastdb_client.retries = orig_retries
-
-
 def test_getdiaobjectinfo( fastdb_client, procver_collection, set_of_lightcurves ):
     roots = set_of_lightcurves
 
@@ -322,6 +259,75 @@ def test_objectsearch( fastdb_client, procver_collection, objstats_realtime_view
             con.commit()
 
 
+# THIS TEST MUST GO AFTER test_objectsearch BECASUE THAT TEST DEPENDS ON
+#   THE objstats_pvc_pv2_view AND objstats_pvc_pv3_view MODULE SCOPE FIXTURES
+#   NOT HAVING BEEN RUN YET.
+def test_countthings( set_of_lightcurves, objstats_realtime_view, objstats_pvc_pv2_view,
+                      objstats_pvc_pv3_view, test_user, fastdb_client ):
+    for pv in ( '', 'pvc_pv2', 'pvc_pv3', 'default' ):
+        for table in ( 'rootobject', 'rootdiaobject', 'rootid' ):
+            suffix = table if pv == '' else f'{table}/{pv}'
+            res = fastdb_client.post( f'/count/{suffix}' )
+            assert res['status'] == 'ok'
+            assert res['table'] == 'rootid'
+            # Even though there are no diaobjectids in pv3, the rootids are there
+            assert res['count'] == 4
+
+        # Counting diabobject isn't understood
+        # for table in ( 'diaobject', 'object' ):
+        #     suffix = table if pv == '' else f'{table}/{pv}'
+        #     res = fastdb_client.post( f'/count/{suffix}' )
+        #     assert res['status'] == 'ok'
+        #     assert res['table'] == 'diaobject'
+        #     assert res['count'] == ( 0 if pv == 'pvc_pv3' else 5 )
+
+        for table in ( 'diasource', 'source' ):
+            suffix = table if pv == '' else f'{table}/{pv}'
+            res = fastdb_client.post( f'/count/{suffix}' )
+            assert res['status'] == 'ok'
+            assert res['table'] == 'diasource'
+            assert res['count'] == 52
+
+        for table in ( 'diaforcedsource', 'forced' ):
+            suffix = table if pv == '' else f'{table}/{pv}'
+            res = fastdb_client.post( f'/count/{suffix}' )
+            assert res['status'] == 'ok'
+            assert res['table'] == 'diaforcedsource'
+            assert res['count'] == 100
+
+
+    for table in ( 'rootobject', 'rootdiaobject', 'rootid' ):
+        res = fastdb_client.post( f'/count/{table}/realtime' )
+        assert res['status'] == 'ok'
+        assert res['table'] == ( 'diaobject' if table in ( 'diaobject', 'object' ) else 'rootid' )
+        assert res['count'] == 3
+
+    for table in ( 'diasource', 'source' ):
+        res = fastdb_client.post( f'/count/{table}/realtime' )
+        assert res['status'] == 'ok'
+        assert res['table'] == 'diasource'
+        assert res['count'] == 39
+
+    for table in ( 'diaforcedsource', 'forced' ):
+        res = fastdb_client.post( f'/count/{table}/realtime' )
+        assert res['status'] == 'ok'
+        assert res['table'] == 'diaforcedsource'
+        assert res['count'] == 55
+
+    # Temporarily reduce retries to 0 so these will fail fast
+    orig_retries = fastdb_client.retries
+    try:
+        fastdb_client.retries = 0
+        with pytest.raises( RuntimeError, match=( 'Error response from server, status 422: Unknown '
+                                                  'processing version this_processing_version_does_not_exist' ) ):
+            res = fastdb_client.post( '/count/root/this_processing_version_does_not_exist' )
+        with pytest.raises( RuntimeError, match=( 'Error response from server, status 422: '
+                                                  'Unknown thing to count: this_table_does_not_exist' ) ):
+            res = fastdb_client.post( '/count/this_table_does_not_exist' )
+    finally:
+        fastdb_client.retries = orig_retries
+
+
 # ======================================================================
 
 # This next one loads up with the database with the ELAsTiCC2 excerpt

From 0fcc01e5a8449e97e5a18c60f192b10cd9a99611 Mon Sep 17 00:00:00 2001
From: Rob Knop <rknop@pobox.com>
Date: Mon, 26 Oct 2026 09:37:39 -0700
Subject: [PATCH 18/35] Trying to figure out why a container fails to start on
 github runner

---
 .github/workflows/run-fastdb-tests.yaml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/run-fastdb-tests.yaml b/.github/workflows/run-fastdb-tests.yaml
index c8adcfd..ec0d432 100644
--- a/.github/workflows/run-fastdb-tests.yaml
+++ b/.github/workflows/run-fastdb-tests.yaml
@@ -57,4 +57,11 @@ jobs:
       - name: run test
         run: |
           docker compose run --rm runtests
+
+      - name: TEMP: mongo log
+        run: |
+          docker compose logs mongodb
+
+      - name: bring docker down
+        run: |
           docker compose down -v

From 6dfc854472dca62ee89f1381b07ab23e1c3aa5ef Mon Sep 17 00:00:00 2001
From: Rob Knop <rknop@pobox.com>
Date: Mon, 26 Oct 2026 09:39:12 -0700
Subject: [PATCH 19/35] tuneup last commit

---
 .github/workflows/run-fastdb-tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-fastdb-tests.yaml b/.github/workflows/run-fastdb-tests.yaml
index ec0d432..aff6ea9 100644
--- a/.github/workflows/run-fastdb-tests.yaml
+++ b/.github/workflows/run-fastdb-tests.yaml
@@ -56,7 +56,7 @@ jobs:
 
       - name: run test
         run: |
-          docker compose run --rm runtests
+          docker compose run runtests
 
       - name: TEMP: mongo log
         run: |

From a0cbca62fdbac5c6b912412dc23231721f556259 Mon Sep 17 00:00:00 2001
From: Rob Knop <rknop@pobox.com>
Date: Mon, 26 Oct 2026 09:41:27 -0700
Subject: [PATCH 20/35] fighting with github

---
 .github/workflows/run-fastdb-tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-fastdb-tests.yaml b/.github/workflows/run-fastdb-tests.yaml
index aff6ea9..ae14cc2 100644
--- a/.github/workflows/run-fastdb-tests.yaml
+++ b/.github/workflows/run-fastdb-tests.yaml
@@ -58,7 +58,7 @@ jobs:
         run: |
           docker compose run runtests
 
-      - name: TEMP: mongo log
+      - name: TEMP - mongo log
         run: |
           docker compose logs mongodb
 

From 93cc2f570f263ee5d1ca887dfcc69f832c14f6be Mon Sep 17 00:00:00 2001
From: Rob Knop <rknop@pobox.com>
Date: Mon, 26 Oct 2026 09:46:00 -0700
Subject: [PATCH 21/35] ....having a hard time getting log information to debug
 what's wrong on github.  All tests are working locally.  (Sink picture.)

---
 .github/workflows/run-fastdb-tests.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/run-fastdb-tests.yaml b/.github/workflows/run-fastdb-tests.yaml
index ae14cc2..88c1902 100644
--- a/.github/workflows/run-fastdb-tests.yaml
+++ b/.github/workflows/run-fastdb-tests.yaml
@@ -56,7 +56,8 @@ jobs:
 
       - name: run test
         run: |
-          docker compose run runtests
+          docker compose up -d webap shell
+          # docker compose run runtests
 
       - name: TEMP - mongo log
         run: |

From e4cf648001fbd96209be5a284d3b133f30a4fa8b Mon Sep 17 00:00:00 2001
From: Rob Knop <rknop@pobox.com>
Date: Mon, 26 Oct 2026 09:55:57 -0700
Subject: [PATCH 22/35] debugging github workflow errors is painful

---
 .github/workflows/run-fastdb-tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-fastdb-tests.yaml b/.github/workflows/run-fastdb-tests.yaml
index 88c1902..3c3f2d1 100644
--- a/.github/workflows/run-fastdb-tests.yaml
+++ b/.github/workflows/run-fastdb-tests.yaml
@@ -56,8 +56,8 @@ jobs:
 
       - name: run test
         run: |
-          docker compose up -d webap shell
           # docker compose run runtests
+          docker compose up -d mongodb
 
       - name: TEMP - mongo log
         run: |

From c741fb50911890c9417228e0eafc3aacd568b689 Mon Sep 17 00:00:00 2001
From: Rob Knop <rknop@pobox.com>
Date: Mon, 26 Oct 2026 10:01:32 -0700
Subject: [PATCH 23/35] Bump base version of mongo for mongo dockerfile; 8.0.4
 didn't work with the version of the Linux kernel on github runners.  (Docker
 is fake virtualization...)

---
 .github/workflows/run-fastdb-tests.yaml |  2 +-
 docker-compose.yaml                     | 20 ++++++++++----------
 docker/mongodb/Dockerfile               |  2 +-
 3 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/run-fastdb-tests.yaml b/.github/workflows/run-fastdb-tests.yaml
index 3c3f2d1..f969540 100644
--- a/.github/workflows/run-fastdb-tests.yaml
+++ b/.github/workflows/run-fastdb-tests.yaml
@@ -56,7 +56,7 @@ jobs:
 
       - name: run test
         run: |
-          # docker compose run runtests
+          # docker compose run --rm runtests
           docker compose up -d mongodb
 
       - name: TEMP - mongo log
diff --git a/docker-compose.yaml b/docker-compose.yaml
index 4b28d91..69fa288 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -3,7 +3,7 @@
 
 services:
   kafka-server:
-    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-kafka-test:${DOCKER_VERSION-test20260918a}
+    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-kafka-test:${DOCKER_VERSION-test20261026a}
     build:
       context: ./docker/kafka
     environment:
@@ -32,7 +32,7 @@ services:
       retries: 10
 
   postgres:
-    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-postgres:${DOCKER_VERSION:-test20260918a}
+    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-postgres:${DOCKER_VERSION:-test20261026a}
     build:
       context: ./docker/postgres
       target: postgres
@@ -53,7 +53,7 @@ services:
       retries: 10
 
   mongodb:
-    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-mongodb:${DOCKER_VERSION:-test20260918a}
+    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-mongodb:${DOCKER_VERSION:-test20261026a}
     build:
       context: ./docker/mongodb
     environment:
@@ -80,7 +80,7 @@ services:
     depends_on:
       postgres:
         condition: service_healthy
-    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-shell:${DOCKER_VERSION:-test20260918a}
+    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-shell:${DOCKER_VERSION:-test20261026a}
     build:
       context: ./docker/webserver
       target: shell
@@ -101,7 +101,7 @@ services:
     depends_on:
       createdb:
         condition: service_completed_successfully
-    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-query-runner:${DOCKER_VERSION:-test20260918a}
+    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-query-runner:${DOCKER_VERSION:-test20261026a}
     build:
       context: ./docker/query_runner
       target: queryrunner
@@ -132,7 +132,7 @@ services:
         condition: service_started
       queryrunner:
         condition: service_started
-    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-webap:${DOCKER_VERSION:-test20260918a}
+    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-webap:${DOCKER_VERSION:-test20261026a}
     build:
       context: ./docker/webserver
       target: webserver
@@ -188,7 +188,7 @@ services:
         condition: service_healthy
       createdb:
         condition: service_completed_successfully
-    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-shell:${DOCKER_VERSION:-test20260918a}
+    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-shell:${DOCKER_VERSION:-test20261026a}
     environment:
       - MONGODB_HOST=mongodb
       - MONGODB_DBNAME=brokeralert
@@ -236,7 +236,7 @@ services:
         condition: service_healthy
       mongodb:
         condition: service_healthy
-    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-shell:${DOCKER_VERSION:-test20260918a}
+    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-shell:${DOCKER_VERSION:-test20261026a}
     environment:
       - MONGODB_HOST=mongodb
       - MONGODB_DBNAME=brokeralert
@@ -279,7 +279,7 @@ services:
     working_dir: /code
 
   makeinstall:
-    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-shell:${DOCKER_VERSION:-test20260918a}
+    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-shell:${DOCKER_VERSION:-test20261026a}
     user: ${USERID:-0}:${GROUPID:-0}
     volumes:
       - type: bind
@@ -314,7 +314,7 @@ services:
         condition: service_started   # TODO : health test for webap
       makeinstall:
         condition: service_completed_successfully
-    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-shell:${DOCKER_VERSION:-test20260918a}
+    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-shell:${DOCKER_VERSION:-test20261026a}
     environment:
       - MONGODB_HOST=mongodb
       - MONGODB_DBNAME=brokeralert
diff --git a/docker/mongodb/Dockerfile b/docker/mongodb/Dockerfile
index 424de34..92bc49c 100644
--- a/docker/mongodb/Dockerfile
+++ b/docker/mongodb/Dockerfile
@@ -1,4 +1,4 @@
-FROM mongo:8.0.15 AS base
+FROM mongo:9.0.4 AS base
 
 SHELL ["/bin/bash", "-c"]
 WORKDIR /usr/src

From 095bf6e805fb68fdfb5876c38dfd2c4c30eed1d0 Mon Sep 17 00:00:00 2001
From: Rob Knop <rknop@pobox.com>
Date: Mon, 26 Oct 2026 10:04:17 -0700
Subject: [PATCH 24/35] Boldly put the tests back to the full run in hopes that
 the mongo change from the last commit is all we need

---
 .github/workflows/run-fastdb-tests.yaml | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/run-fastdb-tests.yaml b/.github/workflows/run-fastdb-tests.yaml
index f969540..d7732db 100644
--- a/.github/workflows/run-fastdb-tests.yaml
+++ b/.github/workflows/run-fastdb-tests.yaml
@@ -56,13 +56,13 @@ jobs:
 
       - name: run test
         run: |
-          # docker compose run --rm runtests
-          docker compose up -d mongodb
+          docker compose run --rm runtests
+      #     docker compose up -d mongodb
 
-      - name: TEMP - mongo log
-        run: |
-          docker compose logs mongodb
+      # - name: TEMP - mongo log
+      #   run: |
+      #     docker compose logs mongodb
 
-      - name: bring docker down
-        run: |
-          docker compose down -v
+      # - name: bring docker down
+      #   run: |
+      #     docker compose down -v

From 0547f050951200ef6cea5cc94203844435e18505 Mon Sep 17 00:00:00 2001
From: Rob Knop <rknop@pobox.com>
Date: Mon, 26 Oct 2026 10:08:19 -0700
Subject: [PATCH 25/35] cleanup cruft

---
 .github/workflows/run-fastdb-tests.yaml | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/.github/workflows/run-fastdb-tests.yaml b/.github/workflows/run-fastdb-tests.yaml
index d7732db..bbce8b2 100644
--- a/.github/workflows/run-fastdb-tests.yaml
+++ b/.github/workflows/run-fastdb-tests.yaml
@@ -57,12 +57,3 @@ jobs:
       - name: run test
         run: |
           docker compose run --rm runtests
-      #     docker compose up -d mongodb
-
-      # - name: TEMP - mongo log
-      #   run: |
-      #     docker compose logs mongodb
-
-      # - name: bring docker down
-      #   run: |
-      #     docker compose down -v

From 61c0d85693e0384709a71134cfbed1298824b164 Mon Sep 17 00:00:00 2001
From: Rob Knop <rknop@pobox.com>
Date: Mon, 26 Oct 2026 10:23:36 -0700
Subject: [PATCH 26/35] Remove ten million uses of "inplace" in pandas to get
 rid of deprecation warnings

---
 src/admin/csv_to_avsc.py      |  2 +-
 src/admin/load_dp1_parquet.py | 16 ++++++-------
 src/ltcv.py                   |  8 +++----
 src/spectrum.py               | 42 +++++++++++++++++------------------
 src/util.py                   |  2 +-
 tests/test_ltcv.py            |  2 +-
 6 files changed, 36 insertions(+), 36 deletions(-)

diff --git a/src/admin/csv_to_avsc.py b/src/admin/csv_to_avsc.py
index c4c6946..56ed1e6 100644
--- a/src/admin/csv_to_avsc.py
+++ b/src/admin/csv_to_avsc.py
@@ -22,7 +22,7 @@ def main():
     schema = pandas.read_csv( args.csvfile, quoting=csv.QUOTE_ALL )
     # pandas spazzes out with iterrows and ittertuples when
     #  column names have a space in them, so remove the spaces.
-    schema.rename( { 'Data Type': 'Data_Type', 'Column Name': 'Column_Name' }, axis=1, inplace=True )
+    schema = schema.rename( { 'Data Type': 'Data_Type', 'Column Name': 'Column_Name' }, axis=1 )
 
     typeconv = {
         'boolean': 'boolean',
diff --git a/src/admin/load_dp1_parquet.py b/src/admin/load_dp1_parquet.py
index bdd485e..7ce20d9 100644
--- a/src/admin/load_dp1_parquet.py
+++ b/src/admin/load_dp1_parquet.py
@@ -33,8 +33,8 @@ def _map_columns( cls, tab, mapper, lcs ):
             else:
                 yanks.append( col )
 
-        tab.rename( renames, axis='columns', inplace=True )
-        tab.drop( yanks, axis='columns', inplace=True )
+        tab = tab.rename( renames, axis='columns' )
+        tab = tab.drop( yanks, axis='columns' )
 
 
     @classmethod
@@ -148,12 +148,12 @@ def load_one_file( self, filepath ):
             #   as the index of the parent dataframe?  If so,
             #   that might be a hint to speeding up the
             #   extraction above....  Hurm.  Really not sure.
-            df.reset_index( inplace=True )
-            df.drop( ['index'], axis='columns', inplace=True )
-            sourcedf.reset_index( inplace=True )
-            sourcedf.drop( ['index'], axis='columns', inplace=True )
-            forceddf.reset_index( inplace=True )
-            forceddf.drop( ['index'], axis='columns', inplace=True )
+            df = df.reset_index()
+            df = df.drop( ['index'], axis='columns' )
+            sourcedf = sourcedf.reset_index()
+            sourcedf = sourcedf.drop( ['index'], axis='columns' )
+            forceddf = forceddf.reset_index()
+            forceddf = forceddf.drop( ['index'], axis='columns' )
 
             self.logger.info( f"Going to try to load {len(df)} objects, {len(sourcedf)} sources, "
                               f"{len(forceddf)} forced" )
diff --git a/src/ltcv.py b/src/ltcv.py
index 0bf7f5e..bfe09f8 100644
--- a/src/ltcv.py
+++ b/src/ltcv.py
@@ -327,7 +327,7 @@ def get_object_infos( objids=None, objids_table=None, processing_version=None, p
                    else pandas.array(v, dtype='object') )
               for k, v in rval.items() }
         )
-        df.set_index( 'rootid', inplace=True )
+        df = df.set_index( 'rootid' )
         FDBLogger.debug( "get_object_infos done, returning a Pandas dataframe" )
         return df
 
@@ -900,9 +900,9 @@ def _extract_rowcache():
                     if list( objinfo.index.values ) != [ x['rootid'] for x in ltcvs ]:
                         raise ValueError( "get_object_info rootids doesn't match manyltcv's!" )
                     if not include_base_procver:
-                        objinfo.drop( 'obj_base_procver', axis='columns', inplace=True )
+                        objinfo = objinfo.drop( 'obj_base_procver', axis='columns' )
                         if return_diaobject_positions:
-                            objinfo.drop( 'pos_base_procver', axis='columns', inplace=True )
+                            objinfo = objinfo.drop( 'pos_base_procver', axis='columns' )
                 elif return_format == 'json':
                     keepers = [ i for i, r in enumerate(objinfo['rootid']) if r in ltcvrootidsset ]
                     objinfo = { k: [ v[k] for k in keepers ] for k, v in objinfo.items() }
@@ -1040,7 +1040,7 @@ def _extract_rowcache():
                                          for k, v in row.items() }
                                        for row in ltcvs ] )
         pd_ltcvs = pd_ltcvs.explode( [ c for c in pd_ltcvs.columns if c != 'rootid' ] ).astype( dtypes )
-        pd_ltcvs.set_index( ['rootid', 'mjd'], inplace=True )
+        pd_ltcvs = pd_ltcvs.set_index( ['rootid', 'mjd'] )
         ltcvs = pd_ltcvs
 
     FDBLogger.debug( "...done with many_object_ltcvs" )
diff --git a/src/spectrum.py b/src/spectrum.py
index 876f022..66a8e89 100644
--- a/src/spectrum.py
+++ b/src/spectrum.py
@@ -238,7 +238,7 @@ def what_spectra_are_wanted( procver='realtime', position_procver=None,
         rows, cols = con.execute( "SELECT * FROM tmp_wanted_no_spec" )
         df = util.laboriously_construct_pandas( rows, columns=cols, doublecols=['ra', 'dec'],
                                                 int16cols=['priority'], ignore_missing_cols=True )
-        df.set_index( 'rootid', inplace=True )
+        df = df.set_index( 'rootid' )
 
         # OK, this is a little profligate.  We could definitely pull less from postgres by doing
         #   the "find latest detection" in SQL (and indeed I used to do it that way).  Not clear
@@ -253,14 +253,14 @@ def what_spectra_are_wanted( procver='realtime', position_procver=None,
                                            objids_table='tmp_wanted_no_spec', return_format='pandas',
                                            return_object_info=False, mjd_now=mjdnow, dbcon=con )
 
-    srcltcvs.reset_index( inplace=True )
-    frcltcvs.reset_index( inplace=True )
+    srcltcvs = srcltcvs.reset_index()
+    frcltcvs = frcltcvs.reset_index()
 
     # Remove unwanted columns
     yanks = [ i for i in srcltcvs.columns if i not in [ 'rootid', 'mjd', 'band', 'flux' ] ]
-    srcltcvs.drop( yanks, axis='columns', inplace=True )
+    srcltcvs = srcltcvs.drop( yanks, axis='columns' )
     yanks = [ i for i in frcltcvs.columns if i not in [ 'rootid', 'mjd', 'band', 'flux' ] ]
-    frcltcvs.drop( yanks, axis='columns', inplace=True )
+    frcltvcs = frcltcvs.drop( yanks, axis='columns' )
 
     # Extract latest row for each object in srcltcvs and frcltcvs
     srcltcvs = srcltcvs.loc[ srcltcvs.groupby(["rootid", "band"])["mjd"].idxmax() ]
@@ -274,9 +274,9 @@ def what_spectra_are_wanted( procver='realtime', position_procver=None,
         photdf['mag'] = 99.
         photdf.loc[ photdf['flux'] > 0, 'mag' ] = (
             -2.5 * np.log10( photdf.loc[ photdf['flux'] > 0, 'flux' ] ) + 31.4 )
-        photdf.drop( [ 'flux' ] , axis='columns', inplace=True )
-    srcltcvs.rename( { 'mjd': 'src_mjd', 'band': 'src_band', 'mag': 'src_mag' }, axis='columns', inplace=True )
-    frcltcvs.rename( { 'mjd': 'frced_mjd', 'band': 'frced_band', 'mag': 'frced_mag' }, axis='columns', inplace=True )
+        photdf = photdf.drop( [ 'flux' ] , axis='columns' )
+    srcltcvs = srcltcvs.rename( { 'mjd': 'src_mjd', 'band': 'src_band', 'mag': 'src_mag' }, axis='columns' )
+    frcltcvs = frcltcvs.rename( { 'mjd': 'frced_mjd', 'band': 'frced_band', 'mag': 'frced_mag' }, axis='columns' )
 
     # Filter by limiting magnitude if necessary
     if lim_mag is not None:
@@ -290,8 +290,8 @@ def what_spectra_are_wanted( procver='realtime', position_procver=None,
             lim_frcltcvs = frcltcvs.loc[ frcltcvs.groupby(["rootid"])["frced_mjd"].idxmax(),
                                         [ "rootid", "frced_mjd", "frced_mag" ] ]
 
-        lim_srcltcvs.set_index( 'rootid', inplace=True )
-        lim_frcltcvs.set_index( 'rootid', inplace=True )
+        lim_srcltcvs = lim_srcltcvs.set_index( 'rootid' )
+        lim_frcltcvs = lim_frcltcvs.set_index( 'rootid' )
         lim_ltcvs = lim_srcltcvs.join( lim_frcltcvs, how='outer' )
         lim_ltcvs.loc[ : , 'mag_for_cut' ] = lim_ltcvs.src_mag
         # WORRY : isnull(), NaN, etc.
@@ -304,7 +304,7 @@ def what_spectra_are_wanted( procver='realtime', position_procver=None,
 
         # This will remove anything from df that doesn't have a rootid in lim_ltcvs
         df = df.join( lim_ltcvs, how='inner' )
-        df.drop( ['mag_for_cut'], axis='columns', inplace=True )
+        df = df.drop( ['mag_for_cut'], axis='columns' )
 
     # Keep only the latest lightcurve point independet of band
     srcltcvs = srcltcvs.loc[ srcltcvs.groupby(["rootid"])["src_mjd"].idxmax() ]
@@ -315,27 +315,27 @@ def what_spectra_are_wanted( procver='realtime', position_procver=None,
         srcltcvs = srcltcvs[ srcltcvs.src_mjd >= detsince ]
 
     # Throw out stuff we don't want from objinfo
-    objinfo.reset_index( inplace=True )
+    objinfo = objinfo.reset_index()
     yanks = [ i for i in objinfo.columns if i not in [ 'rootid', 'diaobjectid', 'ra', 'dec' ] ]
-    objinfo.drop( yanks, axis='columns', inplace=True )
+    objinfo = objinfo.drop( yanks, axis='columns' )
     objinfo = objinfo.groupby( 'rootid' ).agg( 'first' )
-    objinfo.rename( { 'ra': 'diaobj_meanra', 'dec': 'diaobj_meandec' }, axis='columns', inplace=True )
+    objinfo = objinfo.rename( { 'ra': 'diaobj_meanra', 'dec': 'diaobj_meandec' }, axis='columns' )
 
     # Join to latest mags.  We *assume* there are detections, otherwise nobody would want a spectrum.
     # Also, we wouldn't have heard about the object in the first place.
-    srcltcvs.set_index( 'rootid', inplace=True )
+    srcltcvs = srcltcvs.set_index( 'rootid' )
     df = df.join( srcltcvs, how='inner' )
-    df.rename( { 'mjd': 'src_mjd', 'flux': 'src_flux', 'band': 'src_band' }, axis='columns', inplace=True )
-    frcltcvs.set_index( 'rootid', inplace=True )
+    df = df.rename( { 'mjd': 'src_mjd', 'flux': 'src_flux', 'band': 'src_band' }, axis='columns' )
+    frcltcvs = frcltcvs.set_index( 'rootid' )
     df = df.join( frcltcvs, how='left' )
-    df.rename( { 'mjd': 'frced_mjd', 'flux': 'frced_flux', 'band': 'frced_band' }, axis='columns', inplace=True )
-
+    df = df.rename( { 'mjd': 'frced_mjd', 'flux': 'frced_flux', 'band': 'frced_band' }, axis='columns' )
+    
     # Join to obinfo to get ra/dec
     df = df.join( objinfo, how='left' )
 
     # Return
-    df.reset_index( inplace=True )
-    df.rename( { 'rootid': 'root_diaobject_id' }, axis='columns', inplace=True )
+    df = df.reset_index()
+    df = df.rename( { 'rootid': 'root_diaobject_id' }, axis='columns' )
     return df
 
 
diff --git a/src/util.py b/src/util.py
index e6c7f91..9757548 100644
--- a/src/util.py
+++ b/src/util.py
@@ -710,7 +710,7 @@ def get_dtypes( columns ):
             indices = list( indices )
         if keyname is not None:
             indices.insert( 0, keyname )
-        df.set_index( indices, inplace=True )
+        df = df.set_index( indices )
 
     FDBLogger.debug( "...done laboriously constructing a pandas dataframe." )
 
diff --git a/tests/test_ltcv.py b/tests/test_ltcv.py
index f2457d5..3efdf37 100644
--- a/tests/test_ltcv.py
+++ b/tests/test_ltcv.py
@@ -432,7 +432,7 @@ def compare_pandas_to_json( pdltcvs, jsltcvs, pdobjinfo, jsobjinfo ):
 
     if jsobjinfo is not None:
         assert pdobjinfo.index.names == ['rootid']
-        pdobjinfo.reset_index( inplace=True )
+        pdfobjinfo = pdobjinfo.reset_index()
         for k, v in jsobjinfo.items():
             # ROB THIS NEEDS FIXING
             # Need to check None/NA item by item

From ae2df6abf52a4846ffd565d0a4d2b43cfe61120d Mon Sep 17 00:00:00 2001
From: Rob Knop <rknop@pobox.com>
Date: Mon, 26 Oct 2026 11:39:53 -0700
Subject: [PATCH 27/35] Fix two small errors from last commit

---
 src/spectrum.py    | 4 ++--
 tests/test_ltcv.py | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/spectrum.py b/src/spectrum.py
index 66a8e89..816957c 100644
--- a/src/spectrum.py
+++ b/src/spectrum.py
@@ -260,7 +260,7 @@ def what_spectra_are_wanted( procver='realtime', position_procver=None,
     yanks = [ i for i in srcltcvs.columns if i not in [ 'rootid', 'mjd', 'band', 'flux' ] ]
     srcltcvs = srcltcvs.drop( yanks, axis='columns' )
     yanks = [ i for i in frcltcvs.columns if i not in [ 'rootid', 'mjd', 'band', 'flux' ] ]
-    frcltvcs = frcltcvs.drop( yanks, axis='columns' )
+    frcltcvs = frcltcvs.drop( yanks, axis='columns' )
 
     # Extract latest row for each object in srcltcvs and frcltcvs
     srcltcvs = srcltcvs.loc[ srcltcvs.groupby(["rootid", "band"])["mjd"].idxmax() ]
@@ -329,7 +329,7 @@ def what_spectra_are_wanted( procver='realtime', position_procver=None,
     frcltcvs = frcltcvs.set_index( 'rootid' )
     df = df.join( frcltcvs, how='left' )
     df = df.rename( { 'mjd': 'frced_mjd', 'flux': 'frced_flux', 'band': 'frced_band' }, axis='columns' )
-    
+
     # Join to obinfo to get ra/dec
     df = df.join( objinfo, how='left' )
 
diff --git a/tests/test_ltcv.py b/tests/test_ltcv.py
index 3efdf37..9a1348d 100644
--- a/tests/test_ltcv.py
+++ b/tests/test_ltcv.py
@@ -432,7 +432,7 @@ def compare_pandas_to_json( pdltcvs, jsltcvs, pdobjinfo, jsobjinfo ):
 
     if jsobjinfo is not None:
         assert pdobjinfo.index.names == ['rootid']
-        pdfobjinfo = pdobjinfo.reset_index()
+        pdobjinfo = pdobjinfo.reset_index()
         for k, v in jsobjinfo.items():
             # ROB THIS NEEDS FIXING
             # Need to check None/NA item by item

From bee11c4fd36f61d9a841855b4d7bf99d37bf15b7 Mon Sep 17 00:00:00 2001
From: Rob Knop <rknop@pobox.com>
Date: Tue, 27 Oct 2026 11:58:45 -0700
Subject: [PATCH 28/35] Deal with a datetime.datetime deprecation warning

---
 src/spectrum.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/spectrum.py b/src/spectrum.py
index 816957c..5730529 100644
--- a/src/spectrum.py
+++ b/src/spectrum.py
@@ -124,8 +124,7 @@ def what_spectra_are_wanted( procver='realtime', position_procver=None,
 
     now = datetime.datetime.now( tz=datetime.UTC )
     if mjdnow is not None:
-        now = datetime.datetime.utcfromtimestamp( astropy.time.Time( mjdnow, format='mjd', scale='tai' ).unix_tai )
-        now = pytz.utc.localize( now )
+        now = datetime.datetime.fromtimestamp( astropy.time.Time( mjdnow, format='mjd', scale='tai' ).unix_tai, datetime.UTC )
 
     with db.DBCon() as con:
         procver = util.procver_id( procver, dbcon=con )

From c44b3f6082ec33ac1da7401473f0b678e1f3b2d3 Mon Sep 17 00:00:00 2001
From: Rob Knop <rknop@pobox.com>
Date: Tue, 27 Oct 2026 14:36:03 -0700
Subject: [PATCH 29/35] fix two ruff complaints

---
 src/spectrum.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/spectrum.py b/src/spectrum.py
index 5730529..388e1a2 100644
--- a/src/spectrum.py
+++ b/src/spectrum.py
@@ -3,7 +3,6 @@
 import sys
 import io
 import datetime
-import pytz
 import logging
 
 import numpy as np
@@ -124,7 +123,8 @@ def what_spectra_are_wanted( procver='realtime', position_procver=None,
 
     now = datetime.datetime.now( tz=datetime.UTC )
     if mjdnow is not None:
-        now = datetime.datetime.fromtimestamp( astropy.time.Time( mjdnow, format='mjd', scale='tai' ).unix_tai, datetime.UTC )
+        now = datetime.datetime.fromtimestamp( astropy.time.Time( mjdnow, format='mjd', scale='tai' ).unix_tai,
+                                               datetime.UTC )
 
     with db.DBCon() as con:
         procver = util.procver_id( procver, dbcon=con )

From a49c259cf1f6533cb92b5fd7ec9dfee24c226681 Mon Sep 17 00:00:00 2001
From: Rob Knop <rknop@pobox.com>
Date: Tue, 27 Oct 2026 14:46:05 -0700
Subject: [PATCH 30/35] Mark skip on a couple of tests that hang on github. 
 They are tests of something that's only needed for other tests, anyway.

---
 tests/services/test_projectsim.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/tests/services/test_projectsim.py b/tests/services/test_projectsim.py
index 9ab0fb6..96cd4dd 100644
--- a/tests/services/test_projectsim.py
+++ b/tests/services/test_projectsim.py
@@ -267,7 +267,6 @@ def test_send_alerts( snana_fits_ppdb_loaded ):
 # The purpose of these next tests is to see timings when sending
 #   lots of alerts.  To *really* test this, we need a bigger
 #   test set....
-
 def _send_all_alerts( **kwargs ):
     try:
         schema = util.get_alert_schema()
@@ -318,12 +317,18 @@ def _send_all_alerts( **kwargs ):
             con.commit()
 
 
+@pytest.mark.skip( reason=( "This test freezes forever on github; not sure why.  In any event, it's a test of "
+                            "something only used for other tests, so production functionality of FASTDB does "
+                            "not depend on it.  If you're doing timing tests, you can comment out this mark." ) )
 def test_send_all_alerts( snana_fits_ppdb_loaded ):
     alerts = next( _send_all_alerts() )
     assert all( all( a[f] is None for f in [ 'cutoutDifference', 'cutoutScience', 'cutoutTemplate' ]
                     ) for a in alerts )
 
 
+@pytest.mark.skip( reason=( "This test freezes forever on github; not sure why.  In any event, it's a test of "
+                            "something only used for other tests, so production functionality of FASTDB does "
+                            "not depend on it.  if you're doing timing tests, you can comment out this mark." ) )
 def test_send_all_alerts_with_cutouts( snana_fits_ppdb_loaded ):
     alerts = next( _send_all_alerts( make_cutouts=True ) )
     assert all( all( a[f] is not None for f in [ 'cutoutDifference', 'cutoutScience', 'cutoutTemplate' ]

From 05dac07f798b08bbe3e4fda9f9c44278c81747e3 Mon Sep 17 00:00:00 2001
From: Rob Knop <rknop@pobox.com>
Date: Tue, 27 Oct 2026 15:00:47 -0700
Subject: [PATCH 31/35] struggling with github runner

---
 .github/workflows/run-fastdb-tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-fastdb-tests.yaml b/.github/workflows/run-fastdb-tests.yaml
index bbce8b2..bf6c6ff 100644
--- a/.github/workflows/run-fastdb-tests.yaml
+++ b/.github/workflows/run-fastdb-tests.yaml
@@ -56,4 +56,4 @@ jobs:
 
       - name: run test
         run: |
-          docker compose run --rm runtests
+          docker compose run runtests

From 81bde4393cac1439ba4ec69d9ca1b8469371e38a Mon Sep 17 00:00:00 2001
From: Rob Knop <rknop@pobox.com>
Date: Tue, 27 Oct 2026 15:36:48 -0700
Subject: [PATCH 32/35] *facepalm* exit code was coming not from the docker
 command, but the post-start command, which was missing backslashes

---
 docker-compose.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker-compose.yaml b/docker-compose.yaml
index 69fa288..1bd550a 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -342,8 +342,8 @@ services:
         target: /logs
     post_start:
       - command: /bin/bash -c "
-                   chown -R ${USERID:-0}:${GROUPID:-0} /sessions
-                   && chown -R ${USERID:-0}:${GROUPID:-0} /query_results
+                   chown -R ${USERID:-0}:${GROUPID:-0} /sessions \
+                   && chown -R ${USERID:-0}:${GROUPID:-0} /query_results \
                    && chown -R ${USERID:-0}:${GROUPID:-0} /logs
                    "
         privileged: true

From 8ea387b3ed18c8888ef27ac37de769ab614a8657 Mon Sep 17 00:00:00 2001
From: Rob Knop <rknop@pobox.com>
Date: Wed, 28 Oct 2026 08:52:19 -0700
Subject: [PATCH 33/35] Ugh, I committed the docker-compose.yaml fix to the
 wrong branch, now doing it in this branch.

---
 docker-compose.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.yaml b/docker-compose.yaml
index 1bd550a..02384ee 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -340,7 +340,7 @@ services:
       - type: volume
         source: logs
         target: /logs
-    post_start:
+    pre_start:
       - command: /bin/bash -c "
                    chown -R ${USERID:-0}:${GROUPID:-0} /sessions \
                    && chown -R ${USERID:-0}:${GROUPID:-0} /query_results \

From da01edd9f832b51b648b5fd0f2cbeac1061d9704 Mon Sep 17 00:00:00 2001
From: Rob Knop <rknop@pobox.com>
Date: Wed, 28 Oct 2026 10:16:48 -0700
Subject: [PATCH 34/35] RAGE.  Github runners' docker compose don't support
 pre_start.  Torture docker-compose.yaml to compensate.

---
 docker-compose.yaml         | 62 ++++++++++++++++++++++++-------------
 docker/webserver/Dockerfile |  1 +
 2 files changed, 42 insertions(+), 21 deletions(-)

diff --git a/docker-compose.yaml b/docker-compose.yaml
index 02384ee..7966db9 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -3,7 +3,7 @@
 
 services:
   kafka-server:
-    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-kafka-test:${DOCKER_VERSION-test20261026a}
+    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-kafka-test:${DOCKER_VERSION-test20261028a}
     build:
       context: ./docker/kafka
     environment:
@@ -32,7 +32,7 @@ services:
       retries: 10
 
   postgres:
-    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-postgres:${DOCKER_VERSION:-test20261026a}
+    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-postgres:${DOCKER_VERSION:-test20261028a}
     build:
       context: ./docker/postgres
       target: postgres
@@ -53,7 +53,7 @@ services:
       retries: 10
 
   mongodb:
-    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-mongodb:${DOCKER_VERSION:-test20261026a}
+    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-mongodb:${DOCKER_VERSION:-test20261028a}
     build:
       context: ./docker/mongodb
     environment:
@@ -80,7 +80,7 @@ services:
     depends_on:
       postgres:
         condition: service_healthy
-    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-shell:${DOCKER_VERSION:-test20261026a}
+    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-shell:${DOCKER_VERSION:-test20261028a}
     build:
       context: ./docker/webserver
       target: shell
@@ -101,7 +101,7 @@ services:
     depends_on:
       createdb:
         condition: service_completed_successfully
-    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-query-runner:${DOCKER_VERSION:-test20261026a}
+    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-query-runner:${DOCKER_VERSION:-test20261028a}
     build:
       context: ./docker/query_runner
       target: queryrunner
@@ -132,7 +132,7 @@ services:
         condition: service_started
       queryrunner:
         condition: service_started
-    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-webap:${DOCKER_VERSION:-test20261026a}
+    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-webap:${DOCKER_VERSION:-test20261028a}
     build:
       context: ./docker/webserver
       target: webserver
@@ -188,7 +188,7 @@ services:
         condition: service_healthy
       createdb:
         condition: service_completed_successfully
-    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-shell:${DOCKER_VERSION:-test20261026a}
+    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-shell:${DOCKER_VERSION:-test20261028a}
     environment:
       - MONGODB_HOST=mongodb
       - MONGODB_DBNAME=brokeralert
@@ -236,7 +236,7 @@ services:
         condition: service_healthy
       mongodb:
         condition: service_healthy
-    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-shell:${DOCKER_VERSION:-test20261026a}
+    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-shell:${DOCKER_VERSION:-test20261028a}
     environment:
       - MONGODB_HOST=mongodb
       - MONGODB_DBNAME=brokeralert
@@ -279,7 +279,7 @@ services:
     working_dir: /code
 
   makeinstall:
-    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-shell:${DOCKER_VERSION:-test20261026a}
+    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-shell:${DOCKER_VERSION:-test20261028a}
     user: ${USERID:-0}:${GROUPID:-0}
     volumes:
       - type: bind
@@ -314,7 +314,7 @@ services:
         condition: service_started   # TODO : health test for webap
       makeinstall:
         condition: service_completed_successfully
-    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-shell:${DOCKER_VERSION:-test20261026a}
+    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-shell:${DOCKER_VERSION:-test20261028a}
     environment:
       - MONGODB_HOST=mongodb
       - MONGODB_DBNAME=brokeralert
@@ -325,7 +325,6 @@ services:
       - MONGODB_ALERT_READER_PASSWD=reader
       - MONGODB_ALERT_WRITER_USER=alertwriter
       - MONGODB_ALERT_WRITER_PASSWD=writer
-    user: ${USERID:-0}:${GROUPID:-0}
     volumes:
       - type: bind
         source: tests/secrets
@@ -340,17 +339,38 @@ services:
       - type: volume
         source: logs
         target: /logs
-    pre_start:
-      - command: /bin/bash -c "
-                   chown -R ${USERID:-0}:${GROUPID:-0} /sessions \
-                   && chown -R ${USERID:-0}:${GROUPID:-0} /query_results \
-                   && chown -R ${USERID:-0}:${GROUPID:-0} /logs
-                   "
-        privileged: true
-        user: root
     working_dir: /code/tests
-    entrypoint: pytest -v
-    
+    # user: ${USERID:-0}:${GROUPID:-0}
+    # This entrypoint is tortured.  It would be way (WAY) more elegant
+    #   to put the chown command in a pre_start that has user root and
+    #   privileged true, and have the user: line above uncommented; then
+    #   we could get rid of all the if and export mess, and just have
+    #   "pytest -v" in the entrypoint... but the docker compose that
+    #   runs on github CI doesn't support pre_start.
+    # (You might ask: why bother at all with user:, just run everything
+    #   as root inside the container the way the gods of Docker
+    #   intended!  That would work, but it's inconvenient, because it
+    #   will create files on bind-mounted directories that are owned by
+    #   root, which is annoying.  All this USERID/GROUPID stuff is here
+    #   so that the user can make a .env file and put their own userid
+    #   and groupid in it so that they will own files created inside the
+    #   container when they run it on their local system.)
+    entrypoint: /bin/bash -c "
+        chown -R ${USERID:-0}:${GROUPID:-0} /logs && \
+        if ! $(grep -q ${USERID:-0} /etc/passwd); then \
+               echo creating user ${USERID:-0} ; \
+               echo 'test_user:x:${USERID:-0}:${GROUPID:-0}:User,,,:/tmp:/bin/bash' >> /etc/passwd &&  \
+               echo 'test_user:*:19929::::::' >> /etc/shadow ; \
+        fi && \
+        if ! $(grep -q ${GROUPID:-0} /etc/group); then \
+               echo creating group ${GROUPID:-0} ; \
+               echo 'test_group:x:${GROUPID:-0}:' >> /etc/group ; \
+        fi && \
+        export username=$(id -un ${USERID:-0}) && \
+        export groupname=$(id -gn ${GROUPID:-0}) && \
+        sudo -E -u $$username -g $$groupname PYTHONPATH=/fastdb /venv/bin/pytest -v
+        "
+
 volumes:
   postgres_data:
   mongodb_data:
diff --git a/docker/webserver/Dockerfile b/docker/webserver/Dockerfile
index 14fa3a4..f6d59aa 100644
--- a/docker/webserver/Dockerfile
+++ b/docker/webserver/Dockerfile
@@ -48,6 +48,7 @@ RUN  apt-get update \
          python3 \
          rlwrap \
          socat \
+         sudo \
          tmux \
     && apt-get -y autoremove \
     && apt-get clean \

From 72c884abd0050217b2d938b24af491f51d02eb1a Mon Sep 17 00:00:00 2001
From: Rob Knop <rknop@pobox.com>
Date: Wed, 28 Oct 2026 10:26:22 -0700
Subject: [PATCH 35/35] Remove an unused file, set database echoqueries default
 to False, fix a processing version name bobby tables check to allow what
 should be allowed (it was too restrictive)

---
 ...026-09-21_001_procverintables.sql_disabled | 43 -------------------
 src/db.py                                     |  2 +-
 src/webserver/server.py                       |  4 +-
 3 files changed, 3 insertions(+), 46 deletions(-)
 delete mode 100644 db/2026-09-21_001_procverintables.sql_disabled

diff --git a/db/2026-09-21_001_procverintables.sql_disabled b/db/2026-09-21_001_procverintables.sql_disabled
deleted file mode 100644
index cb5b323..0000000
--- a/db/2026-09-21_001_procverintables.sql_disabled
+++ /dev/null
@@ -1,43 +0,0 @@
-ALTER TABLE diaobject ADD COLUMN procver_id uuid;
-ALTER TABLE diaobject_position ADD COLUMN procver_id uuid;
-ALTER TABLE diasource ADD COLUMN procver_id uuid;
-ALTER TABLE diaforcedsource ADD COLUMN procver_id uuid;
-
-UPDATE diaobject o SET procver_id=q.procver_id
-FROM (
- SELECT o.diaobjectid, o.base_procver_id, j.procver_id
- FROM diaobject o
- INNER JOIN base_procver_of_procver j ON o.base_procver_id=j.base_procver_id
-) q
-WHERE q.diaobjectid=o.diaobjectid AND q.base_procver_id=o.base_procver_id;
-
-UPDATE diaobject_position p SET procver_id=q.procver_id
-FROM (
- SELECT p.base_procver_id, p.diaobjectid, j.procver_id
- FROM diaobject_position p
- INNER JOIN base_procver_of_procver j ON p.base_procver_id=j.base_procver_id
-) q
-WHERE q.base_procver_id=p.base_procver_id AND q.diaobjectid=p.diaobjectid;
-
-UPDATE diasource s SET procver_id=q.procver_id
-FROM (
- SELECT s.base_procver_id, s.diasourceid, j.procver_id
- FROM diasource s
- INNER JOIN base_procver_of_procver j ON s.base_procver_id=j.base_procver_id
-) q
-WHERE  q.base_procver_id=s.base_procver_id AND q.diasourceid=s.diasourceid;
-
-UPDATE diaforcedsource f SET procver_id=q.procver_id
-FROM (
- SELECT f.base_procver_id, f.diaobjectid, f.visit, j.procver_id
- FROM diaforcedsource f
- INNER JOIN base_procver_of_procver j ON f.base_procver_id=j.base_procver_id
-) q
-WHERE q.base_procver_id=f.base_procver_id AND q.diaobjectid=f.diaobjectid AND q.visit=f.visit;
-
-ALTER INDEX idx_diaobject_base_procver_id RENAME TO idx_diaobject_position_base_procver_id;
-ALTER INDEX idx_diaobject_procver RENAME TO idx_diaobject_base_procver_id;
-CREATE INDEX idx_diaobject_procver_id ON diaobject (procver_id);
-CREATE INDEX idx_diasource_procver_id ON diasource (procver_id);
-CREATE INDEX idx_diaforcedsource_procver_id ON diaforcedsource (procver_id);
-CREATE INDEX Idx_diaobject_position_procver_id ON diaobject_position (procver_id);
diff --git a/src/db.py b/src/db.py
index 4d4c62d..bf58799 100644
--- a/src/db.py
+++ b/src/db.py
@@ -47,7 +47,7 @@
 # explaining can slow down queries as sometimes it seems that
 # postgres really wants to think about what it's doing before giving you
 # a query plan (I don't know why; is it a pg_hint_plan thing?)
-_echoqueries = True
+_echoqueries = False
 _alwaysexplain = False
 _alwaysanalyze = False
 
diff --git a/src/webserver/server.py b/src/webserver/server.py
index def92f9..61800f8 100644
--- a/src/webserver/server.py
+++ b/src/webserver/server.py
@@ -157,10 +157,10 @@ def do_the_things( self, which, procver='default' ):
                 procver = db.ProcessingVersion.get_procver( procver )
                 # Bobby table check.  One that's not valid should never have passed in the first place,
                 #  but be paranoid.
-                if not re.search( '^[a-z0-9_]+$', procver.description ):
+                if not re.search( r'^[A-Za-z0-9_\-]+$', procver.description ):
                     raise ValueError( f"Invalid processing version name \"{procver.description}\" for "
                                       f"processing version {procver.id}; should only include "
-                                      f"a-z, 0-9, and _" )
+                                      f"A-Z, a-z, 0-9, _, and -" )
             except Exception as ex:
                 raise FASTDBWebException( str(ex) )