Code to import EDP2, changes that that required, modification of materialized views, plus some improvements to webap (mostly object searching)

.github/workflows/run-fastdb-tests.yaml

@@ -56,5 +56,4 @@ jobs:
 
       - name: run test
         run: |
-          docker compose run --rm runtests
-          docker compose down -v
+          docker compose run runtests

admin/spin/production/postgres.yaml

@@ -43,7 +43,7 @@ spec:
       - env:
         - name: POSTGRES_DATA_DIR
           value: /var/lib/postgresql/data
-        image: registry.nersc.gov/m1727/raknop/fastdb-postgres:20260331a
+        image: registry.nersc.gov/m1727/raknop/fastdb-postgres:20260922b
         imagePullPolicy: IfNotPresent
         name: postgres
         resources: {}

docker-compose.yaml

@@ -3,7 +3,7 @@
 
 services:
   kafka-server:
-    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-kafka-test:${DOCKER_VERSION-test20260918a}
+    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-kafka-test:${DOCKER_VERSION-test20261028a}
     build:
       context: ./docker/kafka
     environment:
@@ -32,7 +32,7 @@ services:
       retries: 10
 
   postgres:
-    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-postgres:${DOCKER_VERSION:-test20260918a}
+    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-postgres:${DOCKER_VERSION:-test20261028a}
     build:
       context: ./docker/postgres
       target: postgres
@@ -53,7 +53,7 @@ services:
       retries: 10
 
   mongodb:
-    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-mongodb:${DOCKER_VERSION:-test20260918a}
+    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-mongodb:${DOCKER_VERSION:-test20261028a}
     build:
       context: ./docker/mongodb
     environment:
@@ -80,7 +80,7 @@ services:
     depends_on:
       postgres:
         condition: service_healthy
-    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-shell:${DOCKER_VERSION:-test20260918a}
+    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-shell:${DOCKER_VERSION:-test20261028a}
     build:
       context: ./docker/webserver
       target: shell
@@ -101,7 +101,7 @@ services:
     depends_on:
       createdb:
         condition: service_completed_successfully
-    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-query-runner:${DOCKER_VERSION:-test20260918a}
+    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-query-runner:${DOCKER_VERSION:-test20261028a}
     build:
       context: ./docker/query_runner
       target: queryrunner
@@ -132,7 +132,7 @@ services:
         condition: service_started
       queryrunner:
         condition: service_started
-    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-webap:${DOCKER_VERSION:-test20260918a}
+    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-webap:${DOCKER_VERSION:-test20261028a}
     build:
       context: ./docker/webserver
       target: webserver
@@ -188,7 +188,7 @@ services:
         condition: service_healthy
       createdb:
         condition: service_completed_successfully
-    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-shell:${DOCKER_VERSION:-test20260918a}
+    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-shell:${DOCKER_VERSION:-test20261028a}
     environment:
       - MONGODB_HOST=mongodb
       - MONGODB_DBNAME=brokeralert
@@ -236,7 +236,7 @@ services:
         condition: service_healthy
       mongodb:
         condition: service_healthy
-    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-shell:${DOCKER_VERSION:-test20260918a}
+    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-shell:${DOCKER_VERSION:-test20261028a}
     environment:
       - MONGODB_HOST=mongodb
       - MONGODB_DBNAME=brokeralert
@@ -279,7 +279,7 @@ services:
     working_dir: /code
 
   makeinstall:
-    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-shell:${DOCKER_VERSION:-test20260918a}
+    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-shell:${DOCKER_VERSION:-test20261028a}
     user: ${USERID:-0}:${GROUPID:-0}
     volumes:
       - type: bind
@@ -314,7 +314,7 @@ services:
         condition: service_started   # TODO : health test for webap
       makeinstall:
         condition: service_completed_successfully
-    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-shell:${DOCKER_VERSION:-test20260918a}
+    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-shell:${DOCKER_VERSION:-test20261028a}
     environment:
       - MONGODB_HOST=mongodb
       - MONGODB_DBNAME=brokeralert
@@ -325,7 +325,6 @@ services:
       - MONGODB_ALERT_READER_PASSWD=reader
       - MONGODB_ALERT_WRITER_USER=alertwriter
       - MONGODB_ALERT_WRITER_PASSWD=writer
-    user: ${USERID:-0}:${GROUPID:-0}
     volumes:
       - type: bind
         source: tests/secrets
@@ -340,17 +339,38 @@ services:
       - type: volume
         source: logs
         target: /logs
-    post_start:
-      - command: /bin/bash -c "
-                   chown -R ${USERID:-0}:${GROUPID:-0} /sessions
-                   && chown -R ${USERID:-0}:${GROUPID:-0} /query_results
-                   && chown -R ${USERID:-0}:${GROUPID:-0} /logs
-                   "
-        privileged: true
-        user: root
     working_dir: /code/tests
-    entrypoint: pytest -v
-
+    # user: ${USERID:-0}:${GROUPID:-0}
+    # This entrypoint is tortured.  It would be way (WAY) more elegant
+    #   to put the chown command in a pre_start that has user root and
+    #   privileged true, and have the user: line above uncommented; then
+    #   we could get rid of all the if and export mess, and just have
+    #   "pytest -v" in the entrypoint... but the docker compose that
+    #   runs on github CI doesn't support pre_start.
+    # (You might ask: why bother at all with user:, just run everything
+    #   as root inside the container the way the gods of Docker
+    #   intended!  That would work, but it's inconvenient, because it
+    #   will create files on bind-mounted directories that are owned by
+    #   root, which is annoying.  All this USERID/GROUPID stuff is here
+    #   so that the user can make a .env file and put their own userid
+    #   and groupid in it so that they will own files created inside the
+    #   container when they run it on their local system.)
+    entrypoint: /bin/bash -c "
+        chown -R ${USERID:-0}:${GROUPID:-0} /logs && \
+        if ! $(grep -q ${USERID:-0} /etc/passwd); then \
+               echo creating user ${USERID:-0} ; \
+               echo 'test_user:x:${USERID:-0}:${GROUPID:-0}:User,,,:/tmp:/bin/bash' >> /etc/passwd &&  \
+               echo 'test_user:*:19929::::::' >> /etc/shadow ; \
+        fi && \
+        if ! $(grep -q ${GROUPID:-0} /etc/group); then \
+               echo creating group ${GROUPID:-0} ; \
+               echo 'test_group:x:${GROUPID:-0}:' >> /etc/group ; \
+        fi && \
+        export username=$(id -un ${USERID:-0}) && \
+        export groupname=$(id -gn ${GROUPID:-0}) && \
+        sudo -E -u $$username -g $$groupname PYTHONPATH=/fastdb /venv/bin/pytest -v
+        "
+
 volumes:
   postgres_data:
   mongodb_data:

docker/mongodb/Dockerfile

@@ -1,4 +1,4 @@
-FROM mongo:8.0.15 AS base
+FROM mongo:9.0.4 AS base
 
 SHELL ["/bin/bash", "-c"]
 WORKDIR /usr/src

docker/postgres/postgresql.conf

@@ -182,8 +182,8 @@ dynamic_shared_memory_type = posix	# the default is usually the first option
 
 # - Disk -
 
-# Added RKNOP 2022-08-22
-temp_file_limit = 1073741824            # 1 TB temp file limit
+# Added RKNOP 2022-08-22 ... removed 2026-09-22
+# temp_file_limit = 1073741824            # 1 TB temp file limit
 #temp_file_limit = -1			# limits per-process temp file space
 					# in kilobytes, or -1 for no limit
 
@@ -436,6 +436,11 @@ min_wal_size = 80MB
 
 #seq_page_cost = 1.0			# measured on an arbitrary scale
 #random_page_cost = 4.0			# same scale as above
+# RKNOP 2026-09-22 reducing this to 1.1  Backend drives
+#   should be SDDs, which do way better with random access
+#   than HDDs did.  We don't want this factor to overly
+#   de-emphasize use of indexes.
+random_page_cost = 1.1
 #cpu_tuple_cost = 0.01			# same scale as above
 #cpu_index_tuple_cost = 0.005		# same scale as above
 #cpu_operator_cost = 0.0025		# same scale as above

docker/webserver/Dockerfile

@@ -48,6 +48,7 @@ RUN  apt-get update \
          python3 \
          rlwrap \
          socat \
+         sudo \
          tmux \
     && apt-get -y autoremove \
     && apt-get clean \
@@ -107,7 +108,7 @@ RUN source /venv/bin/activate && \
        "fink-client>=10.1,<11.0" \
        "hats==0.11.0" \
        "healpy>=1.20.0,<2.0.0" \
-       "lsdb==0.11.0" \
+       "lsdb>=0.11.0,<1.0.0" \
        "nested-pandas>=0.7.1,<0.8.0" \
        "numpy>=2.5.1,<3.0.0" \
        "pandas>=3.0.5,<4.0.0" \

src/admin/csv_to_avsc.py

@@ -22,7 +22,7 @@ def main():
     schema = pandas.read_csv( args.csvfile, quoting=csv.QUOTE_ALL )
     # pandas spazzes out with iterrows and ittertuples when
     #  column names have a space in them, so remove the spaces.
-    schema.rename( { 'Data Type': 'Data_Type', 'Column Name': 'Column_Name' }, axis=1, inplace=True )
+    schema = schema.rename( { 'Data Type': 'Data_Type', 'Column Name': 'Column_Name' }, axis=1 )
 
     typeconv = {
         'boolean': 'boolean',

src/admin/fastdb_loader.py

@@ -31,6 +31,10 @@ class FastDBLoader:
     """
 
     def __init__( self, processing_version=None ):
+        if ( processing_version is not None ) and ( not re.search( '^[a-z0-9_]+$', processing_version ) ):
+            raise ValueError( f"Invalid processing version name {processing_version}, must only include "
+                              f"lowercase letters, numbers, and _" )
+
         self._all_tables = db.all_table_names.copy()
         self._all_tables.remove( "authuser" )
         self._all_tables.remove( "passwordlink" )

src/admin/load_dp1_parquet.py

@@ -33,8 +33,8 @@ def _map_columns( cls, tab, mapper, lcs ):
             else:
                 yanks.append( col )
 
-        tab.rename( renames, axis='columns', inplace=True )
-        tab.drop( yanks, axis='columns', inplace=True )
+        tab = tab.rename( renames, axis='columns' )
+        tab = tab.drop( yanks, axis='columns' )
 
 
     @classmethod
@@ -148,12 +148,12 @@ def load_one_file( self, filepath ):
             #   as the index of the parent dataframe?  If so,
             #   that might be a hint to speeding up the
             #   extraction above....  Hurm.  Really not sure.
-            df.reset_index( inplace=True )
-            df.drop( ['index'], axis='columns', inplace=True )
-            sourcedf.reset_index( inplace=True )
-            sourcedf.drop( ['index'], axis='columns', inplace=True )
-            forceddf.reset_index( inplace=True )
-            forceddf.drop( ['index'], axis='columns', inplace=True )
+            df = df.reset_index()
+            df = df.drop( ['index'], axis='columns' )
+            sourcedf = sourcedf.reset_index()
+            sourcedf = sourcedf.drop( ['index'], axis='columns' )
+            forceddf = forceddf.reset_index()
+            forceddf = forceddf.drop( ['index'], axis='columns' )
 
             self.logger.info( f"Going to try to load {len(df)} objects, {len(sourcedf)} sources, "
                               f"{len(forceddf)} forced" )

src/admin/load_edp2_parquet.py

@@ -4,6 +4,7 @@
 import textwrap
 import functools
 import multiprocessing
+import re
 from concurrent.futures import ProcessPoolExecutor
 
 
@@ -19,6 +20,9 @@
 
 class EDP2Loader:
     def __init__( self, processing_version, create_pv=False, test_only=False, slow_test=False, stop_after_n_files=0 ):
+        if not re.search( r'^[a-z0-9_]+$', processing_version ):
+            raise ValueError( "Invalid processing_version, must only include a-z, 0-9, and _" )
+
         self.processing_version_passed = processing_version
         self._test_only = test_only or slow_test
         self._slow_test = slow_test
@@ -33,7 +37,7 @@ def __init__( self, processing_version, create_pv=False, test_only=False, slow_t
                 except Exception:
                     col = 'description'
                 if self._slow_test or ( not self._test_only ):
-                    pgdb.execute( "LOCK TABLE processing_version" )
+                    pgdb.execute( "LOCK TABLE processing_version", explain=False )
                 rows = pgdb.execute( sql.SQL( "SELECT * FROM processing_version WHERE {col}={pv}" )
                                      .format( col=sql.Identifier(col), pv=processing_version ) )
                 if len(rows) > 0:

src/db.py

@@ -506,16 +506,16 @@ def construct_pgsql_where_clause( searchspec, where="WHERE", **kwargs ):
             if util.isSequence( kwargs[field] ):
                 if not fieldinfo[ 'mult' ]:
                     raise ValueError( f"Field {field} can't be a list" )
-                q += sql.SQL( "{where} {field}=ANY(%({sfield})s)" ).format( where=sql.SQL(where),
-                                                                            field=sql.Identifier(field),
-                                                                            sfield=sql.SQL(field) )
+                q += sql.SQL( "{where} {field}=ANY(%({sfield})s)\n" ).format( where=sql.SQL(where),
+                                                                              field=sql.Identifier(field),
+                                                                              sfield=sql.SQL(field) )
                 subdict[field] = list( kwargs[field] )
             else:
-                q += sql.SQL( "{where} {field}=%({sfield})s" ).format( where=sql.SQL(where),
-                                                                       field=sql.Identifier(field),
-                                                                       sfield=sql.SQL(field) )
+                q += sql.SQL( "{where} {field}=%({sfield})s\n" ).format( where=sql.SQL(where),
+                                                                         field=sql.Identifier(field),
+                                                                         sfield=sql.SQL(field) )
                 subdict[field] = kwargs[field]
-            where = " AND"
+            where = "  AND"
             del kwargs[field]
 
         if f'{field}_contains' in kwargs:
@@ -534,38 +534,38 @@ def construct_pgsql_where_clause( searchspec, where="WHERE", **kwargs ):
                                     sfield=sql.SQL(field),
                                     i=sql.SQL(str(i)) ) )
                     subdict[f'{field}_contains_{i}'] = f'%{val}%'
-                q += sql.SQL( ")" )
-                where = " AND"
+                q += sql.SQL( ")\n" )
+                where = "  AND"
             else:
-                q += sql.SQL( "{where} {field} LIKE %({sfield}_contains)s" ).format( where=sql.SQL(where),
-                                                                                     field=sql.Identifier(field),
-                                                                                     sfield=sql.SQL(field) )
+                q += sql.SQL( "{where} {field} LIKE %({sfield}_contains)s\n" ).format( where=sql.SQL(where),
+                                                                                       field=sql.Identifier(field),
+                                                                                       sfield=sql.SQL(field) )
                 subdict[f'{field}_contains'] = f"%{kwargs[f'{field}_contains']}%"
-                where = " AND"
+                where = "  AND"
             del kwargs[f'{field}_contains']
 
         if f'{field}_min' in kwargs:
             if not fieldinfo['minmax']:
                 raise ValueError( f'Field {field} doesn\'t work with "min"' )
             if util.isSequence( kwargs[f'{field}_min'] ):
                 raise ValueError( f"{field}_max can't be a list" )
-            q += sql.SQL( "{where} {field}>=%({sfield}_min)s" ).format( where=sql.SQL(where),
-                                                                        field=sql.Identifier(field),
-                                                                        sfield=sql.SQL(field) )
+            q += sql.SQL( "{where} {field}>=%({sfield}_min)s\n" ).format( where=sql.SQL(where),
+                                                                          field=sql.Identifier(field),
+                                                                          sfield=sql.SQL(field) )
             subdict[f'{field}_min'] = kwargs[f'{field}_min']
-            where = " AND"
+            where = "  AND"
             del kwargs[f'{field}_min']
 
         if f'{field}_max' in kwargs:
             if not fieldinfo['minmax']:
                 raise ValueError( f'Field {field} doesn\'t work with "max"' )
             if util.isSequence( kwargs[f'{field}_max'] ):
                 raise ValueError( f"{field}_max can't be a list" )
-            q += sql.SQL( "{where} {field}<=%({sfield}_max)s" ).format( where=sql.SQL(where),
-                                                                        field=sql.Identifier(field),
-                                                                        sfield=sql.SQL(field) )
+            q += sql.SQL( "{where} {field}<=%({sfield}_max)s\n" ).format( where=sql.SQL(where),
+                                                                          field=sql.Identifier(field),
+                                                                          sfield=sql.SQL(field) )
             subdict[f'{field}_max'] = kwargs[f'{field}_max']
-            where = " AND"
+            where = "  AND"
             del kwargs[f'{field}_max']
 
     # Differences
@@ -594,13 +594,13 @@ def construct_pgsql_where_clause( searchspec, where="WHERE", **kwargs ):
 
         lege = ">=" if minmax == "min" else "<="
 
-        q += sql.SQL( "{where} {field}-{other}{op}%({kw})s" ).format( where=sql.SQL(where),
-                                                                      field=sql.Identifier(field),
-                                                                      other=sql.Identifier(other),
-                                                                      op=sql.SQL(lege),
-                                                                      kw=sql.SQL(kw) )
+        q += sql.SQL( "{where} {field}-{other}{op}%({kw})s\n" ).format( where=sql.SQL(where),
+                                                                        field=sql.Identifier(field),
+                                                                        other=sql.Identifier(other),
+                                                                        op=sql.SQL(lege),
+                                                                        kw=sql.SQL(kw) )
         subdict[kw] = kwargs[kw]
-        where = " AND"
+        where = "  AND"
         yanks.add( kw )
 
     for yank in yanks: