diff --git a/startup/setup_database_new.py b/startup/setup_database_new.py index da63c8d20..06678c97d 100755 --- a/startup/setup_database_new.py +++ b/startup/setup_database_new.py @@ -23,54 +23,86 @@ sys.path.insert(0, str(pinepods_path)) sys.path.insert(0, '/pinepods') # Also add the container path for Docker -def wait_for_postgresql_ready(): - """Wait for PostgreSQL to be ready to accept connections (not just port open)""" +DEFAULT_DB_WAIT_TIMEOUT = 300 + + +def get_db_wait_timeout(): + """Seconds to wait for the database at startup (DB_WAIT_TIMEOUT); 0 waits forever.""" + raw = os.environ.get("DB_WAIT_TIMEOUT", "").strip() + if not raw: + return DEFAULT_DB_WAIT_TIMEOUT + try: + timeout = int(raw) + if timeout < 0: + raise ValueError + return timeout + except ValueError: + logger.warning(f"Invalid DB_WAIT_TIMEOUT {raw!r}, using {DEFAULT_DB_WAIT_TIMEOUT}s") + return DEFAULT_DB_WAIT_TIMEOUT + + +def wait_for_database(label, db_host, db_port, probe): + """Call probe() until it succeeds or DB_WAIT_TIMEOUT runs out. + + The database container often starts slower than PinePods (first-boot init, + slow hosts, no orchestrator-level health gating), so every failure is + treated as transient and retried with a short backoff. + """ import time + + timeout = get_db_wait_timeout() + limit = "no time limit" if timeout == 0 else f"up to {timeout}s" + logger.info(f"Waiting for {label} at {db_host}:{db_port} to be ready ({limit})...") + + start = time.monotonic() + attempt = 1 + delay = 1 + while True: + try: + probe() + logger.info(f"{label} is ready after {attempt} attempt(s)") + return True + except Exception as e: + elapsed = time.monotonic() - start + if timeout and elapsed >= timeout: + logger.error( + f"{label} at {db_host}:{db_port} did not become ready within {timeout}s: {e}. " + f"Check DB_HOST/DB_PORT/DB_USER/DB_PASSWORD, or raise DB_WAIT_TIMEOUT (0 waits forever)." + ) + return False + logger.info(f"{label} not ready yet (attempt {attempt}, {int(elapsed)}s elapsed): {e}") + time.sleep(min(delay, timeout - elapsed) if timeout else delay) + delay = min(delay * 2, 5) + attempt += 1 + + +def wait_for_postgresql_ready(): + """Wait for PostgreSQL to be ready to accept queries (not just port open)""" import psycopg - + db_host = os.environ.get("DB_HOST", "127.0.0.1") db_port = os.environ.get("DB_PORT", "5432") db_user = os.environ.get("DB_USER", "postgres") db_password = os.environ.get("DB_PASSWORD", "password") - - max_attempts = 30 # 30 seconds - attempt = 1 - - logger.info(f"Waiting for PostgreSQL at {db_host}:{db_port} to be ready...") - - while attempt <= max_attempts: - try: - # Try to connect to the postgres database - with psycopg.connect( - host=db_host, - port=db_port, - user=db_user, - password=db_password, - dbname='postgres', - connect_timeout=3 - ) as conn: - with conn.cursor() as cur: - # Test if PostgreSQL is ready to accept queries - cur.execute("SELECT 1") - cur.fetchone() - logger.info(f"PostgreSQL is ready after {attempt} attempts") - return True - except Exception as e: - if "not yet accepting connections" in str(e) or "recovery" in str(e).lower(): - logger.info(f"PostgreSQL not ready yet (attempt {attempt}/{max_attempts}): {e}") - else: - logger.warning(f"Connection attempt {attempt}/{max_attempts} failed: {e}") - - if attempt < max_attempts: - time.sleep(1) - attempt += 1 - - logger.error(f"PostgreSQL failed to become ready after {max_attempts} attempts") - return False + + def probe(): + with psycopg.connect( + host=db_host, + port=db_port, + user=db_user, + password=db_password, + dbname='postgres', + connect_timeout=3 + ) as conn: + with conn.cursor() as cur: + cur.execute("SELECT 1") + cur.fetchone() + + return wait_for_database("PostgreSQL", db_host, db_port, probe) + def wait_for_mysql_ready(): - """Wait for MySQL/MariaDB to be ready to accept connections""" - import time + """Wait for MySQL/MariaDB to be ready to accept queries""" try: import mariadb as mysql_connector except ImportError: @@ -80,40 +112,25 @@ def wait_for_mysql_ready(): db_port = int(os.environ.get("DB_PORT", "3306")) db_user = os.environ.get("DB_USER", "root") db_password = os.environ.get("DB_PASSWORD", "password") - - max_attempts = 30 # 30 seconds - attempt = 1 - - logger.info(f"Waiting for MySQL/MariaDB at {db_host}:{db_port} to be ready...") - - while attempt <= max_attempts: + + def probe(): + conn = mysql_connector.connect( + host=db_host, + port=db_port, + user=db_user, + password=db_password, + connect_timeout=3, + autocommit=True + ) try: - # Try to connect to MySQL/MariaDB - conn = mysql_connector.connect( - host=db_host, - port=db_port, - user=db_user, - password=db_password, - connect_timeout=3, - autocommit=True - ) cursor = conn.cursor() - # Test if MySQL is ready to accept queries cursor.execute("SELECT 1") cursor.fetchone() cursor.close() + finally: conn.close() - logger.info(f"MySQL/MariaDB is ready after {attempt} attempts") - return True - except Exception as e: - logger.info(f"MySQL/MariaDB not ready yet (attempt {attempt}/{max_attempts}): {e}") - - if attempt < max_attempts: - time.sleep(1) - attempt += 1 - - logger.error(f"MySQL/MariaDB failed to become ready after {max_attempts} attempts") - return False + + return wait_for_database("MySQL/MariaDB", db_host, db_port, probe) def create_database_if_not_exists(): """Create the database if it doesn't exist and wait for database to be ready""" @@ -131,9 +148,6 @@ def create_database_if_not_exists(): return # PostgreSQL database creation logic continues below - if not wait_for_postgresql_ready(): - raise Exception("PostgreSQL did not become ready in time") - try: import psycopg