diff --git a/dbuild/docs.py b/dbuild/docs.py index 3b45244..a037a6c 100644 --- a/dbuild/docs.py +++ b/dbuild/docs.py @@ -324,7 +324,7 @@ def _combinations(cfg: Config) -> list[list[tuple[Any, Any]]]: if not cs: return [] ordered = [sorted(c.options, key=lambda o: o.id != c.default) for c in cs] - return [list(zip(cs, picks)) for picks in itertools.product(*ordered)] + return [list(zip(cs, picks, strict=True)) for picks in itertools.product(*ordered)] def _choice_variants(cfg: Config) -> list[tuple[list[tuple[Any, Any]], str, str]]: @@ -477,9 +477,9 @@ def _choice_context(cfg: Config, context_env: list[dict[str, Any]]) -> dict[str, k, sep, v = ln.partition("=") if sep and k.strip() and not k.lstrip().startswith("#"): env_values[k.strip()] = v.partition(" #")[0].strip() - resolve = lambda text: _re.sub( # noqa: E731 + resolve = lambda text, ev=env_values: _re.sub( # noqa: E731 r"\$\{([A-Za-z_][A-Za-z0-9_]*)(?::-([^}]*))?\}", - lambda m: env_values.get(m.group(1)) or (m.group(2) or ""), text) + lambda m: ev.get(m.group(1)) or (m.group(2) or ""), text) parsed = (yaml.safe_load(ct) or {}).get("services") or {} parts, jails = [], [] for sname, svc in parsed.items(): diff --git a/dbuild/lint.py b/dbuild/lint.py index 1d08894..9f1e77c 100644 --- a/dbuild/lint.py +++ b/dbuild/lint.py @@ -44,6 +44,7 @@ def lint_repo(repo_path: Path, verbose: bool = False) -> tuple[list[str], list[s errors: list[str] = [] warnings: list[str] = [] + meta: dict[str, Any] = {} compose_path = repo_path / "compose.yaml" config_path = repo_path / ".daemonless" / "config.yaml" @@ -349,11 +350,7 @@ def lint_repo(repo_path: Path, verbose: bool = False) -> tuple[list[str], list[s if verbose: print(" checking Containerfile") - is_stack_repo = False - try: - is_stack_repo = ((yaml.safe_load((repo_path / "compose.yaml").read_text()) or {}).get("x-daemonless") or {}).get("type") == "stack" - except Exception: - pass + is_stack_repo = meta.get("type") == "stack" has_containerfile = any( (repo_path / name).exists() for name in ( diff --git a/dbuild/test.py b/dbuild/test.py index 8737fc2..c3471ed 100644 --- a/dbuild/test.py +++ b/dbuild/test.py @@ -83,6 +83,15 @@ def _emergency_cleanup(*_args) -> None: r"|is ready" ) +# Fatal crash signatures that indicate immediate unrecoverable container failure +_FATAL_CRASH_PATTERNS = re.compile( + r"panic:\s" + r"|Fatal error:" + r"|Segmentation fault" + r"|core dumped" +) +_CRASH_LOOP_PATTERN = re.compile(r"Service '.*' crashed") + # ── Label reading ───────────────────────────────────────────────────── @@ -254,28 +263,44 @@ def _wait_for_ready( timeout: int, *, backend: ContainerBackend, + poll_interval: float = 3.0, ) -> str: """Poll container logs for ready patterns. Returns ``"ready"`` when a pattern matched, ``"timeout"`` when the timeout elapsed without a match (non-fatal -- the port/health checks - decide), or ``"exited"`` when the container died during the wait. + decide), ``"crashed"`` when a fatal crash pattern is detected, or + ``"exited"`` when the container died during the wait. """ compiled = re.compile(patterns) - poll_interval = 3 - elapsed = 0 + elapsed = 0.0 while elapsed < timeout: if not backend.running(cname, quiet=True): log.error("Container exited during ready wait") - for line in backend.logs(cname).splitlines()[-20:]: + for line in backend.logs(cname).splitlines()[-50:]: log.info(f" {line}") return "exited" - if compiled.search(backend.logs(cname, quiet=True)): - log.info(f"Ready signal after {elapsed}s") - time.sleep(2) + logs = backend.logs(cname, quiet=True) + if compiled.search(logs): + log.info(f"Ready signal after {elapsed:.1f}s") + time.sleep(min(2.0, poll_interval)) return "ready" + crash_match = _FATAL_CRASH_PATTERNS.search(logs) + if crash_match: + log.error(f"Container fatal crash detected: {crash_match.group(0)!r}") + for line in logs.splitlines()[-50:]: + log.info(f" {line}") + return "crashed" + + crashes = len(_CRASH_LOOP_PATTERN.findall(logs)) + if crashes >= 3: + log.error(f"Container service crash loop detected ({crashes} service crashes)") + for line in logs.splitlines()[-50:]: + log.info(f" {line}") + return "crashed" + time.sleep(poll_interval) elapsed += poll_interval @@ -336,22 +361,28 @@ def _test_command( return True -def _test_port(ip: str, port: int, timeout: int) -> bool: +def _test_port( + ip: str, + port: int, + timeout: int, + *, + poll_interval: float = 1.0, +) -> bool: """Wait for a TCP port to be listening using stdlib socket.""" log.info(f"Waiting for {ip}:{port} (timeout: {timeout}s)") - elapsed = 0 + elapsed = 0.0 while elapsed < timeout: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.settimeout(2) + sock.settimeout(min(2.0, max(0.01, float(timeout) - elapsed))) try: result = sock.connect_ex((ip, port)) if result == 0: - log.info(f"Port ready after {elapsed}s") + log.info(f"Port ready after {elapsed:.1f}s") return True finally: sock.close() - time.sleep(1) - elapsed += 1 + time.sleep(poll_interval) + elapsed += poll_interval log.error(f"Port {port} not listening after {timeout}s") return False @@ -363,6 +394,8 @@ def _test_health( path: str, timeout: int, https: bool = False, + *, + poll_interval: float = 2.0, ) -> bool: """Wait for an HTTP endpoint to respond with a non-error status. @@ -381,28 +414,33 @@ def _test_health( ctx.check_hostname = False ctx.verify_mode = ssl.CERT_NONE - elapsed = 0 + elapsed = 0.0 last = "no response" + http_timeout = min(5.0, max(0.1, float(timeout))) while elapsed < timeout: try: if https: - conn = http.client.HTTPSConnection(ip, port, timeout=5, context=ctx) + conn = http.client.HTTPSConnection(ip, port, timeout=http_timeout, context=ctx) else: - conn = http.client.HTTPConnection(ip, port, timeout=5) + conn = http.client.HTTPConnection(ip, port, timeout=http_timeout) conn.request("GET", path) resp = conn.getresponse() code = resp.status + body = resp.read(1024).decode("utf-8", errors="replace").strip() conn.close() last = f"HTTP {code}" + if body and code >= 500: + body_snippet = body.replace("\n", " ")[:200] + last = f"HTTP {code} ({body_snippet})" if code < 500: - log.info(f"Health ready after {elapsed}s (HTTP {code})") + log.info(f"Health ready after {elapsed:.1f}s (HTTP {code})") return True except (ConnectionRefusedError, ConnectionResetError, OSError, http.client.HTTPException): pass - time.sleep(2) - elapsed += 2 + time.sleep(poll_interval) + elapsed += poll_interval log.error(f"Health check failed after {timeout}s (last: {last})") return False @@ -534,17 +572,41 @@ def _dump_logs( compose_file: Path | None, backend: ContainerBackend, cname: str, + lines: int = 50, ) -> None: - """Print the last 10 lines of container/compose logs.""" + """Print the last lines of container/compose logs.""" if compose_mode: assert compose_file is not None output = podman.compose_logs(str(compose_file)) else: output = backend.logs(cname) - for line in output.splitlines()[-10:]: + for line in output.splitlines()[-lines:]: log.info(f" {line}") +def _diagnose_port_failure(backend: ContainerBackend, cname: str, port: int) -> None: + """Inspect listening sockets inside the container to diagnose connection failures.""" + try: + res = backend.exec_in(cname, ["sockstat", "-46l"]) + if res.returncode == 0 and res.stdout: + lines = res.stdout.splitlines() + port_str = f":{port}" + matching = [line for line in lines if port_str in line] + if matching: + log.info("Diagnostic (sockstat -46l inside container):") + for line in matching: + log.info(f" {line}") + if "127.0.0.1" in line or "::1" in line: + log.error( + f"Port {port} is listening on localhost (127.0.0.1/::1) inside container, " + "not 0.0.0.0. Applications in FreeBSD containers must bind to 0.0.0.0 to receive traffic." + ) + else: + log.warn(f"Diagnostic: No process is listening on port {port} inside container {cname}.") + except Exception: + pass + + def _functional_checks( *, backend: ContainerBackend, @@ -596,7 +658,7 @@ def _functional_checks( if mode in ("health", "screenshot"): state = _wait_for_ready(cname, ready_patterns, test.wait, backend=backend) results["ready"] = {"ready": "pass", "timeout": "timeout"}.get(state, "fail") - if state == "exited": + if state in ("exited", "crashed"): return 1 # === PORT TEST === @@ -608,6 +670,8 @@ def _functional_checks( return 1 if not _test_port(ip, port, test.wait): results["port"] = "fail" + if not compose_mode: + _diagnose_port_failure(backend, cname, port) _dump_logs(compose_mode, compose_file, backend, cname) return 1 results["port"] = "pass" @@ -1014,28 +1078,20 @@ def run_screenshot(cfg: Config, args: argparse.Namespace) -> int: log.info(f"Container IP: {ip}") # Wait for ready - _wait_for_ready(container_name, ready_patterns, test.wait, backend=backend) + state = _wait_for_ready(container_name, ready_patterns, test.wait, backend=backend) + if state in ("exited", "crashed"): + return 1 # Wait for port if not _test_port(ip, port, test.wait): - if compose_mode: - assert compose_file is not None - output_logs = podman.compose_logs(str(compose_file)) - else: - output_logs = podman.logs(container_name) - for line in output_logs.splitlines()[-10:]: - log.info(f" {line}") + if not compose_mode: + _diagnose_port_failure(backend, container_name, port) + _dump_logs(compose_mode, compose_file, backend, container_name) return 1 # Wait for health if configured if health and not _test_health(ip, port, health, test.wait, https=https): - if compose_mode: - assert compose_file is not None - output_logs = podman.compose_logs(str(compose_file)) - else: - output_logs = podman.logs(container_name) - for line in output_logs.splitlines()[-10:]: - log.info(f" {line}") + _dump_logs(compose_mode, compose_file, backend, container_name) return 1 # Capture screenshot @@ -1124,7 +1180,7 @@ def _puid_start_and_wait( return False # Timeout is non-fatal (the ownership assert follows); only a dead # container is a hard failure. - return _wait_for_ready(cname, _PUID_READY, wait, backend=backend) != "exited" + return _wait_for_ready(cname, _PUID_READY, wait, backend=backend) not in ("exited", "crashed") def _puid_phase( @@ -1151,7 +1207,9 @@ def _puid_phase( # Make sure cont-init (incl. the usermod chown) has finished before # asserting — shell-mode functional checks don't wait for it. - _wait_for_ready(cname1, _PUID_READY, wait, backend=backend) + if _wait_for_ready(cname1, _PUID_READY, wait, backend=backend) in ("exited", "crashed"): + results["ownership"] = "fail" + return 1 # ── Deploy 1: assert ownership on the running CIT container ── ok, msg = _puid_assert(backend, cname1, init_uid, init_gid, ignore) diff --git a/tests/conftest.py b/tests/conftest.py index 7b3e1cd..a540a37 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -47,7 +47,6 @@ def make_variant(**kwargs) -> Variant: "containerfile": "Containerfile", "args": {}, "aliases": [], - "auto_version": False, "default": True, "pkg_name": None, } diff --git a/tests/test_docs.py b/tests/test_docs.py index cd9729e..ce5316d 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -629,7 +629,7 @@ def _render(self, tmp: Path): def test_a_part_can_be_dropped_and_depends_on_follows(self): with tempfile.TemporaryDirectory() as d: - ctx, outputs, errors = self._render(Path(d)) + _, outputs, errors = self._render(Path(d)) self.assertEqual(errors, []) off = outputs["compose.machine_learning-off.yaml"] self.assertNotIn("\n ml:", off) @@ -642,7 +642,7 @@ def test_a_part_can_be_dropped_and_depends_on_follows(self): def test_authored_director_is_sliced_per_option(self): with tempfile.TemporaryDirectory() as d: - ctx, outputs, _ = self._render(Path(d)) + ctx, _, _ = self._render(Path(d)) off = _combo(ctx, machine_learning="off") self.assertNotIn("photos_ml", off["director_text"]) self.assertNotIn("cache:", off["director_text"], "a volume only the dropped jail used is gone too") @@ -659,7 +659,7 @@ def test_authored_director_is_sliced_per_option(self): def test_readme_has_parts_and_the_authored_director(self): with tempfile.TemporaryDirectory() as d: - ctx, outputs, _ = self._render(Path(d)) + _, outputs, _ = self._render(Path(d)) readme = outputs["README.md"] self.assertIn("## Parts", readme) self.assertIn("| **ml** | `ghcr.io/daemonless/photos-ml:latest` | Faces and search |", readme) diff --git a/tests/test_test.py b/tests/test_test.py index 5cb6ada..4d0f1c9 100644 --- a/tests/test_test.py +++ b/tests/test_test.py @@ -10,7 +10,7 @@ import threading import unittest from pathlib import Path -from unittest.mock import patch +from unittest.mock import MagicMock, patch from dbuild import test as cit from dbuild.config import AppTestConfig @@ -268,7 +268,7 @@ def test_port_listening(self): def test_port_not_listening(self): # Use a port that's almost certainly not listening - result = cit._test_port("127.0.0.1", 19, timeout=2) + result = cit._test_port("127.0.0.1", 19, timeout=0.05, poll_interval=0.01) self.assertFalse(result) @@ -327,7 +327,7 @@ def test_health_404_still_ok(self): self.assertTrue(result) def test_health_connection_refused(self): - result = cit._test_health("127.0.0.1", 19, "/", timeout=2) + result = cit._test_health("127.0.0.1", 19, "/", timeout=0.05, poll_interval=0.01) self.assertFalse(result) @@ -404,6 +404,58 @@ def test_exited_when_container_dies(self): state = cit._wait_for_ready("c", r"Server started", 10, backend=be) self.assertEqual(state, "exited") + def test_crashed_on_panic_pattern(self): + be = _FakeReadyBackend(logs="starting up\npanic: runtime error: invalid memory address\n") + with patch("dbuild.test.time.sleep"): + state = cit._wait_for_ready("c", r"Server started", 10, backend=be) + self.assertEqual(state, "crashed") + + def test_crashed_on_segfault(self): + be = _FakeReadyBackend(logs="starting up\nSegmentation fault (core dumped)\n") + with patch("dbuild.test.time.sleep"): + state = cit._wait_for_ready("c", r"Server started", 10, backend=be) + self.assertEqual(state, "crashed") + + def test_crashed_on_service_crash_loop(self): + be = _FakeReadyBackend(logs=( + "[s6] Service 'radarr' crashed (Exit: 137, Signal: 0)\n" + "[s6] Service 'radarr' crashed (Exit: 137, Signal: 0)\n" + "[s6] Service 'radarr' crashed (Exit: 137, Signal: 0)\n" + )) + with patch("dbuild.test.time.sleep"): + state = cit._wait_for_ready("c", r"Server started", 10, backend=be) + self.assertEqual(state, "crashed") + + def test_single_service_crash_recovers(self): + # A single crash followed by recovery should NOT abort CIT + be = _FakeReadyBackend(logs=( + "[s6] Service 'radarr' crashed (Exit: 137, Signal: 0)\n" + "Server started\n" + )) + with patch("dbuild.test.time.sleep"): + state = cit._wait_for_ready("c", r"Server started", 10, backend=be) + self.assertEqual(state, "ready") + + def test_unhandled_exception_does_not_false_fail(self): + # .NET non-fatal background task exception does not fail CIT + be = _FakeReadyBackend(logs=( + "Unhandled exception. System.InvalidOperationException\n" + "Server started\n" + )) + with patch("dbuild.test.time.sleep"): + state = cit._wait_for_ready("c", r"Server started", 10, backend=be) + self.assertEqual(state, "ready") + + def test_address_in_use_does_not_false_fail(self): + # Dual-stack IPv6 bind failure does not fail CIT + be = _FakeReadyBackend(logs=( + "bind: Address already in use\n" + "Server started\n" + )) + with patch("dbuild.test.time.sleep"): + state = cit._wait_for_ready("c", r"Server started", 10, backend=be) + self.assertEqual(state, "ready") + class TestTestHealth5xx(unittest.TestCase): """5xx responses must NOT count as healthy.""" @@ -420,7 +472,7 @@ def tearDownClass(cls): cls.server.shutdown() def test_health_500_fails(self): - result = cit._test_health("127.0.0.1", self.port, "/", timeout=2) + result = cit._test_health("127.0.0.1", self.port, "/", timeout=0.05, poll_interval=0.01) self.assertFalse(result) @@ -605,5 +657,33 @@ def test_ownership_fail_short_circuits(self): self.assertNotIn("re-chown", results) +class TestDiagnosePortFailure(unittest.TestCase): + """Tests for _diagnose_port_failure().""" + + def test_diagnose_port_listening_localhost(self): + mock_backend = MagicMock() + mock_backend.exec_in.return_value.returncode = 0 + mock_backend.exec_in.return_value.stdout = ( + "USER COMMAND PID FD PROTO LOCAL ADDRESS FOREIGN ADDRESS\n" + "bsd app 1234 4 tcp4 127.0.0.1:8080 *:*\n" + ) + with patch.object(cit.log, "error") as mock_err: + cit._diagnose_port_failure(mock_backend, "c1", 8080) + mock_err.assert_called() + self.assertTrue(any("listening on localhost" in str(arg) for call in mock_err.call_args_list for arg in call[0])) + + def test_diagnose_port_not_listening(self): + mock_backend = MagicMock() + mock_backend.exec_in.return_value.returncode = 0 + mock_backend.exec_in.return_value.stdout = ( + "USER COMMAND PID FD PROTO LOCAL ADDRESS FOREIGN ADDRESS\n" + "bsd app 1234 4 tcp4 *:9000 *:*\n" + ) + with patch.object(cit.log, "warn") as mock_warn: + cit._diagnose_port_failure(mock_backend, "c1", 8080) + mock_warn.assert_called() + self.assertTrue(any("No process is listening" in str(arg) for call in mock_warn.call_args_list for arg in call[0])) + + if __name__ == "__main__": unittest.main()