PS-11393 - Add more group replication test cases

test_scripts/ps/group-replication/conftest.py

@@ -21,7 +21,9 @@
 # Proxy modes the suite can run a test behind. There is intentionally no "direct"
 # entry: every test runs behind a proxy. Each test selects its proxies explicitly
 # with @pytest.mark.parametrize("gr_cluster", [...], indirect=True) — see the test
-# files. The value passed (e.g. "router"/"haproxy") is the key looked up here.
+# files. The value passed is either a proxy name ("router"/"haproxy", giving the
+# default 3 nodes) or a (proxy, num_nodes) tuple for a differently-sized cluster;
+# the proxy name is the key looked up here.
 PROXIES = {
     "router": {"mysql_router": True},
     "haproxy": {"haproxy": True},
@@ -44,16 +46,35 @@ def gr_cluster(request):
     # @pytest.mark.parametrize("gr_cluster", [...], indirect=True). Validate it explicitly
     # so a test that forgets the decorator fails with a clear message instead of an opaque
     # AttributeError (no param) / KeyError (unknown proxy).
-    proxy = getattr(request, "param", None)
-    if proxy is None:
+    #
+    # A test needing a different cluster size passes a (proxy, num_nodes) tuple instead of a
+    # bare proxy name — wrapped in pytest.param(..., id=proxy), or the node id degrades to
+    # "gr_cluster0". Everything else about the fixture is the same either way.
+    param = getattr(request, "param", None)
+    if param is None:
         raise pytest.UsageError(
             'gr_cluster requires a proxy via indirect parametrization, e.g. '
             '@pytest.mark.parametrize("gr_cluster", ["haproxy"], indirect=True)'
         )
-    if proxy not in PROXIES:
+    if isinstance(param, tuple):
+        if len(param) != 2:
+            raise pytest.UsageError(
+                f"gr_cluster tuple parameter must be (proxy, num_nodes); got {param!r}"
+            )
+        proxy, num_nodes = param
+    else:
+        proxy, num_nodes = param, 3
+    # isinstance before the lookup: an unhashable proxy (e.g. a list) would otherwise raise
+    # TypeError from inside the dict membership test rather than reporting the bad value.
+    if not isinstance(proxy, str) or proxy not in PROXIES:
         raise pytest.UsageError(
             f"unknown gr_cluster proxy {proxy!r}; valid options: {sorted(PROXIES)}"
         )
+    # bool is a subclass of int, so without the isinstance guard True would pass as 1 node.
+    if isinstance(num_nodes, bool) or not isinstance(num_nodes, int) or num_nodes < 1:
+        raise pytest.UsageError(
+            f"gr_cluster num_nodes must be a positive integer; got {num_nodes!r}"
+        )
     try:
         helper = DockerHelper()
     except RuntimeError as exc:
@@ -71,7 +92,7 @@ def gr_cluster(request):
     offset = int(m.group()) if m else 0
     cluster = GroupReplication(
         helper,
-        num_nodes=3,
+        num_nodes=num_nodes,
         network=f"grnet-{safe_workerid}",
         node_prefix=f"ps{safe_workerid}-",
         base_host_port=33060 + offset * 100,
@@ -110,6 +131,15 @@ def sysbench(request, gr_cluster):
     try:
         yield sb
     finally:
+        # Drop the tables so a later test sharing this module-scoped cluster can prepare()
+        # again — prepare() creates them outright and fails if they already exist. check=False
+        # and the broad except: a cluster left unhealthy by a failing test must not turn
+        # teardown into a second error that masks the real one.
+        try:
+            cleanup_host, cleanup_port = gr_cluster.rw_endpoint()
+            sb.cleanup(host=cleanup_host, port=cleanup_port, check=False)
+        except Exception as exc:  # noqa: BLE001 - teardown must not mask a test failure
+            gr_cluster.log(f"sysbench cleanup skipped: {exc}")
         gr_cluster.docker.destroy(name)
 
 

test_scripts/ps/group-replication/docker_helper.py

@@ -89,6 +89,7 @@ def create(
         detach: bool = True,
         restart: str | None = None,
         platform: str | None = None,
+        cap_add: list[str] | None = None,
     ) -> ExecResult:
         """Create and start a long-lived (detached) container with the given config."""
         # These containers are long-lived (no --rm), so a run that crashed before teardown
@@ -110,6 +111,8 @@ def create(
             args.extend(["--entrypoint", entrypoint])
         if restart:
             args.extend(["--restart", restart])
+        for cap in cap_add or []:
+            args.extend(["--cap-add", cap])
         for k, v in (environment or {}).items():
             args.extend(["-e", f"{k}={v}"])
         for vol in volumes or []:
@@ -177,12 +180,35 @@ def start(self, name: str) -> ExecResult:
         return self._run(["start", name])
 
     def stop(self, name: str) -> ExecResult:
-        """Stop a running container."""
+        """Stop a running container gracefully (SIGTERM, then a timeout)."""
         return self._run(["stop", name])
 
-    def exec_command(self, name: str, command: str, check: bool = False) -> ExecResult:
-        """Run a shell command inside a running container."""
-        return self._run(["exec", name, "sh", "-c", command], check=check)
+    def kill(self, name: str) -> ExecResult:
+        """SIGKILL a container's main process — an abrupt death, not a clean shutdown.
+
+        The container stays stopped afterwards: a kill counts as manual intervention, so a
+        `--restart always` policy does not bring it back (verified against podman). Use
+        start() to revive it, as with stop().
+        """
+        return self._run(["kill", name])
+
+    def exec_command(
+        self, name: str, command: str, check: bool = False, user: str | None = None
+    ) -> ExecResult:
+        """Run a shell command inside a running container, as `user` when given.
+
+        Without `user` the command runs as whatever the image declares (the server image
+        sets USER mysql, uid 1001). Pass user="root" for anything needing a capability such
+        as NET_ADMIN: --cap-add only fills the container's *bounding* set, which a non-root
+        process does not inherit without ambient or file capabilities. Docker enforces that
+        and the command fails with EPERM; rootless podman happens to let it through, so this
+        is a difference that only shows up on one of the two runtimes the suite supports.
+        """
+        args = ["exec"]
+        if user:
+            args.extend(["-u", user])
+        args.extend([name, "sh", "-c", command])
+        return self._run(args, check=check)
 
     def exec_mysql(
         self,
@@ -261,10 +287,78 @@ def network_remove(self, name: str) -> ExecResult:
         """Remove a container network, ignoring errors if it does not exist."""
         return self._run(["network", "rm", name], check=False)
 
+    def container_networks(self, name: str) -> list[str]:
+        """Return the names of the networks a container is currently attached to.
+
+        Empty means attached to nothing — the normal state of a node that
+        network_disconnect() has isolated, not an error. An inspect that fails (no such
+        container, no daemon) raises rather than returning [], so callers can act on the
+        answer instead of guessing: reporting a failure as "attached to nothing" would make
+        network_disconnect() skip the disconnect and report success, leaving a partition
+        test reasoning about a partition that never happened.
+        """
+        result = self._run(
+            [
+                "inspect",
+                "-f",
+                '{{range $net, $_ := .NetworkSettings.Networks}}{{$net}}{{"\\n"}}{{end}}',
+                name,
+            ],
+        )
+        return [line.strip() for line in result.stdout.splitlines() if line.strip()]
+
+    def network_connect(self, network: str, name: str) -> ExecResult | None:
+        """Attach a running container to a network, doing nothing if it is already attached.
+
+        The idempotence matters for reruns and for healing a partition that was only
+        partially applied: connecting twice otherwise fails with "already exists in network".
+        Returns None when the container was already attached.
+        """
+        if network in self.container_networks(name):
+            return None
+        return self._run(["network", "connect", network, name])
+
+    def network_disconnect(self, network: str, name: str, force: bool = False) -> ExecResult | None:
+        """Detach a running container from a network, doing nothing if it is not attached.
+
+        Unlike stop(), the process inside the container is untouched — it simply loses
+        connectivity — which is what makes this usable for network-partition tests.
+        Returns None when the container was not attached in the first place.
+
+        Note: reconnecting later does not restore the container's published host port
+        mappings (the -p flags given at create time). Nothing in this suite reaches nodes
+        from the host, but a healed node is no longer reachable on its host port.
+        """
+        if network not in self.container_networks(name):
+            return None
+        args = ["network", "disconnect"]
+        if force:
+            args.append("--force")
+        args.extend([network, name])
+        return self._run(args)
+
     def volume_remove(self, name: str) -> ExecResult:
         """Remove a container volume, ignoring errors if it does not exist."""
         return self._run(["volume", "rm", name], check=False)
 
+    def container_ip(self, name: str, network: str) -> str:
+        """Return a container's IPv4 address on the given network, or "" if it is not attached."""
+        template = f'{{{{(index .NetworkSettings.Networks "{network}").IPAddress}}}}'
+        result = self._run(["inspect", "-f", template, name], check=False)
+        return result.stdout.strip() if result.ok else ""
+
+    def container_state(self, name: str) -> str:
+        """Return a container's status and restart count ("running restarts=0"), or "" if unknown.
+
+        Handy in a timeout message: it distinguishes a container that is up but not yet
+        serving from one that has died or is stuck in a restart loop.
+        """
+        result = self._run(
+            ["inspect", "-f", "{{.State.Status}} restarts={{.RestartCount}}", name],
+            check=False,
+        )
+        return result.stdout.strip() if result.ok else ""
+
     def container_exists(self, name: str) -> bool:
         """Return True if a container with the exact given name exists (running or stopped)."""
         result = self._run(