chore(ci): testing with Python3.10 and ZK 3.5.10, 3.6.3, 3.7.1 (#659)

Testing libs have been upgraded.
The used java classpath now considers more .jar possible locations.
test_connection.py tests is now done first, trying to make it less flaky.
conftest.py has been added to give a way to display ZK cluster logs.
Because of ZK 3.6 and 3.7 changes, configurations related to local session has been added.
Disable GA fail-fast because of the flakiness

Author: StephenSorriaux

.github/workflows/release.yml

@@ -13,10 +13,10 @@ jobs:
     - name: Handle the code
       uses: actions/checkout@v2
 
-    - name: Set up Python 3.9
+    - name: Set up Python 3.10
       uses: actions/setup-python@v2
       with:
-        python-version: 3.9
+        python-version: "3.10"
 
     - name: Install pypa/build
       run: >-

.github/workflows/testing.yml

@@ -19,16 +19,19 @@ jobs:
     runs-on: ubuntu-latest
 
     strategy:
+      fail-fast: false
       matrix:
-        python-version: [3.7, 3.8, 3.9, pypy-3.7]
-        zk-version: [3.4.14, 3.5.8]
+        python-version: [3.7, 3.8, 3.9, "3.10", pypy-3.7]
+        zk-version: [3.4.14, 3.5.10, 3.6.3, 3.7.1]
         include:
           - python-version: 3.7
             tox-env: py37
           - python-version: 3.8
             tox-env: py38
           - python-version: 3.9
             tox-env: py39
+          - python-version: "3.10"
+            tox-env: py310
           - python-version: pypy-3.7
             tox-env: pypy3
     steps:

constraints.txt

@@ -1,10 +1,10 @@
 # Consistent testing environment.
 coverage==6.3.2
-flake8==3.7.9
+flake8==3.9.2
 mock==3.0.5
-objgraph==3.4.1
-pytest-cov~=2.12
-pytest~=4.6
+objgraph==3.5.0
+pytest==6.2.5
+pytest-cov==3.0.0
 
 # Documentation building.
 Jinja2==2.7.3

kazoo/testing/common.py

@@ -27,6 +27,7 @@
 import logging
 import os
 import os.path
+import pathlib
 import shutil
 import signal
 import subprocess
@@ -207,6 +208,9 @@ def classpath(self):
             jars.extend(glob(os.path.join(
                 self.install_path,
                 "lib/*.jar")))
+            jars.extend(glob(os.path.join(
+                self.install_path,
+                "*.jar")))
             # support for different file locations on Debian/Ubuntu
             jars.extend(glob(os.path.join(
                 self.install_path,
@@ -268,6 +272,16 @@ def destroy(self):
 
         shutil.rmtree(self.working_path, True)
 
+    def get_logs(self):
+        log_path = pathlib.Path(
+            self.working_path,
+            'zookeeper.log'
+        )
+        if log_path.exists():
+            log_file = log_path.open('r')
+            lines = log_file.readlines()
+            return lines[-100:]
+        return []
 
 class ZookeeperCluster(object):
 
@@ -339,3 +353,9 @@ def terminate(self):
     def reset(self):
         for server in self:
             server.reset()
+
+    def get_logs(self):
+        logs = []
+        for server in self:
+            logs += server.get_logs()
+        return logs

kazoo/testing/harness.py

@@ -21,6 +21,7 @@
     "ZOOKEEPER_PORT_OFFSET": 20000,
     "ZOOKEEPER_CLUSTER_SIZE": 3,
     "ZOOKEEPER_OBSERVER_START_ID": -1,
+    "ZOOKEEPER_LOCAL_SESSION_RO": "false"
 }
 
 
@@ -34,7 +35,8 @@ def get_global_cluster():
                   "ZOOKEEPER_CLUSTER_SIZE",
                   "ZOOKEEPER_VERSION",
                   "ZOOKEEPER_OBSERVER_START_ID",
-                  "ZOOKEEPER_JAAS_AUTH"]
+                  "ZOOKEEPER_JAAS_AUTH",
+                  "ZOOKEEPER_LOCAL_SESSION_RO"]
     }
     if CLUSTER is not None:
         if CLUSTER_CONF == cluster_conf:
@@ -61,9 +63,16 @@ def get_global_cluster():
         "For deb package installations this is /usr/share/java")
 
     if ZK_VERSION >= (3, 5):
+        ZOOKEEPER_LOCAL_SESSION_RO = cluster_conf.get(
+            "ZOOKEEPER_LOCAL_SESSION_RO"
+        )
         additional_configuration_entries = [
             "4lw.commands.whitelist=*",
-            "reconfigEnabled=true"
+            "reconfigEnabled=true",
+            # required to avoid session validation error
+            # in read only test
+            "localSessionsEnabled=" + ZOOKEEPER_LOCAL_SESSION_RO,
+            "localSessionsUpgradingEnabled=" + ZOOKEEPER_LOCAL_SESSION_RO
         ]
         # If defined, this sets the superuser password to "test"
         additional_java_system_properties = [

kazoo/tests/conftest.py

@@ -0,0 +1,10 @@
+import logging
+
+log = logging.getLogger(__name__)
+
+
+def pytest_exception_interact(node, call, report):
+    cluster = node._testcase.cluster
+    log.error('Zookeeper cluster logs:')
+    for logs in cluster.get_logs():
+        log.error(logs)

kazoo/tests/test_connection.py kazoo/tests/test__connection.pykazoo/tests/test_connection.py renamed to kazoo/tests/test__connection.py

@@ -257,6 +257,7 @@ def back(state):
 
 class TestReadOnlyMode(KazooTestCase):
     def setUp(self):
+        os.environ["ZOOKEEPER_LOCAL_SESSION_RO"] = "true"
         self.setup_zookeeper(read_only=True)
         skip = False
         if CI_ZK_VERSION and CI_ZK_VERSION < (3, 4):
@@ -272,6 +273,7 @@ def setUp(self):
 
     def tearDown(self):
         self.client.stop()
+        os.environ.pop('ZOOKEEPER_LOCAL_SESSION_RO', None)
 
     def test_read_only(self):
         from kazoo.exceptions import NotReadOnlyCallError
@@ -288,9 +290,24 @@ def listen(state):
                 ev.set()
 
         try:
-            self.cluster[1].stop()
-            self.cluster[2].stop()
-            ev.wait(6)
+            # stopping both nodes at the same time
+            # else the test seems flaky when on CI hosts
+            zk_stop_threads = []
+            zk_stop_threads.append(
+                threading.Thread(
+                    target=self.cluster[1].stop, daemon=True
+                )
+            )
+            zk_stop_threads.append(
+                threading.Thread(
+                    target=self.cluster[2].stop, daemon=True
+                )
+            )
+            for thread in zk_stop_threads:
+                thread.start()
+            for thread in zk_stop_threads:
+                thread.join()
+            ev.wait(15)
             assert ev.is_set()
             assert client.client_state == KeeperState.CONNECTED_RO
 

kazoo/tests/test_sasl.py

@@ -31,6 +31,7 @@ def setUp(self):
 
     def tearDown(self):
         self.teardown_zookeeper()
+        os.environ.pop('ZOOKEEPER_JAAS_AUTH', None)
 
     def test_connect_sasl_auth(self):
         from kazoo.security import make_acl
@@ -80,6 +81,7 @@ def setUp(self):
 
     def tearDown(self):
         self.teardown_zookeeper()
+        os.environ.pop('ZOOKEEPER_JAAS_AUTH', None)
 
     def test_connect_sasl_auth(self):
         from kazoo.security import make_acl
@@ -145,6 +147,7 @@ def setUp(self):
 
     def tearDown(self):
         self.teardown_zookeeper()
+        os.environ.pop('ZOOKEEPER_JAAS_AUTH', None)
 
     def test_connect_gssapi_auth(self):
         from kazoo.security import make_acl

setup.cfg

@@ -20,6 +20,8 @@ classifiers =
     Programming Language :: Python :: 3
     Programming Language :: Python :: 3.7
     Programming Language :: Python :: 3.8
+    Programming Language :: Python :: 3.9
+    Programming Language :: Python :: 3.10
     Programming Language :: Python :: Implementation :: CPython
     Programming Language :: Python :: Implementation :: PyPy
     Topic :: Communications