feat(tests): update Zookeeper 3.5.2-alpha to 3.5.4-beta

Version of Zookeeper is upgraded from 3.5.2-alpha to 3.5.4-beta for automated tests.
Reconfig now needs superuser authentification, a test is added to cover this feature.
Additionnal configuration and jvm parameters can be added when initializing
ManagedZookeeper. This is needed to ensure reconfig's tests to pass and can be used
in the future for similar needs.

Closes #477

Author: StephenSorriaux

.travis.yml

@@ -18,32 +18,44 @@ matrix:
     env: ZOOKEEPER_VERSION=3.3.6 TOX_VENV=py27
   - python: '2.7'
     env: ZOOKEEPER_VERSION=3.4.13 TOX_VENV=py27
+  - python: '2.7'
+    env: ZOOKEEPER_VERSION=3.5.4-beta TOX_VENV=py27
   - python: '2.7'
     env: ZOOKEEPER_VERSION=3.3.6 TOX_VENV=py27-gevent
   - python: '2.7'
     env: ZOOKEEPER_VERSION=3.4.13 TOX_VENV=py27-gevent
+  - python: '2.7'
+    env: ZOOKEEPER_VERSION=3.5.4-beta TOX_VENV=py27-gevent
   - python: '2.7'
     env: ZOOKEEPER_VERSION=3.3.6 TOX_VENV=py27-eventlet
   - python: '2.7'
     env: ZOOKEEPER_VERSION=3.4.13 TOX_VENV=py27-eventlet
   - python: '2.7'
-    env: ZOOKEEPER_VERSION=3.5.2-alpha TOX_VENV=py27
+    env: ZOOKEEPER_VERSION=3.5.4-beta TOX_VENV=py27-eventlet
   - python: '3.4'
     env: ZOOKEEPER_VERSION=3.3.6 TOX_VENV=py34
   - python: '3.4'
     env: ZOOKEEPER_VERSION=3.4.13 TOX_VENV=py34
+  - python: '3.4'
+    env: ZOOKEEPER_VERSION=3.5.4-beta TOX_VENV=py34
   - python: '3.5'
     env: ZOOKEEPER_VERSION=3.3.6 TOX_VENV=py35
   - python: '3.5'
     env: ZOOKEEPER_VERSION=3.4.13 TOX_VENV=py35
+  - python: '3.5'
+    env: ZOOKEEPER_VERSION=3.5.4-beta TOX_VENV=py35
   - python: '3.6'
     env: ZOOKEEPER_VERSION=3.3.6 TOX_VENV=py36
   - python: '3.6'
     env: ZOOKEEPER_VERSION=3.4.13 TOX_VENV=py36
+  - python: '3.6'
+    env: ZOOKEEPER_VERSION=3.5.4-beta TOX_VENV=py36
   - python: pypy
     env: ZOOKEEPER_VERSION=3.3.6 TOX_VENV=pypy
   - python: pypy
     env: ZOOKEEPER_VERSION=3.4.13 TOX_VENV=pypy
+  - python: 'pypy'
+    env: ZOOKEEPER_VERSION=3.5.4-beta TOX_VENV=pypy
 notifications:
   email: false
 install:

kazoo/testing/common.py

@@ -74,7 +74,8 @@ class ManagedZooKeeper(object):
     future, we may want to do that, especially when run in a
     Hudson/Buildbot context, to ensure more test robustness."""
 
-    def __init__(self, software_path, server_info, peers=(), classpath=None):
+    def __init__(self, software_path, server_info, peers=(), classpath=None,
+                 configuration_entries=[], java_system_properties=[]):
         """Define the ZooKeeper test instance.
 
         @param install_path: The path to the install for ZK
@@ -87,6 +88,8 @@ def __init__(self, software_path, server_info, peers=(), classpath=None):
         self.peers = peers
         self.working_path = tempfile.mkdtemp()
         self._running = False
+        self.configuration_entries = configuration_entries
+        self.java_system_properties = java_system_properties
 
     def run(self):
         """Run the ZooKeeper instance under a temporary directory.
@@ -117,9 +120,12 @@ def run(self):
 maxClientCnxns=0
 admin.serverPort=%s
 authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider
+%s
 """ % (to_java_compatible_path(data_path),
        self.server_info.client_port,
-       self.server_info.admin_port))  # NOQA
+       self.server_info.admin_port,
+       "\n".join(self.configuration_entries)))  # NOQA
+
 
         # setup a replicated setup if peers are specified
         if self.peers:
@@ -175,7 +181,7 @@ def run(self):
 
             # JAAS configuration for SASL authentication
             "-Djava.security.auth.login.config=%s" % jass_config_path,
-
+        ] + self.java_system_properties + [
             "org.apache.zookeeper.server.quorum.QuorumPeerMain",
             config_path,
         ]
@@ -265,11 +271,14 @@ def destroy(self):
 class ZookeeperCluster(object):
 
     def __init__(self, install_path=None, classpath=None,
-                 size=3, port_offset=20000, observer_start_id=-1):
+                 size=3, port_offset=20000, observer_start_id=-1,
+                 configuration_entries=[],
+                 java_system_properties=[]):
         self._install_path = install_path
         self._classpath = classpath
         self._servers = []
 
+
         # Calculate ports and peer group
         port = port_offset
         peers = []
@@ -292,7 +301,9 @@ def __init__(self, install_path=None, classpath=None,
             self._servers.append(
                 ManagedZooKeeper(
                     self._install_path, server_info, server_peers,
-                    classpath=self._classpath))
+                    classpath=self._classpath,
+                    configuration_entries=configuration_entries,
+                    java_system_properties=java_system_properties))
 
     def __getitem__(self, k):
         return self._servers[k]

kazoo/testing/harness.py

@@ -13,7 +13,6 @@
 )
 from kazoo.testing.common import ZookeeperCluster
 
-
 log = logging.getLogger(__name__)
 
 CLUSTER = None
@@ -26,20 +25,41 @@ def get_global_cluster():
         ZK_CLASSPATH = os.environ.get("ZOOKEEPER_CLASSPATH")
         ZK_PORT_OFFSET = int(os.environ.get("ZOOKEEPER_PORT_OFFSET", 20000))
         ZK_CLUSTER_SIZE = int(os.environ.get("ZOOKEEPER_CLUSTER_SIZE", 3))
+        ZK_VERSION = os.environ.get("ZOOKEEPER_VERSION")
+        if '-' in ZK_VERSION:
+            # Ignore pre-release markers like -alpha
+            ZK_VERSION = ZK_VERSION.split('-')[0]
+        ZK_VERSION = tuple([int(n) for n in ZK_VERSION.split('.')])
+
         ZK_OBSERVER_START_ID = int(
             os.environ.get("ZOOKEEPER_OBSERVER_START_ID", -1))
 
-        assert ZK_HOME or ZK_CLASSPATH, (
-            "Either ZOOKEEPER_PATH or ZOOKEEPER_CLASSPATH environment "
-            "variable must be defined.\n"
+        assert ZK_HOME or ZK_CLASSPATH or ZK_VERSION, (
+            "Either ZOOKEEPER_PATH or ZOOKEEPER_CLASSPATH or "
+            "ZOOKEEPER_VERSION environment variable must be defined.\n"
             "For deb package installations this is /usr/share/java")
 
+        if ZK_VERSION >= (3, 5):
+            additional_configuration_entries = [
+                "4lw.commands.whitelist=*",
+                "reconfigEnabled=true"
+            ]
+            # If defines, this sets the superuser password to "test"
+            additional_java_system_properties = [
+                "-Dzookeeper.DigestAuthenticationProvider.superDigest="
+                "super:D/InIHSb7yEEbrWz8b9l71RjZJU="
+            ]
+        else:
+            additional_configuration_entries = []
+            additional_java_system_properties = []
         CLUSTER = ZookeeperCluster(
             install_path=ZK_HOME,
             classpath=ZK_CLASSPATH,
             port_offset=ZK_PORT_OFFSET,
             size=ZK_CLUSTER_SIZE,
-            observer_start_id=ZK_OBSERVER_START_ID
+            observer_start_id=ZK_OBSERVER_START_ID,
+            configuration_entries=additional_configuration_entries,
+            java_system_properties=additional_java_system_properties
         )
         atexit.register(lambda cluster: cluster.terminate(), CLUSTER)
     return CLUSTER

kazoo/tests/test_client.py

@@ -1204,22 +1204,35 @@ def test_unchroot(self):
 
 
 class TestReconfig(KazooTestCase):
-
     def setUp(self):
         KazooTestCase.setUp(self)
+
         if TRAVIS_ZK_VERSION:
             version = TRAVIS_ZK_VERSION
         else:
             version = self.client.server_version()
         if not version or version < (3, 5):
             raise SkipTest("Must use Zookeeper 3.5 or above")
 
+    def test_no_super_auth(self):
+        self.assertRaises(NoAuthError,
+                          self.client.reconfig,
+                          joining='server.999=0.0.0.0:1234:2345:observer;3456',
+                          leaving=None,
+                          new_members=None)
+
     def test_add_remove_observer(self):
         def free_sock_port():
             s = socket.socket()
             s.bind(('', 0))
             return s, s.getsockname()[1]
 
+        username = "super"
+        password = "test"
+        digest_auth = "%s:%s" % (username, password)
+        client = self._get_client(auth_data=[('digest', digest_auth)])
+        client.start()
+
         # get ports for election, zab and client endpoints. we need to use
         # ports for which we'd immediately get a RST upon connect(); otherwise
         # the cluster could crash if it gets a SocketTimeoutException:
@@ -1230,18 +1243,18 @@ def free_sock_port():
 
         joining = 'server.100=0.0.0.0:%d:%d:observer;0.0.0.0:%d' % (
             port1, port2, port3)
-        data, _ = self.client.reconfig(joining=joining,
+        data, _ = client.reconfig(joining=joining,
                                        leaving=None,
                                        new_members=None)
-        self.assertIn(joining, data)
+        self.assertIn(joining.encode('utf8'), data)
 
-        data, _ = self.client.reconfig(joining=None,
+        data, _ = client.reconfig(joining=None,
                                        leaving='100',
                                        new_members=None)
-        self.assertNotIn(joining, data)
+        self.assertNotIn(joining.encode('utf8'), data)
 
         # try to add it again, but a config number in the future
-        curver = int(data.split('\n')[-1].split('=')[1], base=16)
+        curver = int(data.decode().split('\n')[-1].split('=')[1], base=16)
         self.assertRaises(BadVersionError,
                           self.client.reconfig,
                           joining=joining,