diff --git a/cinder/tests/unit/volume/drivers/lightos/test_lightos_storage.py b/cinder/tests/unit/volume/drivers/lightos/test_lightos_storage.py index 026162d3fd1..f3278427fcd 100644 --- a/cinder/tests/unit/volume/drivers/lightos/test_lightos_storage.py +++ b/cinder/tests/unit/volume/drivers/lightos/test_lightos_storage.py @@ -19,6 +19,7 @@ import hashlib import http.client as httpstatus import json +from typing import Any from typing import Dict from typing import List from typing import Tuple @@ -32,6 +33,7 @@ from cinder.tests.unit import utils as test_utils from cinder.volume import configuration as conf from cinder.volume.drivers import lightos +from cinder.volume import volume_utils FAKE_LIGHTOS_CLUSTER_NODES: Dict[str, List] = { @@ -45,10 +47,15 @@ ] } -FAKE_LIGHTOS_CLUSTER_INFO: Dict[str, str] = { +FAKE_LIGHTOS_CLUSTER_INFO: Dict[str, Any] = { 'UUID': "926e6df8-73e1-11ec-a624-07ba3880f6cc", 'subsystemNQN': "nqn.2014-08.org.nvmexpress:NVMf:uuid:" - "f4a89ce0-9fc2-4900-bfa3-00ad27995e7b" + "f4a89ce0-9fc2-4900-bfa3-00ad27995e7b", + 'discoveryEndpoints': [ + '192.168.75.10:8009', + '192.168.75.11:8009', + '192.168.75.12:8009' + ] } FAKE_CLIENT_HOSTNQN = "hostnqn1" @@ -252,6 +259,7 @@ def setUp(self): configuration.lightos_api_address = \ "10.10.10.71,10.10.10.72,10.10.10.73" + configuration.storage_protocol = lightos.NVMEOF_STORAGE_PROTOCOL configuration.lightos_api_port = 443 configuration.lightos_jwt = None configuration.lightos_snapshotname_prefix = 'openstack_' @@ -531,100 +539,41 @@ def test_create_volume_from_snapshot(self): db.volume_destroy(self.ctxt, volume.id) db.snapshot_destroy(self.ctxt, snapshot.id) - def test_initialize_connection(self): - InitialConnectorMock.hostnqn = "hostnqn1" - InitialConnectorMock.found_discovery_client = True + @mock.patch.object(volume_utils, 'brick_get_connector_properties') + def test_initialize_connection_nvmeof_storage_protocol(self, + conn_props_mock): + + conn_props_mock.return_value = \ + {"nqn": "nqn.2014-08.org.nvmexpress:NVMf:uuid:" + "f4a89ce0-9fc2-4900-bfa3-00ad27995e7b"} self.driver.do_setup(None) vol_type = test_utils.create_volume_type(self.ctxt, self, name='my_vol_type') volume = test_utils.create_volume(self.ctxt, size=4, volume_type_id=vol_type.id) self.driver.create_volume(volume) + properties = volume_utils.brick_get_connector_properties() connection_props = \ - self.driver.initialize_connection(volume, - get_connector_properties()) + self.driver.initialize_connection(volume, properties) self.assertIn('driver_volume_type', connection_props) - self.assertEqual('lightos', connection_props['driver_volume_type']) - self.assertEqual(FAKE_CLIENT_HOSTNQN, - connection_props['data']['hostnqn']) - self.assertEqual(FAKE_LIGHTOS_CLUSTER_INFO['subsystemNQN'], - connection_props['data']['nqn']) - self.assertEqual( - self.db.data['projects']['default']['volumes'][0]['UUID'], - connection_props['data']['uuid']) - - self.driver.delete_volume(volume) - db.volume_destroy(self.ctxt, volume.id) - - def test_initialize_connection_no_hostnqn_should_fail(self): - InitialConnectorMock.hostnqn = "" - InitialConnectorMock.found_discovery_client = True - self.driver.do_setup(None) - vol_type = test_utils.create_volume_type(self.ctxt, self, - name='my_vol_type') - volume = test_utils.create_volume(self.ctxt, size=4, - volume_type_id=vol_type.id) - self.driver.create_volume(volume) - self.assertRaises(exception.VolumeBackendAPIException, - self.driver.initialize_connection, volume, - get_connector_properties()) - self.driver.delete_volume(volume) - db.volume_destroy(self.ctxt, volume.id) + self.assertEqual(self.driver.configuration.storage_protocol, + connection_props['driver_volume_type']) + + project_name = self.driver._get_lightos_project_name(volume) + nguid = self.driver._get_lightos_volume_uuid(project_name, volume) + self.assertEqual(nguid, + connection_props["data"]["volume_nguid"]) + + self.assertEqual(len(FAKE_LIGHTOS_CLUSTER_NODES["nodes"]), + len(connection_props['data']['targets'])) + addresses = [addr["nvmeEndpoint"] for addr in + FAKE_LIGHTOS_CLUSTER_NODES["nodes"]] + for target in connection_props['data']['targets']: + self.assertEqual(FAKE_LIGHTOS_CLUSTER_INFO['subsystemNQN'], + target["nqn"]) + data_addr = f'{target["target_portal"]}:{target["target_port"]}' + self.assertIn(data_addr, addresses) - def test_initialize_connection_no_dsc_should_fail(self): - InitialConnectorMock.hostnqn = "hostnqn1" - InitialConnectorMock.found_discovery_client = False - self.driver.do_setup(None) - vol_type = test_utils.create_volume_type(self.ctxt, self, - name='my_vol_type') - volume = test_utils.create_volume(self.ctxt, size=4, - volume_type_id=vol_type.id) - self.driver.create_volume(volume) - self.assertRaises(exception.VolumeBackendAPIException, - self.driver.initialize_connection, volume, - get_connector_properties()) - self.driver.delete_volume(volume) - db.volume_destroy(self.ctxt, volume.id) - - def test_terminate_connection_with_hostnqn(self): - InitialConnectorMock.hostnqn = "hostnqn1" - InitialConnectorMock.found_discovery_client = True - self.driver.do_setup(None) - vol_type = test_utils.create_volume_type(self.ctxt, self, - name='my_vol_type') - volume = test_utils.create_volume(self.ctxt, size=4, - volume_type_id=vol_type.id) - self.driver.create_volume(volume) - self.driver.terminate_connection(volume, get_connector_properties()) - self.driver.delete_volume(volume) - db.volume_destroy(self.ctxt, volume.id) - - def test_terminate_connection_with_empty_hostnqn_should_fail(self): - InitialConnectorMock.hostnqn = "" - InitialConnectorMock.found_discovery_client = True - self.driver.do_setup(None) - vol_type = test_utils.create_volume_type(self.ctxt, self, - name='my_vol_type') - volume = test_utils.create_volume(self.ctxt, size=4, - volume_type_id=vol_type.id) - self.driver.create_volume(volume) - self.assertRaises(exception.VolumeBackendAPIException, - self.driver.terminate_connection, volume, - get_connector_properties()) - self.driver.delete_volume(volume) - db.volume_destroy(self.ctxt, volume.id) - - def test_force_terminate_connection_with_empty_hostnqn(self): - InitialConnectorMock.hostnqn = "" - InitialConnectorMock.found_discovery_client = True - self.driver.do_setup(None) - vol_type = test_utils.create_volume_type(self.ctxt, self, - name='my_vol_type') - volume = test_utils.create_volume(self.ctxt, size=4, - volume_type_id=vol_type.id) - self.driver.create_volume(volume) - self.driver.terminate_connection(volume, get_connector_properties(), - force=True) self.driver.delete_volume(volume) db.volume_destroy(self.ctxt, volume.id) @@ -638,23 +587,10 @@ def test_check_for_setup_error_no_subsysnqn_should_fail(self): InitialConnectorMock.hostnqn = "hostnqn1" InitialConnectorMock.found_discovery_client = True self.driver.do_setup(None) - self.driver.cluster.subsystemNQN = "" - self.assertRaises(exception.VolumeBackendAPIException, - self.driver.check_for_setup_error) - - def test_check_for_setup_error_no_hostnqn_should_fail(self): - InitialConnectorMock.hostnqn = "" - InitialConnectorMock.found_discovery_client = True - self.driver.do_setup(None) + self.driver.lightos_cluster_info["subsystemNQN"] = "" self.assertRaises(exception.VolumeBackendAPIException, self.driver.check_for_setup_error) - def test_check_for_setup_error_no_dsc_should_succeed(self): - InitialConnectorMock.hostnqn = "hostnqn1" - InitialConnectorMock.found_discovery_client = False - self.driver.do_setup(None) - self.driver.check_for_setup_error() - def test_create_clone(self): self.driver.do_setup(None) @@ -689,12 +625,10 @@ def test_get_volume_stats(self): assert volumes_data['driver_version'] == self.driver.VERSION, \ "Expected %s, received %s" % \ (self.driver.VERSION, volumes_data['driver_version']) - assert volumes_data['storage_protocol'] == "lightos", \ - "Expected 'lightos', received %s" % \ - volumes_data['storage_protocol'] - assert volumes_data['reserved_percentage'] == RESERVED_PERCENTAGE, \ - "Expected %d, received %s" % \ - (RESERVED_PERCENTAGE, volumes_data['reserved_percentage']) + self.assertEqual(volumes_data['storage_protocol'], + self.driver.configuration.storage_protocol) + self.assertEqual(volumes_data['reserved_percentage'], + RESERVED_PERCENTAGE) assert volumes_data['QoS_support'] is False, \ "Expected False, received %s" % volumes_data['QoS_support'] assert volumes_data['online_extend_support'] is True, \ diff --git a/cinder/volume/drivers/lightos.py b/cinder/volume/drivers/lightos.py index afdea62572d..bab0cc39c1b 100644 --- a/cinder/volume/drivers/lightos.py +++ b/cinder/volume/drivers/lightos.py @@ -38,8 +38,8 @@ LOG = logging.getLogger(__name__) -ENABLE_TRACE = True LIGHTOS_DEFAULT_PROJECT_NAME = "default" +NVMEOF_STORAGE_PROTOCOL = 'nvmeof' urllib3.disable_warnings() @@ -81,10 +81,9 @@ help='The default time to wait for an API service response') ] + CONF = cfg.CONF CONF.register_opts(lightos_opts, group=config.SHARED_CONF_GROUP) -BLOCK_SIZE = 8 -LIGHTOS = "LIGHTOS" class LightOSConnection(object): @@ -93,9 +92,6 @@ def __init__(self, conf): self.access_key = None self.apiservers = self._init_api_servers() self._cur_api_server_idx = random.randint(0, len(self.apiservers) - 1) - self.targets = dict() - self.lightos_cluster_uuid = None - self.subsystemNQN = None self._stats = {'total_capacity_gb': 0, 'free_capacity_gb': 0} # a single API call must have been answered in this time if the API # service/network were up @@ -373,10 +369,15 @@ def __init__(self, *args, **kwargs): self.configuration.lightos_client = \ "cinder.volume.drivers.lightos.LightOSConnection" + if self.configuration.storage_protocol != NVMEOF_STORAGE_PROTOCOL: + raise exception.InvalidParameterValue( + "storage_protocol must be: " + f"{NVMEOF_STORAGE_PROTOCOL}") + initiator_connector = importutils.import_class( self.configuration.initiator_connector) self.connector = initiator_connector.factory( - LIGHTOS, + protocol=self.configuration.storage_protocol, root_helper=utils.get_root_helper(), message_queue=None, device_scan_attempts= @@ -580,7 +581,7 @@ def _create_new_lightos_volume(self, acl=['ALLOW_NONE'] ) - def _get_lightos_uuid(self, project_name, volume): + def _get_lightos_volume_uuid(self, project_name, volume): lightos_name = self._lightos_volname(volume) timeout = self.logical_op_timeout @@ -779,7 +780,7 @@ def _delete_lightos_volume(self, project_name, lightos_uuid): def delete_volume(self, volume): """Delete volume.""" project_name = self._get_lightos_project_name(volume) - lightos_uuid = self._get_lightos_uuid(project_name, volume) + lightos_uuid = self._get_lightos_volume_uuid(project_name, volume) if not self._delete_lightos_volume(project_name, lightos_uuid): msg = 'Failed to delete LightOS volume with UUID \ @@ -794,23 +795,16 @@ def get_vols(self): LOG.warn('UNIMPLEMENTED: get vols') def check_for_setup_error(self): - subsysnqn = self.cluster.subsystemNQN - if not subsysnqn: + if not self.lightos_cluster_info.get("subsystemNQN"): msg = 'LIGHTOS: Cinder driver requires the \ LightOS cluster subsysnqn' raise exception.VolumeBackendAPIException(message=msg) - hostnqn = self.connector.get_hostnqn() - if not hostnqn: - msg = 'LIGHTOS: Cinder driver requires a local hostnqn for \ - image_to/from_volume operations' + if not self.lightos_nodes_info.get("nodes"): + msg = 'LIGHTOS: Cinder driver requires the \ + LightOS cluster nodes information' raise exception.VolumeBackendAPIException(message=msg) - found_dsc = self.connector.find_dsc() - if not found_dsc: - LOG.warn( - 'LIGHTOS: did not find a discovery client, continuing anyway') - def get_cluster_info(self): status_code, cluster_info = self.cluster.send_cmd( cmd='get_cluster_info', timeout=self.logical_op_timeout) @@ -825,6 +819,7 @@ def get_cluster_info(self): cluster_info['UUID'], cluster_info['subsystemNQN']) self.cluster.lightos_cluster_uuid = cluster_info['UUID'] self.cluster.subsystemNQN = cluster_info['subsystemNQN'] + return cluster_info def get_cluster_stats(self): status_code, cluster_info = self.cluster.send_cmd( @@ -860,19 +855,8 @@ def wait_for_lightos_cluster(self): raise exception.VolumeBackendAPIException(message=msg) def do_setup(self, context): - - self.get_cluster_info() - nodes_info = self.wait_for_lightos_cluster() - - self.cluster.targets = dict() - node_list = nodes_info['nodes'] - for node in node_list: - self.cluster.targets[node['UUID']] = node - - # reduce the logical op timeout if single server LightOS cluster - if len(node_list) == 1: - self.logical_op_timeout = self.configuration. \ - lightos_api_service_timeout + 10 + self.lightos_cluster_info = self.get_cluster_info() + self.lightos_nodes_info = self.wait_for_lightos_cluster() def extend_volume(self, volume, size): # loop because lightos api is async @@ -959,7 +943,6 @@ def get_volume_stats(self, refresh=False): res_percentage = self.configuration.safe_get('reserved_percentage') compression = self.configuration.safe_get( 'lightos_default_compression_enabled') - storage_protocol = 'lightos' # as a tenant we dont have access to cluster stats # in the future we might expose this per project via get_project API # currently we remove this stats call. @@ -968,7 +951,7 @@ def get_volume_stats(self, refresh=False): data = {'vendor_name': 'LightOS Storage', 'volume_backend_name': backend_name or self.__class__.__name__, 'driver_version': self.VERSION, - 'storage_protocol': storage_protocol, + 'storage_protocol': self.configuration.storage_protocol, 'reserved_percentage': res_percentage, 'QoS_support': False, 'online_extend_support': True, @@ -986,24 +969,6 @@ def get_volume_stats(self, refresh=False): return self._stats - def _get_connection_properties(self, project_name, volume): - lightos_targets = {} - for target in self.cluster.targets.values(): - properties = dict() - data_address, _ = target['nvmeEndpoint'].split(':') - properties['target_portal'] = data_address - properties['target_port'] = 8009 # spec specified discovery port - properties['transport_type'] = 'tcp' - lightos_targets[data_address] = properties - - server_properties = {} - server_properties['lightos_nodes'] = lightos_targets - server_properties['uuid'] = ( - self._get_lightos_uuid(project_name, volume)) - server_properties['nqn'] = self.cluster.subsystemNQN - - return server_properties - def set_volume_acl(self, project_name, lightos_uuid, acl, etag): return self.cluster.send_cmd( cmd='update_volume', @@ -1342,21 +1307,42 @@ def _delete_lightos_snapshot(self, project_name, snapshot_name): return False def initialize_connection(self, volume, connector): - hostnqn = connector.get('hostnqn') - found_dsc = connector.get('found_dsc') - LOG.debug( - 'initialize_connection: connector hostnqn is %s found_dsc %s', - hostnqn, - found_dsc) + return self._initialize_connector_nvmeof_protocol( + volume, connector) + + def _initialize_connector_nvmeof_protocol(self, volume, connector): + """Returns connection information for volume + + Example output value: + + .. code-block:: json + + { + "driver_volume_type": "nvmeof", + "data": + { + "volume_nguid": "54613aa8-784c-11ec-b15e-83cb57984331", + "host_nqn": "nqn.2014-08.org.nvmexpress:NVMf:uuid:2", + "targets": [ + { + "target_portal": "1.1.1.1", + "target_port": 4420, + "nqn": "nqn.2014-08.org.nvmexpress:NVMf:uuid:1", + "transport_type": "tcp" + }, + { + "target_portal": "1.1.1.2", + "target_port": 4420, + "nqn": "nqn.2014-08.org.nvmexpress:NVMf:uuid:1", + "transport_type": "tcp" + } + ] + } + } + """ + hostnqn = connector.get('nqn') if not hostnqn: - msg = 'Connector (%s) did not contain a hostnqn, aborting' % ( - connector,) - raise exception.VolumeBackendAPIException(message=msg) - - if not found_dsc: - msg = 'Connector (%s) did not indicate a discovery \ - client, aborting' % ( - connector,) + msg = f'Connector ({connector}) did not contain a nqn, aborting' raise exception.VolumeBackendAPIException(message=msg) lightos_volname = self._lightos_volname(volume) @@ -1368,13 +1354,31 @@ def initialize_connection(self, volume, connector): %s, aborting' % (hostnqn, lightos_volname) raise exception.VolumeBackendAPIException(message=msg) - props = self._get_connection_properties(project_name, volume) - props['hostnqn'] = hostnqn - return {'driver_volume_type': ('lightos'), 'data': props} + lightos_targets_info = [] + for node in self.lightos_nodes_info["nodes"]: + address, port = node["nvmeEndpoint"].split(':') + lightos_targets_info.append({ + 'transport_type': 'tcp', + 'target_portal': address, + 'target_port': int(port), + 'nqn': self.lightos_cluster_info["subsystemNQN"] + }) + + properties = { + "driver_volume_type": self.configuration.storage_protocol, + "data": { + "host_nqn": "TODO", + "volume_nguid": + self._get_lightos_volume_uuid(project_name, volume), + "targets": lightos_targets_info + } + } + + return properties def terminate_connection(self, volume, connector, **kwargs): force = 'force' in kwargs - hostnqn = connector.get('hostnqn') if connector else None + hostnqn = connector.get('nqn') if connector else None LOG.debug( 'terminate_connection: force %s kwargs %s hostnqn %s', force,