Skip to content

Commit

Permalink
libvirt : Add support for --interface option in iscsiadm.
Browse files Browse the repository at this point in the history 
Adds the new libvirt parameter iscsi_transport that can be used to
specify an iscsi transport, which used in conjuction with the
--interface parameter provides offloaded iscsi support.

Also happens to implement code that was originally supposed to be
covered by hw-iscsi-device-name-support as this is a
requirement for transport support.

DocImpact
Closes-Bug: #1370226
Implements: blueprint hw-iscsi-device-name-support
Implements: blueprint add-open-iscsi-transport-support

Change-Id: I1034f1e26e0b00e64430e6347d232793c3401ba8
  • Loading branch information
@anish
anish committed Feb 3, 2015
1 parent dc04361 commit 554647a
Show file tree
Hide file tree
Showing 2 changed files with 139 additions and 44 deletions.
88 changes: 72 additions & 16 deletions nova/tests/unit/virt/libvirt/test_volume.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
# under the License.

import contextlib
import glob
import os
import time

Expand Down Expand Up @@ -253,8 +254,11 @@ def test_libvirt_volume_driver_readonly(self):
readonly = tree.find('./readonly')
self.assertIsNotNone(readonly)

def iscsi_connection(self, volume, location, iqn, auth=False):
def iscsi_connection(self, volume, location, iqn, auth=False,
transport=None):
dev_name = 'ip-%s-iscsi-%s-lun-1' % (location, iqn)
if transport is not None:
dev_name = 'pci-0000:00:00.0-' + dev_name
dev_path = '/dev/disk/by-path/%s' % (dev_name)
ret = {
'driver_volume_type': 'iscsi',
Expand All @@ -276,6 +280,15 @@ def iscsi_connection(self, volume, location, iqn, auth=False):
ret['data']['auth_password'] = 'bar'
return ret

def generate_device(self, transport=None, lun=1, short=False):
dev_format = "ip-%s-iscsi-%s-lun-%s" % (self.location, self.iqn, lun)
if transport:
dev_format = "pci-0000:00:00.0-" + dev_format
if short:
return dev_format
fake_dev_path = "/dev/disk/by-path/" + dev_format
return fake_dev_path

def test_rescan_multipath(self):
libvirt_driver = volume.LibvirtISCSIVolumeDriver(self.fake_conn)
libvirt_driver._rescan_multipath()
Expand Down Expand Up @@ -305,12 +318,30 @@ def test_iscsiadm_discover_parsing(self):
out = driver._get_target_portals_from_iscsiadm_output(sample_input)
self.assertEqual(out, targets)

def test_libvirt_iscsi_driver(self):
def test_libvirt_iscsi_get_host_device(self, transport=None):
libvirt_driver = volume.LibvirtISCSIVolumeDriver(self.fake_conn)
connection_info = self.iscsi_connection(self.vol, self.location,
self.iqn)
iscsi_properties = connection_info['data']
expected_device = self.generate_device(transport, 1, False)
if transport:
self.stubs.Set(glob, 'glob', lambda x: [expected_device])
device = libvirt_driver._get_host_device(iscsi_properties)
self.assertEqual(expected_device, device)

def test_libvirt_iscsi_get_host_device_with_transport(self):
self.flags(iscsi_transport='fake_transport', group='libvirt')
self.test_libvirt_iscsi_get_host_device('fake_transport')

def test_libvirt_iscsi_driver(self, transport=None):
# NOTE(vish) exists is to make driver assume connecting worked
self.stubs.Set(os.path, 'exists', lambda x: True)
libvirt_driver = volume.LibvirtISCSIVolumeDriver(self.fake_conn)
connection_info = self.iscsi_connection(self.vol, self.location,
self.iqn)
self.iqn, False, transport)
if transport is not None:
self.stubs.Set(libvirt_driver, '_get_host_device',
lambda x: self.generate_device(transport, 1, False))
libvirt_driver.connect_volume(connection_info, self.disk_info)
libvirt_driver.disconnect_volume(connection_info, "vde")
expected_commands = [('iscsiadm', '-m', 'node', '-T', self.iqn,
Expand All @@ -330,19 +361,25 @@ def test_libvirt_iscsi_driver(self):
'-p', self.location, '--logout'),
('iscsiadm', '-m', 'node', '-T', self.iqn,
'-p', self.location, '--op', 'delete')]
self.assertEqual(self.executes, expected_commands)
self.assertEqual(expected_commands, self.executes)

def test_libvirt_iscsi_driver_still_in_use(self):
def test_libvirt_iscsi_driver_with_transport(self):
self.flags(iscsi_transport='fake_transport', group='libvirt')
self.test_libvirt_iscsi_driver('fake_transport')

def test_libvirt_iscsi_driver_still_in_use(self, transport=None):
# NOTE(vish) exists is to make driver assume connecting worked
self.stubs.Set(os.path, 'exists', lambda x: True)
libvirt_driver = volume.LibvirtISCSIVolumeDriver(self.fake_conn)
devs = ['/dev/disk/by-path/ip-%s-iscsi-%s-lun-2' % (self.location,
self.iqn)]
dev_name = self.generate_device(transport, 1, True)
if transport is not None:
self.stubs.Set(libvirt_driver, '_get_host_device',
lambda x: self.generate_device(transport, 1, False))
devs = [self.generate_device(transport, 2, False)]
self.stubs.Set(self.fake_conn, '_get_all_block_devices', lambda: devs)
vol = {'id': 1, 'name': self.name}
connection_info = self.iscsi_connection(vol, self.location, self.iqn)
libvirt_driver.connect_volume(connection_info, self.disk_info)
dev_name = 'ip-%s-iscsi-%s-lun-1' % (self.location, self.iqn)
libvirt_driver.disconnect_volume(connection_info, "vde")
expected_commands = [('iscsiadm', '-m', 'node', '-T', self.iqn,
'-p', self.location),
Expand All @@ -358,11 +395,19 @@ def test_libvirt_iscsi_driver_still_in_use(self):
'/sys/block/%s/device/delete' % dev_name)]
self.assertEqual(self.executes, expected_commands)

def test_libvirt_iscsi_driver_disconnect_multipath_error(self):
def test_libvirt_iscsi_driver_still_in_use_with_transport(self):
self.flags(iscsi_transport='fake_transport', group='libvirt')
self.test_libvirt_iscsi_driver_still_in_use('fake_transport')

def test_libvirt_iscsi_driver_disconnect_multipath_error(self,
transport=None):
libvirt_driver = volume.LibvirtISCSIVolumeDriver(self.fake_conn)
devs = ['/dev/disk/by-path/ip-%s-iscsi-%s-lun-2' % (self.location,
self.iqn)]
iscsi_devs = ['ip-fake-ip-iscsi-fake-portal-lun-2']
if transport is None:
prefix = ""
else:
prefix = "pci-0000:00:00.0-"
devs = [self.generate_device(transport, 2, False)]
iscsi_devs = ['%sip-fake-ip-iscsi-fake-portal-lun-2' % prefix]
with contextlib.nested(
mock.patch.object(os.path, 'exists', return_value=True),
mock.patch.object(self.fake_conn, '_get_all_block_devices',
Expand Down Expand Up @@ -393,13 +438,13 @@ def test_libvirt_iscsi_driver_disconnect_multipath_error(self):
['-f', 'fake-multipath-devname'],
check_exit_code=[0, 1])

def test_libvirt_iscsi_driver_get_config(self):
def test_libvirt_iscsi_driver_get_config(self, transport=None):
libvirt_driver = volume.LibvirtISCSIVolumeDriver(self.fake_conn)
dev_name = 'ip-%s-iscsi-%s-lun-1' % (self.location, self.iqn)
dev_name = self.generate_device(transport, 1, True)
dev_path = '/dev/disk/by-path/%s' % (dev_name)
vol = {'id': 1, 'name': self.name}
connection_info = self.iscsi_connection(vol, self.location,
self.iqn)
self.iqn, False, transport)
conf = libvirt_driver.get_config(connection_info, self.disk_info)
tree = conf.format_dom()
self.assertEqual('block', tree.get('type'))
Expand All @@ -411,6 +456,10 @@ def test_libvirt_iscsi_driver_get_config(self):
self.assertEqual('block', tree.get('type'))
self.assertEqual(dev_path, tree.find('./source').get('dev'))

def test_libvirt_iscsi_driver_get_config_with_transport(self):
self.flags(iscsi_transport = 'fake_transport', group='libvirt')
self.test_libvirt_iscsi_driver_get_config('fake_transport')

def test_libvirt_iscsi_driver_multipath_id(self):
libvirt_driver = volume.LibvirtISCSIVolumeDriver(self.fake_conn)
libvirt_driver.use_multipath = True
Expand Down Expand Up @@ -742,7 +791,9 @@ def _get_multipath_device_name(path):

iscsi_devs = ['1.2.3.4-iscsi-%s-lun-1' % iqn,
'%s-iscsi-%s-lun-1' % (location, iqn),
'%s-iscsi-%s-lun-2' % (location, iqn)]
'%s-iscsi-%s-lun-2' % (location, iqn),
'pci-0000:00:00.0-ip-%s-iscsi-%s-lun-3' % (location,
iqn)]
libvirt_driver._get_iscsi_devices = lambda: iscsi_devs

self.stubs.Set(libvirt_driver,
Expand Down Expand Up @@ -871,6 +922,9 @@ def test_libvirt_kvm_iser_volume_with_multipath(self):
self.stubs.Set(libvirt_driver,
'_get_target_portals_from_iscsiadm_output',
lambda x: [[location, iqn]])
self.stubs.Set(libvirt_driver, '_get_host_device',
lambda x: self.generate_device('iser', 0, False))

libvirt_driver.connect_volume(connection_info, disk_info)
conf = libvirt_driver.get_config(connection_info, disk_info)
tree = conf.format_dom()
Expand All @@ -897,6 +951,8 @@ def test_libvirt_kvm_iser_volume_with_multipath_getmpdev(self):
devs = [dev0, dev]
self.stubs.Set(self.fake_conn, '_get_all_block_devices', lambda: devs)
self.stubs.Set(libvirt_driver, '_get_iscsi_devices', lambda: [])
self.stubs.Set(libvirt_driver, '_get_host_device',
lambda x: self.generate_device('iser', 1, False))
connection_info = self.iser_connection(vol, location, iqn)
mpdev_filepath = '/dev/mapper/foo'
disk_info = {
Expand Down
95 changes: 67 additions & 28 deletions nova/virt/libvirt/volume.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,16 @@
cfg.ListOpt('qemu_allowed_storage_drivers',
default=[],
help='Protocols listed here will be accessed directly '
'from QEMU. Currently supported protocols: [gluster]')
'from QEMU. Currently supported protocols: [gluster]'),
cfg.StrOpt('iscsi_transport',
default=None,
help='The iSCSI transport to use to connect to target in case '
'offload support is desired. Supported transports are '
'be2iscsi, bnx2i, cxgb3i, cxgb4i, qla4xx and ocs. '
'Default format is transport_name.hwaddress and can be '
'generated manually or via iscsiadm -m iface'),
# iser is also supported, but use LibvirtISERVolumeDriver
# instead
]

CONF = cfg.CONF
Expand Down Expand Up @@ -283,6 +292,13 @@ def __init__(self, connection):
is_block_dev=True)
self.num_scan_tries = CONF.libvirt.num_iscsi_scan_tries
self.use_multipath = CONF.libvirt.iscsi_use_multipath
if CONF.libvirt.iscsi_transport:
self.transport = CONF.libvirt.iscsi_transport
else:
self.transport = 'default'

def _get_transport(self):
return self.transport

def _run_iscsiadm(self, iscsi_properties, iscsi_command, **kwargs):
check_exit_code = kwargs.pop('check_exit_code', 0)
Expand Down Expand Up @@ -380,7 +396,11 @@ def connect_volume(self, connection_info, disk_info):
# TODO(justinsb): This retry-with-delay is a pattern, move to utils?
tries = 0
disk_dev = disk_info['dev']
while not os.path.exists(host_device):

# Check host_device only when transport is used, since otherwise it is
# directly derived from properties. Only needed for unit tests
while ((self._get_transport() != "default" and not host_device)
or not os.path.exists(host_device)):
if tries >= self.num_scan_tries:
raise exception.NovaException(_("iSCSI device not found at %s")
% (host_device))
Expand All @@ -392,8 +412,14 @@ def connect_volume(self, connection_info, disk_info):
# The rescan isn't documented as being necessary(?), but it helps
self._run_iscsiadm(iscsi_properties, ("--rescan",))

# For offloaded open-iscsi transports, host_device cannot be
# guessed unlike iscsi_tcp where it can be obtained from
# properties, so try and get it again.
if not host_device and self._get_transport() != "default":
host_device = self._get_host_device(iscsi_properties)

tries = tries + 1
if not os.path.exists(host_device):
if not host_device or not os.path.exists(host_device):
time.sleep(tries ** 2)

if tries != 0:
Expand Down Expand Up @@ -437,11 +463,14 @@ def disconnect_volume(self, connection_info, disk_dev):

# NOTE(vish): Only disconnect from the target if no luns from the
# target are in use.
device_prefix = ("/dev/disk/by-path/ip-%s-iscsi-%s-lun-" %
device_byname = ("ip-%s-iscsi-%s-lun-" %
(iscsi_properties['target_portal'],
iscsi_properties['target_iqn']))
devices = self.connection._get_all_block_devices()
devices = [dev for dev in devices if dev.startswith(device_prefix)]
devices = [dev for dev in devices if (device_byname in dev
and
dev.startswith(
'/dev/disk/by-path/'))]
if not devices:
self._disconnect_from_iscsi_portal(iscsi_properties)
elif host_device not in devices:
Expand Down Expand Up @@ -507,6 +536,10 @@ def _disconnect_volume_multipath_iscsi(self, iscsi_properties,
entry_ip_iqn = entry.split("-lun-")[0]
if entry_ip_iqn[:3] == "ip-":
entry_ip_iqn = entry_ip_iqn[3:]
elif entry_ip_iqn[:4] == "pci-":
# Look at an offset of len('pci-0000:00:00.0')
offset = entry_ip_iqn.find("ip-", 16, 21)
entry_ip_iqn = entry_ip_iqn[(offset + 3):]
if (ip_iqn != entry_ip_iqn):
continue
entry_real_path = os.path.realpath("/dev/disk/by-path/%s" %
Expand Down Expand Up @@ -636,7 +669,13 @@ def _get_iscsi_devices(self):
devices = list(os.walk('/dev/disk/by-path'))[0][-1]
except IndexError:
return []
return [entry for entry in devices if entry.startswith("ip-")]
iscsi_devs = []
for entry in devices:
if (entry.startswith("ip-") or
(entry.startswith('pci-') and 'ip-' in entry)):
iscsi_devs.append(entry)

return iscsi_devs

def _delete_mpath(self, iscsi_properties, multipath_device, ips_iqns):
entries = self._get_iscsi_devices()
Expand Down Expand Up @@ -700,14 +739,27 @@ def _rescan_iscsi(self):
def _rescan_multipath(self):
self._run_multipath(['-r'], check_exit_code=[0, 1, 21])

def _get_host_device(self, iscsi_properties):
return ("/dev/disk/by-path/ip-%s-iscsi-%s-lun-%s" %
(iscsi_properties['target_portal'],
iscsi_properties['target_iqn'],
iscsi_properties.get('target_lun', 0)))
def _get_host_device(self, transport_properties):
"""Find device path in devtemfs."""
device = ("ip-%s-iscsi-%s-lun-%s" %
(transport_properties['target_portal'],
transport_properties['target_iqn'],
transport_properties.get('target_lun', 0)))
if self._get_transport() == "default":
return ("/dev/disk/by-path/%s" % device)
else:
host_device = None
look_for_device = glob.glob('/dev/disk/by-path/*%s' % device)
if look_for_device:
host_device = look_for_device[0]
return host_device

def _reconnect(self, iscsi_properties):
self._run_iscsiadm(iscsi_properties, ('--op', 'new'))
# Note: iscsiadm does not support changing iface.iscsi_ifacename
# via --op update, so we do this at creation time
self._run_iscsiadm(iscsi_properties,
('--interface', self._get_transport(),
'--op', 'new'))


class LibvirtISERVolumeDriver(LibvirtISCSIVolumeDriver):
Expand All @@ -717,6 +769,9 @@ def __init__(self, connection):
self.num_scan_tries = CONF.libvirt.num_iser_scan_tries
self.use_multipath = CONF.libvirt.iser_use_multipath

def _get_transport(self):
return 'iser'

def _get_multipath_iqn(self, multipath_device):
entries = self._get_iscsi_devices()
for entry in entries:
Expand All @@ -726,22 +781,6 @@ def _get_multipath_iqn(self, multipath_device):
return entry.split("iser-")[1].split("-lun")[0]
return None

def _get_host_device(self, iser_properties):
time.sleep(1)
host_device = None
device = ("ip-%s-iscsi-%s-lun-%s" %
(iser_properties['target_portal'],
iser_properties['target_iqn'],
iser_properties.get('target_lun', 0)))
look_for_device = glob.glob('/dev/disk/by-path/*%s' % device)
if look_for_device:
host_device = look_for_device[0]
return host_device

def _reconnect(self, iser_properties):
self._run_iscsiadm(iser_properties,
('--interface', 'iser', '--op', 'new'))


class LibvirtNFSVolumeDriver(LibvirtBaseVolumeDriver):
"""Class implements libvirt part of volume driver for NFS."""
Expand Down

0 comments on commit 554647a

Please sign in to comment.