Coverage for drivers/SR.py : 55%
Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#!/usr/bin/python3
2#
3# Copyright (C) Citrix Systems Inc.
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU Lesser General Public License as published
7# by the Free Software Foundation; version 2.1 only.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU Lesser General Public License for more details.
13#
14# You should have received a copy of the GNU Lesser General Public License
15# along with this program; if not, write to the Free Software Foundation, Inc.,
16# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17#
18# SR: Base class for storage repositories
19#
21import VDI
22import xml.dom.minidom
23import xs_errors
24import XenAPI # pylint: disable=import-error
25import xmlrpc.client
26import util
27import copy
28import os
29import traceback
31from cowutil import \
32 ImageFormat, getCowUtilFromImageFormat, getImageStringFromVdiType, getVdiTypeFromImageFormat, parseImageFormats
33from vditype import VdiType
35MOUNT_BASE = '/var/run/sr-mount'
36DEFAULT_TAP = "vhd,qcow2"
37MASTER_LVM_CONF = '/etc/lvm/master'
39# LUN per VDI key for XenCenter
40LUNPERVDI = "LUNperVDI"
42DEFAULT_IMAGE_FORMATS = [ImageFormat.QCOW2, ImageFormat.VHD]
47def deviceCheck(op):
48 def wrapper(self, *args):
49 if 'device' not in self.dconf:
50 raise xs_errors.XenError('ConfigDeviceMissing')
51 return op(self, *args)
52 return wrapper
55backends = []
58def registerSR(SRClass):
59 """Register SR with handler. All SR subclasses should call this in
60 the module file
61 """
62 backends.append(SRClass)
65def driver(type):
66 """Find the SR for the given dconf string"""
67 for d in backends: 67 ↛ 70line 67 didn't jump to line 70, because the loop on line 67 didn't complete
68 if d.handles(type):
69 return d
70 raise xs_errors.XenError('SRUnknownType')
73class SR(object):
74 """Semi-abstract storage repository object.
76 Attributes:
77 uuid: string, UUID
78 label: string
79 description: string
80 vdis: dictionary, VDI objects indexed by UUID
81 physical_utilisation: int, bytes consumed by VDIs
82 virtual_allocation: int, bytes allocated to this repository (virtual)
83 physical_size: int, bytes consumed by this repository
84 sr_vditype: string, repository type
85 """
87 @staticmethod
88 def handles(type) -> bool:
89 """Returns True if this SR class understands the given dconf string"""
90 return False
92 def __init__(self, srcmd, sr_uuid):
93 """Base class initializer. All subclasses should call SR.__init__
94 in their own
95 initializers.
97 Arguments:
98 srcmd: SRCommand instance, contains parsed arguments
99 """
100 try:
101 self.other_config = {}
102 self.srcmd = srcmd
103 self.dconf = srcmd.dconf
104 if 'session_ref' in srcmd.params:
105 self.session_ref = srcmd.params['session_ref']
106 self.session = XenAPI.xapi_local()
107 self.session._session = self.session_ref
108 if 'subtask_of' in self.srcmd.params: 108 ↛ 109line 108 didn't jump to line 109, because the condition on line 108 was never true
109 self.session.transport.add_extra_header('Subtask-of', self.srcmd.params['subtask_of'])
110 else:
111 self.session = None
113 if 'host_ref' not in self.srcmd.params:
114 self.host_ref = ""
115 else:
116 self.host_ref = self.srcmd.params['host_ref']
118 self.sr_ref = self.srcmd.params.get('sr_ref')
120 if 'device_config' in self.srcmd.params:
121 if self.dconf.get("SRmaster") == "true":
122 os.environ['LVM_SYSTEM_DIR'] = MASTER_LVM_CONF
124 if 'device_config' in self.srcmd.params:
125 if 'SCSIid' in self.srcmd.params['device_config']:
126 dev_path = '/dev/disk/by-scsid/' + self.srcmd.params['device_config']['SCSIid']
127 os.environ['LVM_DEVICE'] = dev_path
128 util.SMlog('Setting LVM_DEVICE to %s' % dev_path)
130 except TypeError:
131 raise Exception(traceback.format_exc())
132 except Exception as e:
133 raise e
134 raise xs_errors.XenError('SRBadXML')
136 self.uuid = sr_uuid
138 self.label = ''
139 self.description = ''
140 self.cmd = srcmd.params['command']
141 self.vdis = {}
142 self.physical_utilisation = 0
143 self.virtual_allocation = 0
144 self.physical_size = 0
145 self.sr_vditype = ''
146 self.passthrough = False
147 # XXX: if this is really needed then we must make a deep copy
148 self.original_srcmd = copy.deepcopy(self.srcmd)
149 self.default_vdi_visibility = True
150 self.scheds = ['none', 'noop']
151 self._mpathinit()
152 self.direct = False
153 self.ops_exclusive = []
154 self.driver_config = {}
155 self._is_shared = None
157 self.load(sr_uuid)
159 @staticmethod
160 def from_uuid(session, sr_uuid):
161 import importlib.util
163 _SR = session.xenapi.SR
164 sr_ref = _SR.get_by_uuid(sr_uuid)
165 sm_type = _SR.get_type(sr_ref)
166 # NB. load the SM driver module
168 _SM = session.xenapi.SM
169 sms = _SM.get_all_records_where('field "type" = "%s"' % sm_type)
170 sm_ref, sm = sms.popitem()
171 assert not sms
173 driver_path = _SM.get_driver_filename(sm_ref)
174 driver_real = os.path.realpath(driver_path)
175 module_name = os.path.basename(driver_path)
177 spec = importlib.util.spec_from_file_location(module_name, driver_real)
178 module = importlib.util.module_from_spec(spec)
179 spec.loader.exec_module(module)
181 target = driver(sm_type)
182 # NB. get the host pbd's device_config
184 host_ref = util.get_localhost_ref(session)
186 _PBD = session.xenapi.PBD
187 pbds = _PBD.get_all_records_where('field "SR" = "%s" and' % sr_ref +
188 'field "host" = "%s"' % host_ref)
189 pbd_ref, pbd = pbds.popitem()
190 assert not pbds
192 device_config = _PBD.get_device_config(pbd_ref)
193 # NB. make srcmd, to please our supersized SR constructor.
194 # FIXME
196 from SRCommand import SRCommand
197 cmd = SRCommand(module.DRIVER_INFO)
198 cmd.dconf = device_config
199 cmd.params = {'session_ref': session._session,
200 'host_ref': host_ref,
201 'device_config': device_config,
202 'sr_ref': sr_ref,
203 'sr_uuid': sr_uuid,
204 'command': 'nop'}
206 return target(cmd, sr_uuid)
208 def block_setscheduler(self, dev):
209 try:
210 realdev = os.path.realpath(dev)
211 disk = util.diskFromPartition(realdev)
213 # the normal case: the sr default scheduler (typically none/noop),
214 # potentially overridden by SR.other_config:scheduler
215 other_config = self.session.xenapi.SR.get_other_config(self.sr_ref)
216 sched = other_config.get('scheduler')
217 if not sched or sched in self.scheds: 217 ↛ 218line 217 didn't jump to line 218, because the condition on line 217 was never true
218 scheds = self.scheds
219 else:
220 scheds = [sched]
222 # special case: BFQ/CFQ if the underlying disk holds dom0's file systems.
223 if disk in util.dom0_disks(): 223 ↛ 224, 223 ↛ 2262 missed branches: 1) line 223 didn't jump to line 224, because the condition on line 223 was never true, 2) line 223 didn't jump to line 226, because the condition on line 223 was never false
224 scheds = ['bfq', 'cfq']
226 util.SMlog("Block scheduler: %s (%s) wants %s" % (dev, disk, scheds))
227 util.set_scheduler(realdev[5:], scheds)
228 except Exception as e:
229 util.SMlog("Failed to set block scheduler on %s: %s" % (dev, e))
231 def _addLUNperVDIkey(self):
232 try:
233 self.session.xenapi.SR.add_to_sm_config(self.sr_ref, LUNPERVDI, "true")
234 except:
235 pass
237 def is_shared(self):
238 if not self._is_shared:
239 self._is_shared = self.session.xenapi.SR.get_shared(self.sr_ref)
240 return self._is_shared
242 def create(self, uuid, size) -> None:
243 """Create this repository.
244 This operation may delete existing data.
246 The operation is NOT idempotent. The operation will fail
247 if an SR of the same UUID and driver type already exits.
249 Returns:
250 None
251 Raises:
252 SRUnimplementedMethod
253 """
254 raise xs_errors.XenError('Unimplemented')
256 def delete(self, uuid) -> None:
257 """Delete this repository and its contents.
259 This operation IS idempotent -- it will succeed if the repository
260 exists and can be deleted or if the repository does not exist.
261 The caller must ensure that all VDIs are deactivated and detached
262 and that the SR itself has been detached before delete().
263 The call will FAIL if any VDIs in the SR are in use.
265 Returns:
266 None
267 Raises:
268 SRUnimplementedMethod
269 """
270 raise xs_errors.XenError('Unimplemented')
272 def update(self, uuid) -> None:
273 """Refresh the fields in the SR object
275 Returns:
276 None
277 Raises:
278 SRUnimplementedMethod
279 """
280 # no-op unless individual backends implement it
281 return
283 def attach(self, uuid) -> None:
284 """Initiate local access to the SR. Initialises any
285 device state required to access the substrate.
287 Idempotent.
289 Returns:
290 None
291 Raises:
292 SRUnimplementedMethod
293 """
294 raise xs_errors.XenError('Unimplemented')
296 def after_master_attach(self, uuid) -> None:
297 """Perform actions required after attaching on the pool master
298 Return:
299 None
300 """
301 try:
302 self.scan(uuid)
303 except Exception as e:
304 util.SMlog("Error in SR.after_master_attach %s" % e)
305 msg_name = "POST_ATTACH_SCAN_FAILED"
306 msg_body = "Failed to scan SR %s after attaching, " \
307 "error %s" % (uuid, e)
308 self.session.xenapi.message.create(
309 msg_name, 2, "SR", uuid, msg_body)
311 def detach(self, uuid) -> None:
312 """Remove local access to the SR. Destroys any device
313 state initiated by the sr_attach() operation.
315 Idempotent. All VDIs must be detached in order for the operation
316 to succeed.
318 Returns:
319 None
320 Raises:
321 SRUnimplementedMethod
322 """
323 raise xs_errors.XenError('Unimplemented')
325 def probe(self) -> str:
326 """Perform a backend-specific scan, using the current dconf. If the
327 dconf is complete, then this will return a list of the SRs present of
328 this type on the device, if any. If the dconf is partial, then a
329 backend-specific scan will be performed, returning results that will
330 guide the user in improving the dconf.
332 Idempotent.
334 xapi will ensure that this is serialised wrt any other probes, or
335 attach or detach operations on this host.
337 Returns:
338 An XML fragment containing the scan results. These are specific
339 to the scan being performed, and the current backend.
340 Raises:
341 SRUnimplementedMethod
342 """
343 raise xs_errors.XenError('Unimplemented')
345 def scan(self, uuid) -> None:
346 """
347 Returns:
348 """
349 # Update SR parameters
350 self._db_update()
351 # Synchronise VDI list
352 scanrecord = ScanRecord(self)
353 scanrecord.synchronise()
355 def replay(self, uuid) -> None:
356 """Replay a multi-stage log entry
358 Returns:
359 None
360 Raises:
361 SRUnimplementedMethod
362 """
363 raise xs_errors.XenError('Unimplemented')
365 def content_type(self, uuid) -> str:
366 """Returns the 'content_type' of an SR as a string"""
367 return xmlrpc.client.dumps((str(self.sr_vditype), ), "", True)
369 def load(self, sr_uuid) -> None:
370 """Post-init hook"""
371 pass
373 def check_sr(self, sr_uuid) -> None:
374 """Hook to check SR health"""
375 pass
377 def vdi(self, uuid) -> 'VDI.VDI':
378 """Return VDI object owned by this repository"""
379 raise xs_errors.XenError('Unimplemented')
381 def forget_vdi(self, uuid) -> None:
382 vdi = self.session.xenapi.VDI.get_by_uuid(uuid)
383 self.session.xenapi.VDI.db_forget(vdi)
385 def cleanup(self) -> None:
386 # callback after the op is done
387 pass
389 def _db_update(self):
390 sr = self.session.xenapi.SR.get_by_uuid(self.uuid)
391 self.session.xenapi.SR.set_virtual_allocation(sr, str(self.virtual_allocation))
392 self.session.xenapi.SR.set_physical_size(sr, str(self.physical_size))
393 self.session.xenapi.SR.set_physical_utilisation(sr, str(self.physical_utilisation))
395 def _toxml(self):
396 dom = xml.dom.minidom.Document()
397 element = dom.createElement("sr")
398 dom.appendChild(element)
400 # Add default uuid, physical_utilisation, physical_size and
401 # virtual_allocation entries
402 for attr in ('uuid', 'physical_utilisation', 'virtual_allocation',
403 'physical_size'):
404 try:
405 aval = getattr(self, attr)
406 except AttributeError:
407 raise xs_errors.XenError(
408 'InvalidArg', opterr='Missing required field [%s]' % attr)
410 entry = dom.createElement(attr)
411 element.appendChild(entry)
412 textnode = dom.createTextNode(str(aval))
413 entry.appendChild(textnode)
415 # Add the default_vdi_visibility entry
416 entry = dom.createElement('default_vdi_visibility')
417 element.appendChild(entry)
418 if not self.default_vdi_visibility:
419 textnode = dom.createTextNode('False')
420 else:
421 textnode = dom.createTextNode('True')
422 entry.appendChild(textnode)
424 # Add optional label and description entries
425 for attr in ('label', 'description'):
426 try:
427 aval = getattr(self, attr)
428 except AttributeError:
429 continue
430 if aval:
431 entry = dom.createElement(attr)
432 element.appendChild(entry)
433 textnode = dom.createTextNode(str(aval))
434 entry.appendChild(textnode)
436 # Create VDI sub-list
437 if self.vdis:
438 for uuid in self.vdis:
439 if not self.vdis[uuid].deleted:
440 vdinode = dom.createElement("vdi")
441 element.appendChild(vdinode)
442 self.vdis[uuid]._toxml(dom, vdinode)
444 return dom
446 def _fromxml(self, str, tag):
447 dom = xml.dom.minidom.parseString(str)
448 objectlist = dom.getElementsByTagName(tag)[0]
449 taglist = {}
450 for node in objectlist.childNodes:
451 taglist[node.nodeName] = ""
452 for n in node.childNodes:
453 if n.nodeType == n.TEXT_NODE:
454 taglist[node.nodeName] += n.data
455 return taglist
457 def _splitstring(self, str):
458 elementlist = []
459 for i in range(0, len(str)):
460 elementlist.append(str[i])
461 return elementlist
463 def _mpathinit(self):
464 self.mpath = "false"
465 try:
466 if 'multipathing' in self.dconf and \ 466 ↛ 468line 466 didn't jump to line 468, because the condition on line 466 was never true
467 'multipathhandle' in self.dconf:
468 self.mpath = self.dconf['multipathing']
469 self.mpathhandle = self.dconf['multipathhandle']
470 else:
471 hconf = self.session.xenapi.host.get_other_config(self.host_ref)
472 self.mpath = hconf['multipathing']
473 self.mpathhandle = hconf.get('multipathhandle', 'dmp')
475 if self.mpath != "true": 475 ↛ 479line 475 didn't jump to line 479, because the condition on line 475 was never false
476 self.mpath = "false"
477 self.mpathhandle = "null"
479 if not os.path.exists("/opt/xensource/sm/mpath_%s.py" % self.mpathhandle): 479 ↛ 484line 479 didn't jump to line 484, because the condition on line 479 was never false
480 raise IOError("File does not exist = %s" % self.mpathhandle)
481 except:
482 self.mpath = "false"
483 self.mpathhandle = "null"
484 module_name = "mpath_%s" % self.mpathhandle
485 self.mpathmodule = __import__(module_name)
487 def _mpathHandle(self):
488 if self.mpath == "true": 488 ↛ 489line 488 didn't jump to line 489, because the condition on line 488 was never true
489 self.mpathmodule.activate()
490 else:
491 self.mpathmodule.deactivate()
493 def _pathrefresh(self, obj):
494 SCSIid = getattr(self, 'SCSIid')
495 self.dconf['device'] = self.mpathmodule.path(SCSIid)
496 super(obj, self).load(self.uuid)
498 def _setMultipathableFlag(self, SCSIid=''):
499 try:
500 sm_config = self.session.xenapi.SR.get_sm_config(self.sr_ref)
501 sm_config['multipathable'] = 'true'
502 self.session.xenapi.SR.set_sm_config(self.sr_ref, sm_config)
504 if self.mpath == "true" and len(SCSIid): 504 ↛ 505line 504 didn't jump to line 505, because the condition on line 504 was never true
505 cmd = ['/opt/xensource/sm/mpathcount.py', SCSIid]
506 util.pread2(cmd)
507 except:
508 pass
510 def check_dconf(self, key_list, raise_flag=True):
511 """ Checks if all keys in 'key_list' exist in 'self.dconf'.
513 Input:
514 key_list: a list of keys to check if they exist in self.dconf
515 raise_flag: if true, raise an exception if there are 1 or more
516 keys missing
518 Return: set() containing the missing keys (empty set() if all exist)
519 Raise: xs_errors.XenError('ConfigParamsMissing')
520 """
522 missing_keys = {key for key in key_list if key not in self.dconf}
524 if missing_keys and raise_flag:
525 errstr = 'device-config is missing the following parameters: ' + \
526 ', '.join([key for key in missing_keys])
527 raise xs_errors.XenError('ConfigParamsMissing', opterr=errstr)
529 return missing_keys
531 def _init_preferred_image_formats(self) -> None:
532 self.preferred_image_formats = parseImageFormats(
533 self.dconf and self.dconf.get('preferred-image-formats'),
534 DEFAULT_IMAGE_FORMATS
535 )
537 def _get_snap_vdi_type(self, vdi_type: str, size: int) -> str:
538 if VdiType.isCowImage(vdi_type): 538 ↛ 540line 538 didn't jump to line 540, because the condition on line 538 was never false
539 return vdi_type
540 if vdi_type == VdiType.RAW:
541 for image_format in self.preferred_image_formats:
542 if getCowUtilFromImageFormat(image_format).canSnapshotRaw(size):
543 return getVdiTypeFromImageFormat(image_format)
544 raise xs_errors.XenError('VDISnapshot', opterr=f"cannot snap from `{vdi_type}`")
546class ScanRecord:
547 def __init__(self, sr):
548 self.sr = sr
549 self.__xenapi_locations = {}
550 self.__xenapi_records = util.list_VDI_records_in_sr(sr)
551 for vdi in list(self.__xenapi_records.keys()): 551 ↛ 552line 551 didn't jump to line 552, because the loop on line 551 never started
552 self.__xenapi_locations[util.to_plain_string(self.__xenapi_records[vdi]['location'])] = vdi
553 self.__sm_records = {}
554 for vdi in list(sr.vdis.values()):
555 # We initialise the sm_config field with the values from the database
556 # The sm_config_overrides contains any new fields we want to add to
557 # sm_config, and also any field to delete (by virtue of having
558 # sm_config_overrides[key]=None)
559 try:
560 if not hasattr(vdi, "sm_config"): 560 ↛ 566line 560 didn't jump to line 566, because the condition on line 560 was never false
561 vdi.sm_config = self.__xenapi_records[self.__xenapi_locations[vdi.location]]['sm_config'].copy()
562 except:
563 util.SMlog("missing config for vdi: %s" % vdi.location)
564 vdi.sm_config = {}
566 if "image-format" not in vdi.sm_config: 566 ↛ 572line 566 didn't jump to line 572, because the condition on line 566 was never false
567 try:
568 vdi.sm_config["image-format"] = getImageStringFromVdiType(vdi.vdi_type)
569 except:
570 pass # No image format for this VDI type.
572 vdi._override_sm_config(vdi.sm_config)
574 self.__sm_records[vdi.location] = vdi
576 xenapi_locations = set(self.__xenapi_locations.keys())
577 sm_locations = set(self.__sm_records.keys())
579 # These ones are new on disk
580 self.new = sm_locations.difference(xenapi_locations)
581 # These have disappeared from the disk
582 self.gone = xenapi_locations.difference(sm_locations)
583 # These are the ones which are still present but might have changed...
584 existing = sm_locations.intersection(xenapi_locations)
585 # Synchronise the uuid fields using the location as the primary key
586 # This ensures we know what the UUIDs are even though they aren't stored
587 # in the storage backend.
588 for location in existing: 588 ↛ 589line 588 didn't jump to line 589, because the loop on line 588 never started
589 sm_vdi = self.get_sm_vdi(location)
590 xenapi_vdi = self.get_xenapi_vdi(location)
591 sm_vdi.uuid = util.default(sm_vdi, "uuid", lambda: xenapi_vdi['uuid'])
593 # Only consider those whose configuration looks different
594 self.existing = [x for x in existing if not(self.get_sm_vdi(x).in_sync_with_xenapi_record(self.get_xenapi_vdi(x)))]
596 if len(self.new) != 0:
597 util.SMlog("new VDIs on disk: " + repr(self.new))
598 if len(self.gone) != 0: 598 ↛ 599line 598 didn't jump to line 599, because the condition on line 598 was never true
599 util.SMlog("VDIs missing from disk: " + repr(self.gone))
600 if len(self.existing) != 0: 600 ↛ 601line 600 didn't jump to line 601, because the condition on line 600 was never true
601 util.SMlog("VDIs changed on disk: " + repr(self.existing))
603 def get_sm_vdi(self, location):
604 return self.__sm_records[location]
606 def get_xenapi_vdi(self, location):
607 return self.__xenapi_records[self.__xenapi_locations[location]]
609 def all_xenapi_locations(self):
610 return set(self.__xenapi_locations.keys())
612 def synchronise_new(self):
613 """Add XenAPI records for new disks"""
614 for location in self.new:
615 vdi = self.get_sm_vdi(location)
616 util.SMlog("Introducing VDI with location=%s" % (vdi.location))
617 vdi._db_introduce()
619 def synchronise_gone(self):
620 """Delete XenAPI record for old disks"""
621 for location in self.gone: 621 ↛ 622line 621 didn't jump to line 622, because the loop on line 621 never started
622 vdi = self.get_xenapi_vdi(location)
623 util.SMlog("Forgetting VDI with location=%s uuid=%s" % (util.to_plain_string(vdi['location']), vdi['uuid']))
624 try:
625 self.sr.forget_vdi(vdi['uuid'])
626 except XenAPI.Failure as e:
627 if util.isInvalidVDI(e):
628 util.SMlog("VDI %s not found, ignoring exception" %
629 vdi['uuid'])
630 else:
631 raise
633 def synchronise_existing(self):
634 """Update existing XenAPI records"""
635 for location in self.existing: 635 ↛ 636line 635 didn't jump to line 636, because the loop on line 635 never started
636 vdi = self.get_sm_vdi(location)
638 util.SMlog("Updating VDI with location=%s uuid=%s" % (vdi.location, vdi.uuid))
639 vdi._db_update()
641 def synchronise(self):
642 """Perform the default SM -> xenapi synchronisation; ought to be good enough
643 for most plugins."""
644 self.synchronise_new()
645 self.synchronise_gone()
646 self.synchronise_existing()