Coverage for drivers/qcow2util.py : 26%
Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from sm_typing import Any, Callable, Dict, Final, List, Optional, Tuple, cast, override
2from typing import BinaryIO
4import errno
5import os
6import re
7import time
8import struct
9import zlib
10from pathlib import Path
12import util
13import xs_errors
14from blktap2 import TapCtl
15from cowutil import CowUtil, CowImageInfo
16from lvmcache import LVMCache
17from constants import NS_PREFIX_LVM, VG_PREFIX
19MAX_QCOW_CHAIN_LENGTH: Final = 30
21QCOW2_DEFAULT_CLUSTER_SIZE: Final = 64 * 1024 # 64 KiB
23MIN_QCOW_SIZE: Final = QCOW2_DEFAULT_CLUSTER_SIZE
25MAX_QCOW_SIZE: Final = 16 * 1024 * 1024 * 1024 * 1024
27QEMU_IMG: Final = "/usr/bin/qemu-img"
28QCOW2_HELPER = "/opt/xensource/libexec/qcow2_helper"
30QCOW2_TYPE: Final = "qcow2"
31RAW_TYPE: Final = "raw"
33class QCowUtil(CowUtil):
35 # We followed specifications found here:
36 # https://github.com/qemu/qemu/blob/master/docs/interop/qcow2.txt
38 QCOW2_MAGIC = 0x514649FB # b"QFI\xfb": Magic number for QCOW2 files
39 QCOW2_HEADER_SIZE = 104 # In fact the last information we need is at offset 40-47
40 QCOW2_L2_SIZE = QCOW2_DEFAULT_CLUSTER_SIZE
41 QCOW2_BACKING_FILE_OFFSET = 8
43 ALLOCATED_ENTRY_BIT = (
44 0x8000_0000_0000_0000 # Bit 63 is the allocated bit for standard cluster
45 )
46 CLUSTER_TYPE_BIT = 0x4000_0000_0000_0000 # 0 for standard, 1 for compressed cluster
47 L2_OFFSET_MASK = 0x00FF_FFFF_FFFF_FF00 # Bits 9-55 are offset of L2 table.
48 CLUSTER_DESCRIPTION_MASK = 0x3FFF_FFFF_FFFF_FFFF # Bit 0-61 is cluster description
49 STANDARD_CLUSTER_OFFSET_MASK = (
50 0x00FF_FFFF_FFFF_FF00 # Bits 9-55 are offset of standard cluster
51 )
53 def __init__(self):
54 self.qcow_read = False
56 def _read_qcow2(self, path: str, read_clusters: bool = False):
57 phys_disk_size = self.getSizePhys(path)
58 with open(path, "rb") as qcow2_file:
59 self.filename = path # Keep the filename if clean is called
60 self.header = self._read_qcow2_header(qcow2_file)
61 if read_clusters:
62 self.l1 = self._get_l1_entries(qcow2_file)
63 # The l1_to_l2 allows to get L2 entries for a given L1. If L1 entry
64 # is not allocated we store an empty list.
65 self.l1_to_l2: Dict[int, List[int]] = {}
67 for l1_entry in self.l1:
68 l2_offset = l1_entry & self.L2_OFFSET_MASK
69 if l2_offset == 0:
70 self.l1_to_l2[l1_entry] = []
71 elif l2_offset > phys_disk_size: #TODO: This sometime happen for a correct VDI (while coalescing online?)
72 raise xs_errors.XenError("VDISize", "L2 Offset is bigger than physical disk {}".format(path))
73 else:
74 self.l1_to_l2[l1_entry] = self._get_l2_entries(
75 qcow2_file, l2_offset
76 )
77 self.qcow_read = True
79 def _get_l1_entries(self, file: BinaryIO) -> List[int]:
80 """Returns the list of all L1 entries.
82 Args:
83 file: The qcow2 file object.
85 Returns:
86 list: List of all L1 entries
87 """
88 l1_table_offset = self.header["l1_table_offset"]
89 file.seek(l1_table_offset)
91 l1_table_size = self.header["l1_size"] * 8 # Each L1 entry is 8 bytes
92 l1_table = file.read(l1_table_size)
94 return [
95 struct.unpack(">Q", l1_table[i : i + 8])[0]
96 for i in range(0, len(l1_table), 8)
97 ]
99 @staticmethod
100 def _get_l2_entries(file: BinaryIO, l2_offset: int) -> List[int]:
101 """Returns the list of all L2 entries at a given L2 offset.
103 Args:
104 file: The qcow2 file.
105 l2_offset: the L2 offset where to look for entries
107 Returns:
108 list: List of all L2 entries
109 """
110 # The size of L2 is 65536 bytes and each entry is 8 bytes.
111 file.seek(l2_offset)
112 l2_table = file.read(QCowUtil.QCOW2_L2_SIZE)
114 return [
115 struct.unpack(">Q", l2_table[i : i + 8])[0]
116 for i in range(0, len(l2_table), 8)
117 ]
119 @staticmethod
120 def _read_qcow2_backingfile(file: BinaryIO, backing_file_offset: int , backing_file_size: int) -> str:
121 if backing_file_offset == 0:
122 return ""
124 file.seek(backing_file_offset)
125 parent_name = file.read(backing_file_size)
126 return parent_name.decode("UTF-8")
128 @staticmethod
129 def _read_qcow2_header(file: BinaryIO) -> Dict[str, Any]:
130 """Returns a dict containing some information from QCow2 header.
132 Args:
133 file: The qcow2 file object.
135 Returns:
136 dict: magic, version, cluster_bits, l1_size and l1_table_offset.
138 Raises:
139 ValueError: if qcow2 magic is not recognized or cluster size not supported.
140 """
141 # The header is as follow:
142 #
143 # magic: u32, // Magic string "QFI\xfb"
144 # version: u32, // Version (2 or 3)
145 # backing_file_offset: u64, // Offset to the backing file name
146 # backing_file_size: u32, // Size of the backing file name
147 # cluster_bits: u32, // Bits used for addressing within a cluster
148 # size: u64, // Virtual disk size
149 # crypt_method: u32, // 0 = no encryption, 1 = AES encryption
150 # l1_size: u32, // Number of entries in the L1 table
151 # l1_table_offset: u64, // Offset to the active L1 table
152 # refcount_table_offset: u64, // Offset to the refcount table
153 # refcount_table_clusters: u32, // Number of clusters for the refcount table
154 # nb_snapshots: u32, // Number of snapshots in the image
155 # snapshots_offset: u64, // Offset to the snapshot table
157 file.seek(0)
158 header = file.read(QCowUtil.QCOW2_HEADER_SIZE)
159 (
160 magic,
161 version,
162 backing_file_offset,
163 backing_file_size,
164 cluster_bits,
165 size,
166 _,
167 l1_size,
168 l1_table_offset,
169 refcount_table_offset,
170 _,
171 _,
172 snapshots_offset,
173 ) = struct.unpack(">IIQIIQIIQQIIQ", header[:72])
175 if magic != QCowUtil.QCOW2_MAGIC:
176 raise ValueError("Not a valid QCOW2 file")
178 parent_name = QCowUtil._read_qcow2_backingfile(file, backing_file_offset, backing_file_size)
180 return {
181 "version": version,
182 "backing_file_offset": backing_file_offset,
183 "backing_file_size": backing_file_size,
184 "virtual_disk_size": size,
185 "cluster_bits": cluster_bits,
186 "l1_size": l1_size,
187 "l1_table_offset": l1_table_offset,
188 "refcount_table_offset": refcount_table_offset,
189 "snapshots_offset": snapshots_offset,
190 "parent": parent_name,
191 }
193 @staticmethod
194 def _is_l1_allocated(entry: int) -> bool:
195 """Checks if the given L1 entry is allocated.
197 If the offset is 0 then the L2 table and all clusters described
198 by this L2 table are unallocated.
200 Args:
201 entry: L1 entry
203 Returns:
204 bool: True if the L1 entry is allocated (ie has a valid offset).
205 False otherwise.
206 """
207 return (entry & QCowUtil.L2_OFFSET_MASK) != 0
209 @staticmethod
210 def _is_l2_allocated(entry: int) -> bool:
211 """Checks if a given entry is allocated.
213 Currently we only support standard clusters. And for standard clusters
214 the bit 63 is set to 1 for allocated ones or offset is not 0.
216 Args:
217 entry: L2 entry
219 Returns:
220 bool: Returns True if the L2 entry is allocated, False otherwise
222 Raises:
223 raise an exception if the cluster is not a standard one.
224 """
225 assert entry & QCowUtil.CLUSTER_TYPE_BIT == 0
226 return (entry & QCowUtil.ALLOCATED_ENTRY_BIT != 0) or (
227 entry & QCowUtil.STANDARD_CLUSTER_OFFSET_MASK != 0
228 )
230 @staticmethod
231 def _get_allocated_clusters(l2_entries: List[int]) -> List[int]:
232 """Get all allocated clusters in a given list of L2 entries.
234 Args:
235 l2_entries: A list of L2 entries.
237 Returns:
238 A list of all allocated entries
239 """
240 return [entry for entry in l2_entries if QCowUtil._is_l2_allocated(entry)]
242 @staticmethod
243 def _get_cluster_to_byte(clusters: int, cluster_bits: int) -> int:
244 # (1 << cluster_bits) give cluster size in byte
245 return clusters * (1 << cluster_bits)
247 def _get_number_of_allocated_clusters(self) -> int:
248 """Get the number of allocated clusters.
250 Args:
251 self: A QcowInfo object.
253 Returns:
254 An integer that is the list of allocated clusters.
255 """
256 assert(self.qcow_read)
258 allocated_clusters = 0
260 for l2_entries in self.l1_to_l2.values():
261 allocated_clusters += len(self._get_allocated_clusters(l2_entries))
263 return allocated_clusters
265 @staticmethod
266 def _move_backing_file(
267 f: BinaryIO, old_offset: int, new_offset: int, data_size: int
268 ) -> None:
269 """Move a number of bytes from old_offset to new_offset and replaces the old
270 value by 0s. It is up to the caller to save the current position in the file
271 if needed.
273 Args:
274 f: the file the will be modified
275 old_offset: the current offset
276 new_offset: the new offset where we want to move data
277 data_size: Size in bytes of data that we want to move
279 Returns:
280 Nothing but the file f is modified and the position in the file also.
281 """
282 # Read the string at backing_file_offset
283 f.seek(old_offset)
284 data = f.read(data_size)
286 # Write zeros at the original location
287 f.seek(old_offset)
288 f.write(b"\x00" * data_size)
290 # Write the string to the new location
291 f.seek(new_offset)
292 f.write(data)
294 def _add_or_find_custom_header(self) -> int:
295 """Add custom header at the end of header extensions
297 It finds the end of the header extensions and add the custom header.
298 If the header already exists nothing is done.
300 Args:
302 Returns:
303 It returns the data offset where custom header is found or created.
304 If data offset is 0 something weird happens.
305 The qcow2 file in self.filename can be modified.
306 """
307 assert self.qcow_read
309 header_length = 72 # This is the default value for version 2 images
311 custom_header_type = 0x76617465 # vate: it is easy to recognize with hexdump -C
312 custom_header_length = 8
313 custom_header_data = 0
314 # We don't need padding because we are already aligned
315 custom_header = struct.pack(
316 ">IIQ", custom_header_type, custom_header_length, custom_header_data
317 )
319 with open(self.filename, "rb+") as qcow2_file:
320 if self.header["version"] == 3:
321 qcow2_file.seek(100) # 100 is the offset of header_length
322 header_length = int.from_bytes(qcow2_file.read(4), "big")
324 # After the image header we found Header extension. So we need to find the end of
325 # the header extension area and add our custom header.
326 qcow2_file.seek(header_length)
328 custom_data_offset = 0
330 while True:
331 ext_type = int.from_bytes(qcow2_file.read(4), "big")
332 ext_len = int.from_bytes(qcow2_file.read(4), "big")
334 if ext_type == custom_header_type:
335 # A custom header is already there
336 custom_data_offset = qcow2_file.tell()
337 break
339 if ext_type == 0x00000000:
340 # End mark found. If we found the end mark it means that we didn't find
341 # the custom header. So we need to add it.
342 custom_data_offset = qcow2_file.tell()
344 # We will overwrite the end marker so rewind a little bit to
345 # write the new type extension and the new length. But if there is
346 # a backing file we need to move it to make some space.
347 if self.header["backing_file_offset"]:
348 # Keep current position
349 saved_pos = qcow2_file.tell()
351 bf_offset = self.header["backing_file_offset"]
352 bf_size = self.header["backing_file_size"]
353 bf_new_offset = bf_offset + len(custom_header)
354 self._move_backing_file(
355 qcow2_file, bf_offset, bf_new_offset, bf_size
356 )
358 # Update the header to match the new backing file offset
359 self.header["backing_file_offset"] = bf_new_offset
360 qcow2_file.seek(self.QCOW2_BACKING_FILE_OFFSET)
361 qcow2_file.write(struct.pack(">Q", bf_new_offset))
363 # Restore saved position
364 qcow2_file.seek(saved_pos)
366 qcow2_file.seek(-8, 1)
367 qcow2_file.write(custom_header)
368 break
370 # Round up the header extension size to the next multiple of 8
371 ext_len = (ext_len + 7) & 0xFFFFFFF8
372 qcow2_file.seek(ext_len, 1)
374 return custom_data_offset
376 def _set_l1_zero(self):
377 zero = int(0).to_bytes(1, "little")
378 nb_of_entries_per_cluster = QCOW2_DEFAULT_CLUSTER_SIZE/8
379 return list(zero * int(nb_of_entries_per_cluster/8))
381 def _set_l2_zero(self, b, i):
382 return b & ~(1 << i)
384 def _set_l2_one(self, b, i):
385 return b | (1 << i)
387 def _create_bitmap(self) -> bytes:
388 idx: int = 0
389 bitmap = list()
390 b = 0
391 for l1_entry in self.l1:
392 if not self._is_l1_allocated(l1_entry):
393 bitmap.extend(self._set_l1_zero())
394 continue
396 l2_table = self.l1_to_l2[l1_entry] #L2 is cluster_size/8 entries of cluster_size page
397 for l2_entry in l2_table:
398 if self._is_l2_allocated(l2_entry):
399 b = self._set_l2_one(b, idx)
400 else:
401 b = self._set_l2_zero(b, idx)
402 idx += 1
403 if idx == 8:
404 bitmap.append(b)
405 b = 0
406 idx = 0
407 return struct.pack("B"*len(bitmap), *bitmap)
409 # ----
410 # Implementation of CowUtil
411 # ----
413 @override
414 def getMinImageSize(self) -> int:
415 return MIN_QCOW_SIZE
417 @override
418 def getMaxImageSize(self) -> int:
419 return MAX_QCOW_SIZE
421 @override
422 def getBlockSize(self, path: str) -> int:
423 self._read_qcow2(path)
424 return 1 << self.header["cluster_bits"]
426 @override
427 def getFooterSize(self) -> int:
428 return 0
430 @override
431 def getDefaultPreallocationSizeVirt(self) -> int:
432 return MAX_QCOW_SIZE
434 @override
435 def getMaxChainLength(self) -> int:
436 return MAX_QCOW_CHAIN_LENGTH
438 @override
439 def calcOverheadEmpty(self, virtual_size: int) -> int:
440 size_l1 = QCOW2_DEFAULT_CLUSTER_SIZE
441 size_header = QCOW2_DEFAULT_CLUSTER_SIZE
442 size_l2 = (virtual_size * 8) / QCOW2_DEFAULT_CLUSTER_SIZE #It is only an estimation
444 size = size_l1 + size_l2 + size_header
446 return util.roundup(QCOW2_DEFAULT_CLUSTER_SIZE, size)
448 @override
449 def calcOverheadBitmap(self, virtual_size: int) -> int:
450 return 0 #TODO: What do we send back?
452 @override
453 def getInfo(
454 self,
455 path: str,
456 extractUuidFunction: Callable[[str], str],
457 includeParent: bool = True,
458 resolveParent: bool = True,
459 useBackupFooter: bool = False
460 ) -> CowImageInfo:
461 #TODO: handle resolveParent
462 self._read_qcow2(path)
463 basename = Path(path).name
464 uuid = extractUuidFunction(basename)
465 cowinfo = CowImageInfo(uuid)
466 cowinfo.path = basename
467 cowinfo.sizeVirt = self.header["virtual_disk_size"]
468 cowinfo.sizePhys = self.getSizePhys(path)
469 cowinfo.hidden = self.getHidden(path)
470 cowinfo.sizeAllocated = self.getAllocatedSize(path)
471 if includeParent:
472 parent_path = self.header["parent"]
473 if parent_path != "":
474 cowinfo.parentPath = parent_path
475 cowinfo.parentUuid = extractUuidFunction(parent_path)
477 return cowinfo
479 @override
480 def getInfoFromLVM(
481 self, lvName: str, extractUuidFunction: Callable[[str], str], vgName: str
482 ) -> Optional[CowImageInfo]:
483 lvcache = LVMCache(vgName)
484 return self._getInfoLV(lvcache, extractUuidFunction, vgName, lvName)
486 def _getInfoLV(
487 self, lvcache: LVMCache, extractUuidFunction: Callable[[str], str], vgName: str, lvName: str
488 ) -> Optional[CowImageInfo]:
489 lvPath = "/dev/{}/{}".format(vgName, lvName)
490 lvcache.refresh()
491 if lvName not in lvcache.lvs:
492 util.SMlog("{} does not exist anymore".format(lvName))
493 return None
495 vdiUuid = extractUuidFunction(lvPath)
496 srUuid = vgName.replace(VG_PREFIX, "")
498 ns = NS_PREFIX_LVM + srUuid
499 lvcache.activate(ns, vdiUuid, lvName, False)
500 try:
501 cowinfo = self.getInfo(lvPath, extractUuidFunction)
502 finally:
503 lvcache.deactivate(ns, vdiUuid, lvName, False)
504 return cowinfo
506 @override
507 def getAllInfoFromVG(
508 self,
509 pattern: str,
510 extractUuidFunction: Callable[[str], str],
511 vgName: Optional[str] = None,
512 parents: bool = False,
513 exitOnError: bool = False
514 ) -> Dict[str, CowImageInfo]:
515 result: Dict[str, CowImageInfo] = dict()
516 #TODO: handle exitOnError
517 if vgName: 517 ↛ 518line 517 didn't jump to line 518, because the condition on line 517 was never true
518 reg = re.compile(pattern)
519 lvcache = LVMCache(vgName)
520 lvcache.refresh()
521 # We get size in lvcache.lvs[lvName].size (in bytes)
522 # We could read the header from the PV directly
523 lvList = list(lvcache.lvs.keys())
524 for lvName in lvList:
525 # lvinfo = lvcache.lvs[lvName]
526 if reg.match(lvName):
527 cowinfo = self._getInfoLV(lvcache, extractUuidFunction, vgName, lvName)
528 if cowinfo is None: #We get None if the LV stopped existing in the meanwhile
529 continue
530 result[cowinfo.uuid] = cowinfo
531 if parents:
532 parentUuid = cowinfo.parentUuid
533 parentPath = cowinfo.parentPath
534 while parentUuid != "":
535 parentLvName = parentPath.split("/")[-1]
536 parent_cowinfo = self._getInfoLV(lvcache, extractUuidFunction, vgName, parentLvName)
537 if parent_cowinfo is None: #Parent disappeared while scanning
538 raise util.SMException("Parent of {} wasn't found during scan".format(lvName))
539 result[parent_cowinfo.uuid] = parent_cowinfo
540 parentUuid = parent_cowinfo.parentUuid
541 parentPath = parent_cowinfo.parentPath
542 return result
543 else:
544 pattern_p: Path = Path(pattern)
545 list_qcow = list(pattern_p.parent.glob(pattern_p.name))
546 for qcow in list_qcow: 546 ↛ 547line 546 didn't jump to line 547, because the loop on line 546 never started
547 qcow_str = str(qcow)
548 info = self.getInfo(qcow_str, extractUuidFunction)
549 result[info.uuid] = info
550 return result
552 @override
553 def getParent(self, path: str, extractUuidFunction: Callable[[str], str]) -> Optional[str]:
554 parent = self.getParentNoCheck(path)
555 if parent:
556 return extractUuidFunction(parent)
557 return None
559 @override
560 def getParentNoCheck(self, path: str) -> Optional[str]:
561 self._read_qcow2(path)
562 parent_path = self.header["parent"]
563 if parent_path == "":
564 return None
565 return parent_path
567 @override
568 def hasParent(self, path: str) -> bool:
569 if self.getParentNoCheck(path):
570 return True
571 return False
573 @override
574 def setParent(self, path: str, parentPath: str, parentRaw: bool) -> None:
575 pid_openers = util.get_openers_pid(path)
576 if pid_openers:
577 util.SMlog("Rebasing while process {} has the VDI opened".format(pid_openers))
579 parentType = QCOW2_TYPE
580 if parentRaw:
581 parentType = RAW_TYPE
582 cmd = [QEMU_IMG, "rebase", "-u", "-f", QCOW2_TYPE, "-F", parentType, "-b", parentPath, path]
583 self._ioretry(cmd)
585 @override
586 def getHidden(self, path: str) -> bool:
587 """Get hidden property according to the value b
589 Args:
591 Returns:
592 True if hidden is set, False otherwise
593 """
594 self._read_qcow2(path)
595 custom_data_offset = self._add_or_find_custom_header()
596 if custom_data_offset == 0:
597 raise util.SMException("Custom data offset not found... should not reach this")
599 with open(path, "rb") as qcow2_file:
600 qcow2_file.seek(custom_data_offset)
601 hidden = qcow2_file.read(1)
602 if hidden == b"\x00":
603 return False
604 return True
606 @override
607 def setHidden(self, path: str, hidden: bool = True) -> None:
608 """Set hidden property according to the value b
610 Args:
611 bool: True if you want to set the property. False otherwise
613 Returns:
614 nothing. If the custom headers is not found it is created so the
615 qcow file can be modified.
616 """
617 self._read_qcow2(path)
618 custom_data_offset = self._add_or_find_custom_header()
619 if custom_data_offset == 0:
620 raise util.SMException("Custom data offset not found... should not reach this")
622 with open(self.filename, "rb+") as qcow2_file:
623 qcow2_file.seek(custom_data_offset)
624 if hidden:
625 qcow2_file.write(b"\x01")
626 else:
627 qcow2_file.write(b"\x00")
629 @override
630 def getSizeVirt(self, path: str) -> int:
631 self._read_qcow2(path)
632 return self.header['virtual_disk_size']
634 @override
635 def setSizeVirt(self, path: str, size: int, jFile: str) -> None:
636 """
637 size: byte
638 jFile: a journal file used for resizing with VHD, not useful for QCOW2
639 """
640 cmd = [QEMU_IMG, "resize", path, str(size)]
641 self._ioretry(cmd)
643 @override
644 def setSizeVirtFast(self, path: str, size: int) -> None:
645 self.setSizeVirt(path, size, "")
647 @override
648 def getMaxResizeSize(self, path: str) -> int:
649 return 0
651 @override
652 def getSizePhys(self, path: str) -> int:
653 size = os.stat(path).st_size
654 if size == 0:
655 size = int(self._ioretry(["blockdev", "--getsize64", path]))
656 return size
658 @override
659 def setSizePhys(self, path: str, size: int, debug: bool = True) -> None:
660 pass #TODO: Doesn't exist for QCow2, do we need to use it?
662 @override
663 def getAllocatedSize(self, path: str) -> int:
664 cmd = [QCOW2_HELPER, "allocated", path]
665 return int(self._ioretry(cmd))
667 @override
668 def getResizeJournalSize(self) -> int:
669 return 0
671 @override
672 def killData(self, path: str) -> None:
673 """Remove all data and reset L1/L2 table.
675 Args:
676 self: The QcowInfo object.
678 Returns:
679 nothing.
680 """
681 self._read_qcow2(path, read_clusters=True)
682 # We need to reset L1 entries and then just truncate the file right
683 # after L1 entries
684 with open(self.filename, "r+b") as file:
685 l1_table_offset = self.header["l1_table_offset"]
686 file.seek(l1_table_offset)
688 l1_table_size = (
689 self.header["l1_size"] * 8
690 ) # size in bytes, each entry is 8 bytes
691 file.write(b"\x00" * l1_table_size)
692 file.truncate(l1_table_offset + l1_table_size)
694 @override
695 def getDepth(self, path: str) -> int:
696 cmd = [QEMU_IMG, "info", "--backing-chain", "--output=json", path]
697 ret = str(self._ioretry(cmd))
698 depth = len(re.findall("\"backing-filename\"", ret))+1
699 #chain depth is beginning at one for VHD, meaning a VHD without parent has depth = 1
700 return depth
702 @override
703 def getBlockBitmap(self, path: str) -> bytes:
704 cmd = [QCOW2_HELPER, "bitmap", path]
705 text = cast(bytes, self._ioretry(cmd, text=False))
706 return zlib.compress(text)
708 def _getTapdisk(self, path: str) -> Tuple[int, int]:
709 """
710 Return a tuple of (PID, Minor) for the given path
711 """
712 pid_openers = util.get_openers_pid(path)
713 if pid_openers:
714 if len(pid_openers) > 1:
715 raise xs_errors.XenError("Multiple openers for {}".format(path)) # TODO: There might be multiple PID? Yes, we can have the chain enabled for multiple leaf (i.e. after a clone), taken into account in cleanup.py
716 pid = pid_openers[0]
717 tapdiskList = TapCtl.list(pid=pid)
718 if len(tapdiskList) > 1: #TODO: There might more than one minor for this blktap?
719 raise xs_errors.XenError("TapdiskAlreadyRunning", "There is multiple minor for this tapdisk process")
720 minor = tapdiskList[0]["minor"]
721 return (pid, minor)
722 raise xs_errors.XenError("TapdiskFailed", "No tapdisk process found for {}".format(path))
724 @override
725 def coalesceOnline(self, path: str) -> int:
726 pid, minor = self._getTapdisk(path)
727 logger = util.LoggerCounter(10)
729 try:
730 TapCtl.commit(pid, minor, QCOW2_TYPE, path)
731 # We need to wait for query to return concluded
732 # We are technically ininterruptible since being interrupted will only stop checking if the job is done.
733 # We need to call `tap-ctl cancel` if we are interrupted, it is done in cleanup.py code.
735 status, nb, _ = TapCtl.query(pid, minor)
736 if status == "undefined":
737 util.SMlog("Tapdisk {} (m: {}) coalesce status undefined for {}".format(pid, minor, path))
738 return 0
740 while status != "concluded":
741 time.sleep(1)
742 status, nb, _ = TapCtl.query(pid, minor, quiet=True)
743 logger.log("Got status {} for tapdisk {} (m: {})".format(status, pid, minor))
744 return nb
745 except TapCtl.CommandFailure:
746 util.SMlog("Query command failed on tapdisk instance {}. Raising...".format(pid))
747 raise
749 @override
750 def cancelCoalesceOnline(self, path: str) -> None:
751 pid, minor = self._getTapdisk(path)
753 try:
754 TapCtl.cancel_commit(pid, minor)
755 except TapCtl.CommandFailure:
756 util.SMlog("Cancel command failed on tapdisk instance {}. Raising...".format(pid))
757 raise
759 @override
760 def coalesce(self, path: str) -> int:
761 # -d on commit make it not empty the original image since we don't intend to keep it
762 cmd = [QEMU_IMG, "commit", "-f", QCOW2_TYPE, path, "-d"]
763 ret = cast(str, self._ioretry(cmd)) # Allows to parse for byte coalesced, our qemu-img is supposed to be patched to output it.
764 lines = ret.splitlines()
765 if re.match("Image committed.", lines[-1]):
766 res_line = lines[-2]
767 else:
768 res_line = lines[-1]
770 results = re.match(r"\((\d+)/(\d+)\)", res_line)
771 if results:
772 committed_bytes = int(results.group(1))
773 return committed_bytes
774 raise xs_errors.XenError("TapdiskFailed", "Couldn't get commited size from qemu-img commit call") # TODO: We might not want to raise in this case, it would break if the qemu-img called isn't modified to print the coalesce result even if it succeeded in coalesceing
776 @override
777 def create(self, path: str, size: int, static: bool, msize: int = 0, block_size: Optional[int] = None) -> None:
778 cmd = [QEMU_IMG, "create", "-f", QCOW2_TYPE, path, str(size)]
779 if static:
780 cmd.extend(["-o", "preallocation=full"])
781 if block_size:
782 cmd.extend(["-o", f"cluster_size={str(block_size)}"])
783 self._ioretry(cmd)
784 self.setHidden(path, False) #We add hidden header at creation
786 @override
787 def snapshot(
788 self,
789 path: str,
790 parent: str,
791 parentRaw: bool,
792 msize: int = 0,
793 checkEmpty: bool = True
794 ) -> None:
795 if parentRaw:
796 parent_type = RAW_TYPE
797 parent_cluster_size = QCOW2_DEFAULT_CLUSTER_SIZE
798 else:
799 parent_type = QCOW2_TYPE
800 parent_cluster_size = self.getBlockSize(parent)
802 cmd = [QEMU_IMG, "create", "-f", QCOW2_TYPE, "-b", parent, "-F", parent_type, "-o", f"cluster_size={parent_cluster_size}", path]
803 self._ioretry(cmd)
804 self.setHidden(path, False) #We add hidden header at creation
806 @override
807 def canSnapshotRaw(self, size: int) -> bool:
808 return True
810 @override
811 def check(
812 self,
813 path: str,
814 ignoreMissingFooter: bool = False,
815 fast: bool = False
816 ) -> CowUtil.CheckResult:
817 cmd = [QEMU_IMG, "check", path]
818 try:
819 self._ioretry(cmd)
820 return CowUtil.CheckResult.Success
821 except util.CommandException as e:
822 if e.code in (errno.EROFS, errno.EMEDIUMTYPE):
823 return CowUtil.CheckResult.Unavailable
824 # 1/EPERM is error in internal during check
825 # 2/ENOENT is QCOW corrupted
826 # 3/ESRCH is QCow has leaked clusters
827 # 63/ENOSR is check unavailable on this image type
828 return CowUtil.CheckResult.Fail
830 @override
831 def revert(self, path: str, jFile: str) -> None:
832 pass #Used to get back from a failed operation using a journal, NOOP for qcow2
834 @override
835 def repair(self, path: str) -> None:
836 cmd = [QEMU_IMG, "check", "-f", QCOW2_TYPE, "-r", "all", path]
837 self._ioretry(cmd)
839 @override
840 def validateAndRoundImageSize(self, size: int) -> int:
841 if size < 0 or size > MAX_QCOW_SIZE:
842 raise xs_errors.XenError(
843 "VDISize",
844 opterr="VDI size must be between {} MB and {} MB".format(MIN_QCOW_SIZE // (1024*1024), MAX_QCOW_SIZE // (1024 * 1024))
845 )
847 return util.roundup(QCOW2_DEFAULT_CLUSTER_SIZE, size)
849 @override
850 def getKeyHash(self, path: str) -> Optional[str]:
851 pass
853 @override
854 def setKey(self, path: str, key_hash: str) -> None:
855 pass
857 @override
858 def isCoalesceableOnRemote(self) -> bool:
859 return True