Coverage for drivers/vhdutil.py: 51%

Hot-keys on this page

r m x p toggle line displays

j k next/prev highlighted chunk

0 (zero) top of page

1 (one) first highlighted chunk

1# Copyright (C) Citrix Systems Inc.

3# This program is free software; you can redistribute it and/or modify

4# it under the terms of the GNU Lesser General Public License as published

5# by the Free Software Foundation; version 2.1 only.

7# This program is distributed in the hope that it will be useful,

8# but WITHOUT ANY WARRANTY; without even the implied warranty of

9# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

10# GNU Lesser General Public License for more details.

11#

12# You should have received a copy of the GNU Lesser General Public License

13# along with this program; if not, write to the Free Software Foundation, Inc.,

14# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA

15#

16# Helper functions pertaining to VHD operations

17#

19from sm_typing import Callable, Dict, Final, Optional, Sequence, cast, override

21from abc import abstractmethod

23import errno

24import os

25import re

26import zlib

28import util

29import XenAPI # pylint: disable=import-error

30import xs_errors

32from cowutil import CowImageInfo, CowUtil, ImageFormat

34# ------------------------------------------------------------------------------

36MIN_VHD_SIZE: Final = 2 * 1024 * 1024

37MAX_VHD_SIZE: Final = 2040 * 1024 * 1024 * 1024

38VHD_MAX_VOLUME_SIZE: Final = 2 * 1024 * 1024 * 1024 * 1024

40MAX_VHD_JOURNAL_SIZE: Final = 6 * 1024 * 1024 # 2MB VHD block size, max 2TB VHD size.

42VHD_BLOCK_SIZE: Final = 2 * 1024 * 1024

44VHD_FOOTER_SIZE: Final = 512

46VHD_SECTOR_SIZE: Final = 512

48MAX_VHD_CHAIN_LENGTH: Final = 30

50VHD_UTIL: Final = "/usr/bin/vhd-util"

52OPT_LOG_ERR: Final = "--debug"

54# ------------------------------------------------------------------------------

56class VhdUtil(CowUtil):

57 @override

58 def getMinImageSize(self) -> int:

59 return MIN_VHD_SIZE

61 @override

62 def getMaxImageSize(self) -> int:

63 return MAX_VHD_SIZE

65 @override

66 def getBlockSize(self, path: str) -> int:

67 return VHD_BLOCK_SIZE

69 @override

70 def getFooterSize(self) -> int:

71 return VHD_FOOTER_SIZE

73 @override

74 def getDefaultPreallocationSizeVirt(self) -> int:

75 return VHD_MAX_VOLUME_SIZE

77 @override

78 def getMaxChainLength(self) -> int:

79 return MAX_VHD_CHAIN_LENGTH

81 @override

82 def calcOverheadEmpty(self, virtual_size: int, block_size: Optional[int] = None) -> int:

83 """

84 Calculate the VHD space overhead (metadata size) for an empty VDI of

85 size virtual_size.

86 """

87 overhead = 0

88 size_mb = virtual_size // (1024 * 1024)

90 # Footer + footer copy + header + possible CoW parent locator fields

91 overhead = 3 * 1024

93 # BAT 4 Bytes per block segment

94 overhead += (size_mb // 2) * 4

95 overhead = util.roundup(512, overhead)

97 # BATMAP 1 bit per block segment

98 overhead += (size_mb // 2) // 8

99 overhead = util.roundup(4096, overhead)

100

101 return overhead

102

103 @override

104 def calcOverheadBitmap(self, virtual_size: int) -> int:

105 num_blocks = virtual_size // VHD_BLOCK_SIZE

106 if virtual_size % VHD_BLOCK_SIZE:

107 num_blocks += 1

108 return num_blocks * 4096

109

110 @override

111 def getInfo(

112 self,

113 path: str,

114 extractUuidFunction: Callable[[str], str],

115 includeParent: bool = True,

116 resolveParent: bool = True,

117 useBackupFooter: bool = False

118 ) -> CowImageInfo:

119 """

120 Get the VHD info. The parent info may optionally be omitted: vhd-util

121 tries to verify the parent by opening it, which results in error if the VHD

122 resides on an inactive LV.

123 """

124 opts = "-vsaf"

125 if includeParent: 125 ↛ 129line 125 didn't jump to line 129, because the condition on line 125 was never false

126 opts += "p"

127 if not resolveParent: 127 ↛ 128line 127 didn't jump to line 128, because the condition on line 127 was never true

128 opts += "u"

129 if useBackupFooter:

130 opts += "b"

131

132 ret = cast(str, self._ioretry([VHD_UTIL, "query", OPT_LOG_ERR, opts, "-n", path]))

133 fields = ret.strip().split("\n")

134 uuid = extractUuidFunction(path)

135 vhdInfo = CowImageInfo(uuid)

136 vhdInfo.sizeVirt = int(fields[0]) * 1024 * 1024

137 vhdInfo.sizePhys = int(fields[1])

138 nextIndex = 2

139 if includeParent: 139 ↛ 144line 139 didn't jump to line 144, because the condition on line 139 was never false

140 if fields[nextIndex].find("no parent") == -1: 140 ↛ 141line 140 didn't jump to line 141, because the condition on line 140 was never true

141 vhdInfo.parentPath = fields[nextIndex]

142 vhdInfo.parentUuid = extractUuidFunction(fields[nextIndex])

143 nextIndex += 1

144 vhdInfo.hidden = bool(int(fields[nextIndex].replace("hidden: ", "")))

145 vhdInfo.sizeAllocated = self._convertAllocatedSizeToBytes(int(fields[nextIndex+1]))

146 vhdInfo.path = path

147 return vhdInfo

148

149 @override

150 def getInfoFromLVM(

151 self, lvName: str, extractUuidFunction: Callable[[str], str], vgName: str

152 ) -> Optional[CowImageInfo]:

153 """

154 Get the VHD info. This function does not require the container LV to be

155 active, but uses LVs & VGs.

156 """

157 ret = cast(str, self._ioretry([VHD_UTIL, "scan", "-f", "-l", vgName, "-m", lvName]))

158 return self._parseVHDInfo(ret, extractUuidFunction)

159

160 @override

161 def getAllInfoFromVG(

162 self,

163 pattern: str,

164 extractUuidFunction: Callable[[str], str],

165 vgName: Optional[str] = None,

166 parents: bool = False,

167 exitOnError: bool = False

168 ) -> Dict[str, CowImageInfo]:

169 result: Dict[str, CowImageInfo] = dict()

170 cmd = [VHD_UTIL, "scan", "-f", "-m", pattern]

171 if vgName:

172 cmd.append("-l")

173 cmd.append(vgName)

174 if parents:

175 cmd.append("-a")

176 try:

177 ret = cast(str, self._ioretry(cmd))

178 except Exception as e:

179 util.SMlog("WARN: VHD scan failed: output: %s" % e)

180 ret = cast(str, self._ioretry(cmd + ["-c"]))

181 util.SMlog("WARN: VHD scan with NOFAIL flag, output: %s" % ret)

182 for line in ret.split('\n'):

183 if not line.strip():

184 continue

185 info = self._parseVHDInfo(line, extractUuidFunction)

186 if info:

187 if info.error != 0 and exitOnError:

188 # Just return an empty dict() so the scan will be done

189 # again by getParentChain. See CA-177063 for details on

190 # how this has been discovered during the stress tests.

191 return dict()

192 result[info.uuid] = info

193 else:

194 util.SMlog("WARN: VHD info line doesn't parse correctly: %s" % line)

195 return result

196

197 @override

198 def getParent(self, path: str, extractUuidFunction: Callable[[str], str]) -> Optional[str]:

199 ret = cast(str, self._ioretry([VHD_UTIL, "query", OPT_LOG_ERR, "-p", "-n", path]))

200 if ret.find("query failed") != -1 or ret.find("Failed opening") != -1:

201 raise util.SMException("VHD query returned %s" % ret)

202 if ret.find("no parent") != -1:

203 return None

204 return extractUuidFunction(ret)

205

206 @override

207 def getParentNoCheck(self, path: str) -> Optional[str]:

208 text = util.pread([VHD_UTIL, "read", "-p", "-n", "%s" % path])

209 util.SMlog(text)

210 for line in text.split("\n"):

211 if line.find("decoded name :") != -1:

212 val = line.split(":")[1].strip()

213 vdi = val.replace("--", "-")[-40:]

214 if vdi[1:].startswith("LV-"):

215 vdi = vdi[1:]

216 return vdi

217 return None

218

219 @override

220 def hasParent(self, path: str) -> bool:

221 """

222 Check if the VHD has a parent. A VHD has a parent iff its type is

223 'Differencing'. This function does not need the parent to actually

224 be present (e.g. the parent LV to be activated).

225 """

226 ret = cast(str, self._ioretry([VHD_UTIL, "read", OPT_LOG_ERR, "-p", "-n", path]))

227 # pylint: disable=no-member

228 m = re.match(r".*Disk type\s+: (\S+) hard disk.*", ret, flags=re.S)

229 if m:

230 vhd_type = m.group(1)

231 assert vhd_type == "Differencing" or vhd_type == "Dynamic"

232 return vhd_type == "Differencing"

233 assert False, f"Ill-formed {VHD_UTIL} output detected during VHD parent parsing"

234

235 @override

236 def setParent(self, path: str, parentPath: str, parentRaw: bool) -> None:

237 normpath = os.path.normpath(parentPath)

238 cmd = [VHD_UTIL, "modify", OPT_LOG_ERR, "-p", normpath, "-n", path]

239 if parentRaw:

240 cmd.append("-m")

241 self._ioretry(cmd)

242

243 @override

244 def getHidden(self, path: str) -> bool:

245 ret = cast(str, self._ioretry([VHD_UTIL, "query", OPT_LOG_ERR, "-f", "-n", path]))

246 return bool(int(ret.split(":")[-1].strip()))

247

248 @override

249 def setHidden(self, path: str, hidden: bool = True) -> None:

250 opt = "1"

251 if not hidden: 251 ↛ 252line 251 didn't jump to line 252, because the condition on line 251 was never true

252 opt = "0"

253 self._ioretry([VHD_UTIL, "set", OPT_LOG_ERR, "-n", path, "-f", "hidden", "-v", opt])

254

255 @override

256 def getSizeVirt(self, path: str) -> int:

257 ret = self._ioretry([VHD_UTIL, "query", OPT_LOG_ERR, "-v", "-n", path])

258 return int(ret) * 1024 * 1024

259

260 @override

261 def setSizeVirt(self, path: str, size: int, jFile: str) -> None:

262 """

263 Resize VHD offline

264 """

265 size_mb = size // (1024 * 1024)

266 self._ioretry([VHD_UTIL, "resize", OPT_LOG_ERR, "-s", str(size_mb), "-n", path, "-j", jFile])

267

268 @override

269 def setSizeVirtFast(self, path: str, size: int) -> None:

270 """

271 Resize VHD online.

272 """

273 size_mb = size // (1024 * 1024)

274 self._ioretry([VHD_UTIL, "resize", OPT_LOG_ERR, "-s", str(size_mb), "-n", path, "-f"])

275

276 @override

277 def getMaxResizeSize(self, path: str) -> int:

278 """

279 Get the max virtual size for fast resize.

280 """

281 ret = self._ioretry([VHD_UTIL, "query", OPT_LOG_ERR, "-S", "-n", path])

282 return int(ret) * 1024 * 1024

283

284 @override

285 def getSizePhys(self, path: str) -> int:

286 return int(self._ioretry([VHD_UTIL, "query", OPT_LOG_ERR, "-s", "-n", path]))

287

288 @override

289 def setSizePhys(self, path: str, size: int, debug: bool = True) -> None:

290 """

291 Set physical utilisation (applicable to VHD's on fixed-size files).

292 """

293 if debug:

294 cmd = [VHD_UTIL, "modify", OPT_LOG_ERR, "-s", str(size), "-n", path]

295 else:

296 cmd = [VHD_UTIL, "modify", "-s", str(size), "-n", path]

297 self._ioretry(cmd)

298

299 @override

300 def getAllocatedSize(self, path: str) -> int:

301 ret = self._ioretry([VHD_UTIL, "query", OPT_LOG_ERR, "-a", "-n", path])

302 return self._convertAllocatedSizeToBytes(int(ret))

303

304 @override

305 def getResizeJournalSize(self) -> int:

306 return MAX_VHD_JOURNAL_SIZE

307

308 @override

309 def killData(self, path: str) -> None:

310 """

311 Zero out the disk (kill all data inside the VHD file).

312 """

313 self._ioretry([VHD_UTIL, "modify", OPT_LOG_ERR, "-z", "-n", path])

314

315 @override

316 def getDepth(self, path: str) -> int:

317 """

318 Get the VHD parent chain depth.

319 """

320 text = cast(str, self._ioretry([VHD_UTIL, "query", OPT_LOG_ERR, "-d", "-n", path]))

321 depth = -1

322 if text.startswith("chain depth:"):

323 depth = int(text.split(":")[1].strip())

324 return depth

325

326 @override

327 def getBlockBitmap(self, path: str) -> bytes:

328 text = cast(bytes, self._ioretry([VHD_UTIL, "read", OPT_LOG_ERR, "-B", "-n", path], text=False))

329 return zlib.compress(text)

330

331 @override

332 def coalesce(self, path: str) -> int:

333 """

334 Coalesce the VHD, on success it returns the number of bytes coalesced.

335 """

336 text = cast(str, self._ioretry([VHD_UTIL, "coalesce", OPT_LOG_ERR, "-n", path]))

337 match = re.match(r"^Coalesced (\d+) sectors", text)

338 if match:

339 return int(match.group(1)) * VHD_SECTOR_SIZE

340 return 0

341

342 @override

343 def create(self, path: str, size: int, static: bool, msize: int = 0, block_size: Optional[int] = None) -> None:

344 cmd = [VHD_UTIL, "create", OPT_LOG_ERR, "-n", path, "-s", str(size // (1024 * 1024))]

345 if static:

346 cmd.append("-r")

347 if msize:

348 cmd.append("-S")

349 cmd.append(str(max(msize, size) // (1024 * 1024)))

350 self._ioretry(cmd)

351

352 @override

353 def snapshot(

354 self,

355 path: str,

356 parent: str,

357 parentRaw: bool,

358 msize: int = 0,

359 checkEmpty: bool = True,

360 is_mirror_image: bool = False

361 ) -> None:

362 cmd = [VHD_UTIL, "snapshot", OPT_LOG_ERR, "-n", path, "-p", parent]

363 if parentRaw:

364 cmd.append("-m")

365 if msize:

366 cmd.append("-S")

367 cmd.append(str(msize // (1024 * 1024)))

368 if not checkEmpty:

369 cmd.append("-e")

370 self._ioretry(cmd)

371

372 @override

373 def canSnapshotRaw(self, size: int) -> bool:

374 return size <= MAX_VHD_SIZE

375

376 @override

377 def check(

378 self,

379 path: str,

380 ignoreMissingFooter: bool = False,

381 fast: bool = False

382 ) -> CowUtil.CheckResult:

383 cmd = [VHD_UTIL, "check", OPT_LOG_ERR, "-n", path]

384 if ignoreMissingFooter:

385 cmd.append("-i")

386 if fast:

387 cmd.append("-B")

388 try:

389 self._ioretry(cmd)

390 return CowUtil.CheckResult.Success

391 except util.CommandException as e:

392 if e.code in (errno.ENOENT, errno.EROFS, errno.EMEDIUMTYPE):

393 return CowUtil.CheckResult.Unavailable

394 return CowUtil.CheckResult.Fail

395

396 @override

397 def revert(self, path: str, jFile: str) -> None:

398 self._ioretry([VHD_UTIL, "revert", OPT_LOG_ERR, "-n", path, "-j", jFile])

399

400 @override

401 def repair(self, path: str) -> None:

402 """

403 Repairs a VHD.

404 """

405 self._ioretry([VHD_UTIL, "repair", "-n", path])

406

407 @override

408 def validateAndRoundImageSize(self, size: int) -> int:

409 """

410 Take the supplied vhd size, in bytes, and check it is positive and less

411 that the maximum supported size, rounding up to the next block boundary.

412 """

413 if size < 0 or size > MAX_VHD_SIZE:

414 raise xs_errors.XenError(

415 "VDISize",

416 opterr="VDI size must be between 1 MB and %d MB" % (MAX_VHD_SIZE // (1024 * 1024))

417 )

418

419 if size < MIN_VHD_SIZE: 419 ↛ 420line 419 didn't jump to line 420, because the condition on line 419 was never true

420 size = MIN_VHD_SIZE

421

422 return util.roundup(VHD_BLOCK_SIZE, size)

423

424 @override

425 def getKeyHash(self, path: str) -> Optional[str]:

426 """

427 Extract the hash of the encryption key from the header of an encrypted VHD.

428 """

429 ret = cast(str, self._ioretry([VHD_UTIL, "key", "-p", "-n", path])).strip()

430 if ret == "none":

431 return None

432 vals = ret.split()

433 if len(vals) != 2:

434 util.SMlog("***** malformed output from vhd-util for VHD {}: \"{}\"".format(path, ret))

435 return None

436 [_nonce, key_hash] = vals

437 return key_hash

438

439 @override

440 def setKey(self, path: str, key_hash: str) -> None:

441 """

442 Set the encryption key for a VHD.

443 """

444 self._ioretry([VHD_UTIL, "key", "-s", "-n", path, "-H", key_hash])

445

446 @override

447 def isCoalesceableOnRemote(self) -> bool:

448 return False

449

450 @override

451 def coalesceOnline(self, path: str) -> int:

452 raise NotImplementedError("Online coalesce not implemented for vhdutil")

453

454 @override

455 def cancelCoalesceOnline(self, path: str) -> None:

456 raise NotImplementedError("Online coalesce not implemented for vhdutil")

457

458 @staticmethod

459 def _convertAllocatedSizeToBytes(size: int):

460 # Assume we have standard 2MB allocation blocks

461 return size * 2 * 1024 * 1024

462

463 @staticmethod

464 def _parseVHDInfo(line: str, extractUuidFunction: Callable[[str], str]) -> Optional[CowImageInfo]:

465 vhdInfo = None

466 valueMap = line.split()

467

468 try:

469 (key, val) = valueMap[0].split("=")

470 except:

471 return None

472

473 if key != "vhd":

474 return None

475

476 uuid = extractUuidFunction(val)

477 if not uuid:

478 util.SMlog("***** malformed output, no UUID: %s" % valueMap)

479 return None

480 vhdInfo = CowImageInfo(uuid)

481 vhdInfo.path = val

482

483 for keyval in valueMap:

484 (key, val) = keyval.split("=")

485 if key == "scan-error":

486 vhdInfo.error = line

487 util.SMlog("***** VHD scan error: %s" % line)

488 break

489 elif key == "capacity":

490 vhdInfo.sizeVirt = int(val)

491 elif key == "size":

492 vhdInfo.sizePhys = int(val)

493 elif key == "hidden":

494 vhdInfo.hidden = bool(int(val))

495 elif key == "parent" and val != "none":

496 vhdInfo.parentPath = val

497 vhdInfo.parentUuid = extractUuidFunction(val)

498 return vhdInfo

Coverage for drivers/vhdutil.py : 51%

316 statements

Coverage for drivers/vhdutil.py : 51%

316 statements 182 run 134 missing 0 excluded 6 partial

316 statements