1# Copyright (C) Citrix Systems Inc. 

2# 

3# This program is free software; you can redistribute it and/or modify 

4# it under the terms of the GNU Lesser General Public License as published 

5# by the Free Software Foundation; version 2.1 only. 

6# 

7# This program is distributed in the hope that it will be useful, 

8# but WITHOUT ANY WARRANTY; without even the implied warranty of 

9# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

10# GNU Lesser General Public License for more details. 

11# 

12# You should have received a copy of the GNU Lesser General Public License 

13# along with this program; if not, write to the Free Software Foundation, Inc., 

14# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 

15# 

16# Helper functions pertaining to VHD operations 

17# 

18 

19from sm_typing import Callable, Dict, Final, Optional, Sequence, cast, override 

20 

21from abc import abstractmethod 

22 

23import errno 

24import os 

25import re 

26import zlib 

27 

28import util 

29import XenAPI # pylint: disable=import-error 

30import xs_errors 

31 

32from cowutil import CowImageInfo, CowUtil, ImageFormat 

33 

34# ------------------------------------------------------------------------------ 

35 

36MIN_VHD_SIZE: Final = 2 * 1024 * 1024 

37MAX_VHD_SIZE: Final = 2040 * 1024 * 1024 * 1024 

38VHD_MAX_VOLUME_SIZE: Final = 2 * 1024 * 1024 * 1024 * 1024 

39 

40MAX_VHD_JOURNAL_SIZE: Final = 6 * 1024 * 1024 # 2MB VHD block size, max 2TB VHD size. 

41 

42VHD_BLOCK_SIZE: Final = 2 * 1024 * 1024 

43 

44VHD_FOOTER_SIZE: Final = 512 

45 

46VHD_SECTOR_SIZE: Final = 512 

47 

48MAX_VHD_CHAIN_LENGTH: Final = 30 

49 

50VHD_UTIL: Final = "/usr/bin/vhd-util" 

51 

52OPT_LOG_ERR: Final = "--debug" 

53 

54# ------------------------------------------------------------------------------ 

55 

56class VhdUtil(CowUtil): 

57 @override 

58 def getMinImageSize(self) -> int: 

59 return MIN_VHD_SIZE 

60 

61 @override 

62 def getMaxImageSize(self) -> int: 

63 return MAX_VHD_SIZE 

64 

65 @override 

66 def getBlockSize(self, path: str) -> int: 

67 return VHD_BLOCK_SIZE 

68 

69 @override 

70 def getFooterSize(self) -> int: 

71 return VHD_FOOTER_SIZE 

72 

73 @override 

74 def getDefaultPreallocationSizeVirt(self) -> int: 

75 return VHD_MAX_VOLUME_SIZE 

76 

77 @override 

78 def getMaxChainLength(self) -> int: 

79 return MAX_VHD_CHAIN_LENGTH 

80 

81 @override 

82 def calcOverheadEmpty(self, virtual_size: int) -> int: 

83 """ 

84 Calculate the VHD space overhead (metadata size) for an empty VDI of 

85 size virtual_size. 

86 """ 

87 overhead = 0 

88 size_mb = virtual_size // (1024 * 1024) 

89 

90 # Footer + footer copy + header + possible CoW parent locator fields 

91 overhead = 3 * 1024 

92 

93 # BAT 4 Bytes per block segment 

94 overhead += (size_mb // 2) * 4 

95 overhead = util.roundup(512, overhead) 

96 

97 # BATMAP 1 bit per block segment 

98 overhead += (size_mb // 2) // 8 

99 overhead = util.roundup(4096, overhead) 

100 

101 return overhead 

102 

103 @override 

104 def calcOverheadBitmap(self, virtual_size: int) -> int: 

105 num_blocks = virtual_size // VHD_BLOCK_SIZE 

106 if virtual_size % VHD_BLOCK_SIZE: 

107 num_blocks += 1 

108 return num_blocks * 4096 

109 

110 @override 

111 def getInfo( 

112 self, 

113 path: str, 

114 extractUuidFunction: Callable[[str], str], 

115 includeParent: bool = True, 

116 resolveParent: bool = True, 

117 useBackupFooter: bool = False 

118 ) -> CowImageInfo: 

119 """ 

120 Get the VHD info. The parent info may optionally be omitted: vhd-util 

121 tries to verify the parent by opening it, which results in error if the VHD 

122 resides on an inactive LV. 

123 """ 

124 opts = "-vsaf" 

125 if includeParent: 125 ↛ 129line 125 didn't jump to line 129, because the condition on line 125 was never false

126 opts += "p" 

127 if not resolveParent: 127 ↛ 128line 127 didn't jump to line 128, because the condition on line 127 was never true

128 opts += "u" 

129 if useBackupFooter: 

130 opts += "b" 

131 

132 ret = cast(str, self._ioretry([VHD_UTIL, "query", OPT_LOG_ERR, opts, "-n", path])) 

133 fields = ret.strip().split("\n") 

134 uuid = extractUuidFunction(path) 

135 vhdInfo = CowImageInfo(uuid) 

136 vhdInfo.sizeVirt = int(fields[0]) * 1024 * 1024 

137 vhdInfo.sizePhys = int(fields[1]) 

138 nextIndex = 2 

139 if includeParent: 139 ↛ 144line 139 didn't jump to line 144, because the condition on line 139 was never false

140 if fields[nextIndex].find("no parent") == -1: 140 ↛ 141line 140 didn't jump to line 141, because the condition on line 140 was never true

141 vhdInfo.parentPath = fields[nextIndex] 

142 vhdInfo.parentUuid = extractUuidFunction(fields[nextIndex]) 

143 nextIndex += 1 

144 vhdInfo.hidden = bool(int(fields[nextIndex].replace("hidden: ", ""))) 

145 vhdInfo.sizeAllocated = self._convertAllocatedSizeToBytes(int(fields[nextIndex+1])) 

146 vhdInfo.path = path 

147 return vhdInfo 

148 

149 @override 

150 def getInfoFromLVM( 

151 self, lvName: str, extractUuidFunction: Callable[[str], str], vgName: str 

152 ) -> Optional[CowImageInfo]: 

153 """ 

154 Get the VHD info. This function does not require the container LV to be 

155 active, but uses LVs & VGs. 

156 """ 

157 ret = cast(str, self._ioretry([VHD_UTIL, "scan", "-f", "-l", vgName, "-m", lvName])) 

158 return self._parseVHDInfo(ret, extractUuidFunction) 

159 

160 @override 

161 def getAllInfoFromVG( 

162 self, 

163 pattern: str, 

164 extractUuidFunction: Callable[[str], str], 

165 vgName: Optional[str] = None, 

166 parents: bool = False, 

167 exitOnError: bool = False 

168 ) -> Dict[str, CowImageInfo]: 

169 result: Dict[str, CowImageInfo] = dict() 

170 cmd = [VHD_UTIL, "scan", "-f", "-m", pattern] 

171 if vgName: 

172 cmd.append("-l") 

173 cmd.append(vgName) 

174 if parents: 

175 cmd.append("-a") 

176 try: 

177 ret = cast(str, self._ioretry(cmd)) 

178 except Exception as e: 

179 util.SMlog("WARN: VHD scan failed: output: %s" % e) 

180 ret = cast(str, self._ioretry(cmd + ["-c"])) 

181 util.SMlog("WARN: VHD scan with NOFAIL flag, output: %s" % ret) 

182 for line in ret.split('\n'): 

183 if not line.strip(): 

184 continue 

185 info = self._parseVHDInfo(line, extractUuidFunction) 

186 if info: 

187 if info.error != 0 and exitOnError: 

188 # Just return an empty dict() so the scan will be done 

189 # again by getParentChain. See CA-177063 for details on 

190 # how this has been discovered during the stress tests. 

191 return dict() 

192 result[info.uuid] = info 

193 else: 

194 util.SMlog("WARN: VHD info line doesn't parse correctly: %s" % line) 

195 return result 

196 

197 @override 

198 def getParent(self, path: str, extractUuidFunction: Callable[[str], str]) -> Optional[str]: 

199 ret = cast(str, self._ioretry([VHD_UTIL, "query", OPT_LOG_ERR, "-p", "-n", path])) 

200 if ret.find("query failed") != -1 or ret.find("Failed opening") != -1: 

201 raise util.SMException("VHD query returned %s" % ret) 

202 if ret.find("no parent") != -1: 

203 return None 

204 return extractUuidFunction(ret) 

205 

206 @override 

207 def getParentNoCheck(self, path: str) -> Optional[str]: 

208 text = util.pread([VHD_UTIL, "read", "-p", "-n", "%s" % path]) 

209 util.SMlog(text) 

210 for line in text.split("\n"): 

211 if line.find("decoded name :") != -1: 

212 val = line.split(":")[1].strip() 

213 vdi = val.replace("--", "-")[-40:] 

214 if vdi[1:].startswith("LV-"): 

215 vdi = vdi[1:] 

216 return vdi 

217 return None 

218 

219 @override 

220 def hasParent(self, path: str) -> bool: 

221 """ 

222 Check if the VHD has a parent. A VHD has a parent iff its type is 

223 'Differencing'. This function does not need the parent to actually 

224 be present (e.g. the parent LV to be activated). 

225 """ 

226 ret = cast(str, self._ioretry([VHD_UTIL, "read", OPT_LOG_ERR, "-p", "-n", path])) 

227 # pylint: disable=no-member 

228 m = re.match(r".*Disk type\s+: (\S+) hard disk.*", ret, flags=re.S) 

229 if m: 

230 vhd_type = m.group(1) 

231 assert vhd_type == "Differencing" or vhd_type == "Dynamic" 

232 return vhd_type == "Differencing" 

233 assert False, f"Ill-formed {VHD_UTIL} output detected during VHD parent parsing" 

234 

235 @override 

236 def setParent(self, path: str, parentPath: str, parentRaw: bool) -> None: 

237 normpath = os.path.normpath(parentPath) 

238 cmd = [VHD_UTIL, "modify", OPT_LOG_ERR, "-p", normpath, "-n", path] 

239 if parentRaw: 

240 cmd.append("-m") 

241 self._ioretry(cmd) 

242 

243 @override 

244 def getHidden(self, path: str) -> bool: 

245 ret = cast(str, self._ioretry([VHD_UTIL, "query", OPT_LOG_ERR, "-f", "-n", path])) 

246 return bool(int(ret.split(":")[-1].strip())) 

247 

248 @override 

249 def setHidden(self, path: str, hidden: bool = True) -> None: 

250 opt = "1" 

251 if not hidden: 251 ↛ 252line 251 didn't jump to line 252, because the condition on line 251 was never true

252 opt = "0" 

253 self._ioretry([VHD_UTIL, "set", OPT_LOG_ERR, "-n", path, "-f", "hidden", "-v", opt]) 

254 

255 @override 

256 def getSizeVirt(self, path: str) -> int: 

257 ret = self._ioretry([VHD_UTIL, "query", OPT_LOG_ERR, "-v", "-n", path]) 

258 return int(ret) * 1024 * 1024 

259 

260 @override 

261 def setSizeVirt(self, path: str, size: int, jFile: str) -> None: 

262 """ 

263 Resize VHD offline 

264 """ 

265 size_mb = size // (1024 * 1024) 

266 self._ioretry([VHD_UTIL, "resize", OPT_LOG_ERR, "-s", str(size_mb), "-n", path, "-j", jFile]) 

267 

268 @override 

269 def setSizeVirtFast(self, path: str, size: int) -> None: 

270 """ 

271 Resize VHD online. 

272 """ 

273 size_mb = size // (1024 * 1024) 

274 self._ioretry([VHD_UTIL, "resize", OPT_LOG_ERR, "-s", str(size_mb), "-n", path, "-f"]) 

275 

276 @override 

277 def getMaxResizeSize(self, path: str) -> int: 

278 """ 

279 Get the max virtual size for fast resize. 

280 """ 

281 ret = self._ioretry([VHD_UTIL, "query", OPT_LOG_ERR, "-S", "-n", path]) 

282 return int(ret) * 1024 * 1024 

283 

284 @override 

285 def getSizePhys(self, path: str) -> int: 

286 return int(self._ioretry([VHD_UTIL, "query", OPT_LOG_ERR, "-s", "-n", path])) 

287 

288 @override 

289 def setSizePhys(self, path: str, size: int, debug: bool = True) -> None: 

290 """ 

291 Set physical utilisation (applicable to VHD's on fixed-size files). 

292 """ 

293 if debug: 

294 cmd = [VHD_UTIL, "modify", OPT_LOG_ERR, "-s", str(size), "-n", path] 

295 else: 

296 cmd = [VHD_UTIL, "modify", "-s", str(size), "-n", path] 

297 self._ioretry(cmd) 

298 

299 @override 

300 def getAllocatedSize(self, path: str) -> int: 

301 ret = self._ioretry([VHD_UTIL, "query", OPT_LOG_ERR, "-a", "-n", path]) 

302 return self._convertAllocatedSizeToBytes(int(ret)) 

303 

304 @override 

305 def getResizeJournalSize(self) -> int: 

306 return MAX_VHD_JOURNAL_SIZE 

307 

308 @override 

309 def killData(self, path: str) -> None: 

310 """ 

311 Zero out the disk (kill all data inside the VHD file). 

312 """ 

313 self._ioretry([VHD_UTIL, "modify", OPT_LOG_ERR, "-z", "-n", path]) 

314 

315 @override 

316 def getDepth(self, path: str) -> int: 

317 """ 

318 Get the VHD parent chain depth. 

319 """ 

320 text = cast(str, self._ioretry([VHD_UTIL, "query", OPT_LOG_ERR, "-d", "-n", path])) 

321 depth = -1 

322 if text.startswith("chain depth:"): 

323 depth = int(text.split(":")[1].strip()) 

324 return depth 

325 

326 @override 

327 def getBlockBitmap(self, path: str) -> bytes: 

328 text = cast(bytes, self._ioretry([VHD_UTIL, "read", OPT_LOG_ERR, "-B", "-n", path], text=False)) 

329 return zlib.compress(text) 

330 

331 @override 

332 def coalesce(self, path: str) -> int: 

333 """ 

334 Coalesce the VHD, on success it returns the number of bytes coalesced. 

335 """ 

336 text = cast(str, self._ioretry([VHD_UTIL, "coalesce", OPT_LOG_ERR, "-n", path])) 

337 match = re.match(r"^Coalesced (\d+) sectors", text) 

338 if match: 

339 return int(match.group(1)) * VHD_SECTOR_SIZE 

340 return 0 

341 

342 @override 

343 def create(self, path: str, size: int, static: bool, msize: int = 0, block_size: Optional[int] = None) -> None: 

344 cmd = [VHD_UTIL, "create", OPT_LOG_ERR, "-n", path, "-s", str(size // (1024 * 1024))] 

345 if static: 

346 cmd.append("-r") 

347 if msize: 

348 cmd.append("-S") 

349 cmd.append(str(max(msize, size) // (1024 * 1024))) 

350 self._ioretry(cmd) 

351 

352 @override 

353 def snapshot( 

354 self, 

355 path: str, 

356 parent: str, 

357 parentRaw: bool, 

358 msize: int = 0, 

359 checkEmpty: bool = True 

360 ) -> None: 

361 cmd = [VHD_UTIL, "snapshot", OPT_LOG_ERR, "-n", path, "-p", parent] 

362 if parentRaw: 

363 cmd.append("-m") 

364 if msize: 

365 cmd.append("-S") 

366 cmd.append(str(msize // (1024 * 1024))) 

367 if not checkEmpty: 

368 cmd.append("-e") 

369 self._ioretry(cmd) 

370 

371 @override 

372 def canSnapshotRaw(self, size: int) -> bool: 

373 return size <= MAX_VHD_SIZE 

374 

375 @override 

376 def check( 

377 self, 

378 path: str, 

379 ignoreMissingFooter: bool = False, 

380 fast: bool = False 

381 ) -> CowUtil.CheckResult: 

382 cmd = [VHD_UTIL, "check", OPT_LOG_ERR, "-n", path] 

383 if ignoreMissingFooter: 

384 cmd.append("-i") 

385 if fast: 

386 cmd.append("-B") 

387 try: 

388 self._ioretry(cmd) 

389 return CowUtil.CheckResult.Success 

390 except util.CommandException as e: 

391 if e.code in (errno.ENOENT, errno.EROFS, errno.EMEDIUMTYPE): 

392 return CowUtil.CheckResult.Unavailable 

393 return CowUtil.CheckResult.Fail 

394 

395 @override 

396 def revert(self, path: str, jFile: str) -> None: 

397 self._ioretry([VHD_UTIL, "revert", OPT_LOG_ERR, "-n", path, "-j", jFile]) 

398 

399 @override 

400 def repair(self, path: str) -> None: 

401 """ 

402 Repairs a VHD. 

403 """ 

404 self._ioretry([VHD_UTIL, "repair", "-n", path]) 

405 

406 @override 

407 def validateAndRoundImageSize(self, size: int) -> int: 

408 """ 

409 Take the supplied vhd size, in bytes, and check it is positive and less 

410 that the maximum supported size, rounding up to the next block boundary. 

411 """ 

412 if size < 0 or size > MAX_VHD_SIZE: 

413 raise xs_errors.XenError( 

414 "VDISize", 

415 opterr="VDI size must be between 1 MB and %d MB" % (MAX_VHD_SIZE // (1024 * 1024)) 

416 ) 

417 

418 if size < MIN_VHD_SIZE: 418 ↛ 419line 418 didn't jump to line 419, because the condition on line 418 was never true

419 size = MIN_VHD_SIZE 

420 

421 return util.roundup(VHD_BLOCK_SIZE, size) 

422 

423 @override 

424 def getKeyHash(self, path: str) -> Optional[str]: 

425 """ 

426 Extract the hash of the encryption key from the header of an encrypted VHD. 

427 """ 

428 ret = cast(str, self._ioretry([VHD_UTIL, "key", "-p", "-n", path])).strip() 

429 if ret == "none": 

430 return None 

431 vals = ret.split() 

432 if len(vals) != 2: 

433 util.SMlog("***** malformed output from vhd-util for VHD {}: \"{}\"".format(path, ret)) 

434 return None 

435 [_nonce, key_hash] = vals 

436 return key_hash 

437 

438 @override 

439 def setKey(self, path: str, key_hash: str) -> None: 

440 """ 

441 Set the encryption key for a VHD. 

442 """ 

443 self._ioretry([VHD_UTIL, "key", "-s", "-n", path, "-H", key_hash]) 

444 

445 @override 

446 def isCoalesceableOnRemote(self) -> bool: 

447 return False 

448 

449 @override 

450 def coalesceOnline(self, path: str) -> int: 

451 raise NotImplementedError("Online coalesce not implemented for vhdutil") 

452 

453 @override 

454 def cancelCoalesceOnline(self, path: str) -> None: 

455 raise NotImplementedError("Online coalesce not implemented for vhdutil") 

456 

457 @staticmethod 

458 def _convertAllocatedSizeToBytes(size: int): 

459 # Assume we have standard 2MB allocation blocks 

460 return size * 2 * 1024 * 1024 

461 

462 @staticmethod 

463 def _parseVHDInfo(line: str, extractUuidFunction: Callable[[str], str]) -> Optional[CowImageInfo]: 

464 vhdInfo = None 

465 valueMap = line.split() 

466 

467 try: 

468 (key, val) = valueMap[0].split("=") 

469 except: 

470 return None 

471 

472 if key != "vhd": 

473 return None 

474 

475 uuid = extractUuidFunction(val) 

476 if not uuid: 

477 util.SMlog("***** malformed output, no UUID: %s" % valueMap) 

478 return None 

479 vhdInfo = CowImageInfo(uuid) 

480 vhdInfo.path = val 

481 

482 for keyval in valueMap: 

483 (key, val) = keyval.split("=") 

484 if key == "scan-error": 

485 vhdInfo.error = line 

486 util.SMlog("***** VHD scan error: %s" % line) 

487 break 

488 elif key == "capacity": 

489 vhdInfo.sizeVirt = int(val) 

490 elif key == "size": 

491 vhdInfo.sizePhys = int(val) 

492 elif key == "hidden": 

493 vhdInfo.hidden = bool(int(val)) 

494 elif key == "parent" and val != "none": 

495 vhdInfo.parentPath = val 

496 vhdInfo.parentUuid = extractUuidFunction(val) 

497 return vhdInfo