1#!/usr/bin/env python3 

2# 

3# Copyright (C) 2020 Vates SAS - ronan.abhamon@vates.fr 

4# 

5# This program is free software: you can redistribute it and/or modify 

6# it under the terms of the GNU General Public License as published by 

7# the Free Software Foundation, either version 3 of the License, or 

8# (at your option) any later version. 

9# This program is distributed in the hope that it will be useful, 

10# but WITHOUT ANY WARRANTY; without even the implied warranty of 

11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

12# GNU General Public License for more details. 

13# 

14# You should have received a copy of the GNU General Public License 

15# along with this program. If not, see <https://www.gnu.org/licenses/>. 

16 

17from sm_typing import override 

18 

19import base64 

20import errno 

21import json 

22import socket 

23import time 

24 

25from cowutil import CowImageInfo, CowUtil, getCowUtil 

26import util 

27import xs_errors 

28 

29from linstorjournaler import LinstorJournaler 

30from linstorvolumemanager import LinstorVolumeManager 

31from vditype import VdiType 

32 

33MANAGER_PLUGIN = 'linstor-manager' 

34 

35 

36def call_remote_method(session, host_ref, method, args): 

37 try: 

38 response = session.xenapi.host.call_plugin( 

39 host_ref, MANAGER_PLUGIN, method, args 

40 ) 

41 except Exception as e: 

42 util.SMlog('call-plugin ({} with {}) exception: {}'.format( 

43 method, args, e 

44 )) 

45 raise util.SMException(str(e)) 

46 

47 util.SMlog('call-plugin ({} with {}) returned: {}'.format( 

48 method, args, response 

49 )) 

50 

51 return response 

52 

53 

54class LinstorCallException(util.SMException): 

55 def __init__(self, cmd_err): 

56 self.cmd_err = cmd_err 

57 

58 @override 

59 def __str__(self) -> str: 

60 return str(self.cmd_err) 

61 

62 

63class ErofsLinstorCallException(LinstorCallException): 

64 pass 

65 

66 

67class NoPathLinstorCallException(LinstorCallException): 

68 pass 

69 

70 

71def linstorhostcall(local_method, remote_method): 

72 def decorated(response_parser): 

73 def wrapper(*args, **kwargs): 

74 self = args[0] 

75 vdi_uuid = args[1] 

76 

77 device_path = self._linstor.build_device_path( 

78 self._linstor.get_volume_name(vdi_uuid) 

79 ) 

80 

81 # A. Try a call using directly the DRBD device to avoid 

82 # remote request. 

83 

84 # Try to read locally if the device is not in use or if the device 

85 # is up to date and not diskless. 

86 (node_names, in_use_by) = \ 

87 self._linstor.find_up_to_date_diskful_nodes(vdi_uuid) 

88 

89 local_e = None 

90 try: 

91 if not in_use_by or socket.gethostname() in node_names: 

92 return self._call_local_method(local_method, device_path, *args[2:], **kwargs) 

93 except ErofsLinstorCallException as e: 

94 local_e = e.cmd_err 

95 except Exception as e: 

96 local_e = e 

97 

98 util.SMlog( 

99 'unable to execute `{}` locally, retry using a readable host... (cause: {})'.format( 

100 remote_method, local_e if local_e else 'local diskless + in use or not up to date' 

101 ) 

102 ) 

103 

104 if in_use_by: 

105 node_names = {in_use_by} 

106 

107 # B. Execute the plugin on master or slave. 

108 remote_args = { 

109 'devicePath': device_path, 

110 'groupName': self._linstor.group_name, 

111 'vdiType': self._vdi_type 

112 } 

113 remote_args.update(**kwargs) 

114 remote_args = {str(key): str(value) for key, value in remote_args.items()} 

115 

116 try: 

117 def remote_call(): 

118 host_ref = self._get_readonly_host(vdi_uuid, device_path, node_names) 

119 return call_remote_method(self._session, host_ref, remote_method, remote_args) 

120 response = util.retry(remote_call, 5, 2) 

121 except Exception as remote_e: 

122 self._raise_openers_exception(device_path, local_e or remote_e) 

123 

124 return response_parser(self, vdi_uuid, response) 

125 return wrapper 

126 return decorated 

127 

128 

129def linstormodifier(): 

130 def decorated(func): 

131 def wrapper(*args, **kwargs): 

132 self = args[0] 

133 

134 ret = func(*args, **kwargs) 

135 self._linstor.invalidate_resource_cache() 

136 return ret 

137 return wrapper 

138 return decorated 

139 

140 

141class LinstorCowUtil(object): 

142 def __init__(self, session, linstor, vdi_type: str): 

143 self._session = session 

144 self._linstor = linstor 

145 self._cowutil = getCowUtil(vdi_type) 

146 self._vdi_type = vdi_type 

147 

148 @property 

149 def cowutil(self) -> CowUtil: 

150 return self._cowutil 

151 

152 def create_chain_paths(self, vdi_uuid, readonly=False): 

153 # OPTIMIZE: Add a limit_to_first_allocated_block param to limit cowutil calls. 

154 # Useful for the snapshot code algorithm. 

155 

156 leaf_vdi_path = self._linstor.get_device_path(vdi_uuid) 

157 path = leaf_vdi_path 

158 while True: 

159 if not util.pathexists(path): 

160 raise xs_errors.XenError( 

161 'VDIUnavailable', opterr='Could not find: {}'.format(path) 

162 ) 

163 

164 # Diskless path can be created on the fly, ensure we can open it. 

165 def check_volume_usable(): 

166 while True: 

167 try: 

168 with open(path, 'r' if readonly else 'r+'): 

169 pass 

170 except IOError as e: 

171 if e.errno == errno.ENODATA: 

172 time.sleep(2) 

173 continue 

174 if e.errno == errno.EROFS: 

175 util.SMlog('Volume not attachable because RO. Openers: {}'.format( 

176 self._linstor.get_volume_openers(vdi_uuid) 

177 )) 

178 raise 

179 break 

180 util.retry(check_volume_usable, 15, 2) 

181 

182 vdi_uuid = self.get_info(vdi_uuid).parentUuid 

183 if not vdi_uuid: 

184 break 

185 path = self._linstor.get_device_path(vdi_uuid) 

186 readonly = True # Non-leaf is always readonly. 

187 

188 return leaf_vdi_path 

189 

190 # -------------------------------------------------------------------------- 

191 # Getters: read locally and try on another host in case of failure. 

192 # -------------------------------------------------------------------------- 

193 

194 def check(self, vdi_uuid, ignore_missing_footer=False, fast=False): 

195 kwargs = { 

196 'ignoreMissingFooter': ignore_missing_footer, 

197 'fast': fast 

198 } 

199 return self._check(vdi_uuid, **kwargs) 

200 

201 @linstorhostcall(CowUtil.check, 'check') 

202 def _check(self, vdi_uuid, response): 

203 return CowUtil.CheckResult(response) 

204 

205 def get_info(self, vdi_uuid, include_parent=True): 

206 kwargs = { 

207 'includeParent': include_parent, 

208 'resolveParent': False 

209 } 

210 return self._get_info(vdi_uuid, self._extract_uuid, **kwargs) 

211 

212 @linstorhostcall(CowUtil.getInfo, 'getInfo') 

213 def _get_info(self, vdi_uuid, response): 

214 obj = json.loads(response) 

215 

216 image_info = CowImageInfo(vdi_uuid) 

217 image_info.sizeVirt = obj['sizeVirt'] 

218 image_info.sizePhys = obj['sizePhys'] 

219 if 'parentPath' in obj: 

220 image_info.parentPath = obj['parentPath'] 

221 image_info.parentUuid = obj['parentUuid'] 

222 image_info.hidden = obj['hidden'] 

223 image_info.path = obj['path'] 

224 

225 return image_info 

226 

227 @linstorhostcall(CowUtil.hasParent, 'hasParent') 

228 def has_parent(self, vdi_uuid, response): 

229 return util.strtobool(response) 

230 

231 def get_parent(self, vdi_uuid): 

232 return self._get_parent(vdi_uuid, self._extract_uuid) 

233 

234 @linstorhostcall(CowUtil.getParent, 'getParent') 

235 def _get_parent(self, vdi_uuid, response): 

236 return response 

237 

238 @linstorhostcall(CowUtil.getSizeVirt, 'getSizeVirt') 

239 def get_size_virt(self, vdi_uuid, response): 

240 return int(response) 

241 

242 @linstorhostcall(CowUtil.getMaxResizeSize, 'getMaxResizeSize') 

243 def get_max_resize_size(self, vdi_uuid, response): 

244 return int(response) 

245 

246 @linstorhostcall(CowUtil.getSizePhys, 'getSizePhys') 

247 def get_size_phys(self, vdi_uuid, response): 

248 return int(response) 

249 

250 @linstorhostcall(CowUtil.getAllocatedSize, 'getAllocatedSize') 

251 def get_allocated_size(self, vdi_uuid, response): 

252 return int(response) 

253 

254 @linstorhostcall(CowUtil.getDepth, 'getDepth') 

255 def get_depth(self, vdi_uuid, response): 

256 return int(response) 

257 

258 @linstorhostcall(CowUtil.getKeyHash, 'getKeyHash') 

259 def get_key_hash(self, vdi_uuid, response): 

260 return response or None 

261 

262 @linstorhostcall(CowUtil.getBlockBitmap, 'getBlockBitmap') 

263 def get_block_bitmap(self, vdi_uuid, response): 

264 return base64.b64decode(response) 

265 

266 @linstorhostcall('_get_drbd_size', 'getDrbdSize') 

267 def get_drbd_size(self, vdi_uuid, response): 

268 return int(response) 

269 

270 def _get_drbd_size(self, cowutil_inst, path): 

271 (ret, stdout, stderr) = util.doexec(['blockdev', '--getsize64', path]) 

272 if ret == 0: 

273 return int(stdout.strip()) 

274 raise util.SMException('Failed to get DRBD size: {}'.format(stderr)) 

275 

276 # -------------------------------------------------------------------------- 

277 # Setters: only used locally. 

278 # -------------------------------------------------------------------------- 

279 

280 @linstormodifier() 

281 def create(self, path, size, static, msize=0): 

282 return self._call_local_method_or_fail(CowUtil.create, path, size, static, msize) 

283 

284 @linstormodifier() 

285 def set_size_phys(self, path, size, debug=True): 

286 return self._call_local_method_or_fail(CowUtil.setSizePhys, path, size, debug) 

287 

288 @linstormodifier() 

289 def set_parent(self, path, parentPath, parentRaw=False): 

290 return self._call_local_method_or_fail(CowUtil.setParent, path, parentPath, parentRaw) 

291 

292 @linstormodifier() 

293 def set_hidden(self, path, hidden=True): 

294 return self._call_local_method_or_fail(CowUtil.setHidden, path, hidden) 

295 

296 @linstormodifier() 

297 def set_key(self, path, key_hash): 

298 return self._call_local_method_or_fail(CowUtil.setKey, path, key_hash) 

299 

300 @linstormodifier() 

301 def kill_data(self, path): 

302 return self._call_local_method_or_fail(CowUtil.killData, path) 

303 

304 @linstormodifier() 

305 def snapshot(self, path, parent, parentRaw, msize=0, checkEmpty=True): 

306 return self._call_local_method_or_fail(CowUtil.snapshot, path, parent, parentRaw, msize, checkEmpty) 

307 

308 def inflate(self, journaler, vdi_uuid, vdi_path, new_size, old_size): 

309 # Only inflate if the LINSTOR volume capacity is not enough. 

310 new_size = LinstorVolumeManager.round_up_volume_size(new_size) 

311 if new_size <= old_size: 

312 return 

313 

314 util.SMlog( 

315 'Inflate {} (size={}, previous={})' 

316 .format(vdi_path, new_size, old_size) 

317 ) 

318 

319 journaler.create( 

320 LinstorJournaler.INFLATE, vdi_uuid, old_size 

321 ) 

322 self._linstor.resize_volume(vdi_uuid, new_size) 

323 

324 result_size = self.get_drbd_size(vdi_uuid) 

325 if result_size < new_size: 

326 util.SMlog( 

327 'WARNING: Cannot inflate volume to {}B, result size: {}B' 

328 .format(new_size, result_size) 

329 ) 

330 

331 self._zeroize(vdi_path, result_size - self._cowutil.getFooterSize()) 

332 self.set_size_phys(vdi_path, result_size, False) 

333 journaler.remove(LinstorJournaler.INFLATE, vdi_uuid) 

334 

335 def deflate(self, vdi_path, new_size, old_size, zeroize=False): 

336 if zeroize: 

337 assert old_size > self._cowutil.getFooterSize() 

338 self._zeroize(vdi_path, old_size - self._cowutil.getFooterSize()) 

339 

340 new_size = LinstorVolumeManager.round_up_volume_size(new_size) 

341 if new_size >= old_size: 

342 return 

343 

344 util.SMlog( 

345 'Deflate {} (new size={}, previous={})' 

346 .format(vdi_path, new_size, old_size) 

347 ) 

348 

349 self.set_size_phys(vdi_path, new_size) 

350 # TODO: Change the LINSTOR volume size using linstor.resize_volume. 

351 

352 # -------------------------------------------------------------------------- 

353 # Remote setters: write locally and try on another host in case of failure. 

354 # -------------------------------------------------------------------------- 

355 

356 @linstormodifier() 

357 def set_size_virt(self, path, size, jFile): 

358 kwargs = { 

359 'size': size, 

360 'jFile': jFile 

361 } 

362 return self._call_method(CowUtil.setSizeVirt, 'setSizeVirt', path, use_parent=False, **kwargs) 

363 

364 @linstormodifier() 

365 def set_size_virt_fast(self, path, size): 

366 kwargs = { 

367 'size': size 

368 } 

369 return self._call_method(CowUtil.setSizeVirtFast, 'setSizeVirtFast', path, use_parent=False, **kwargs) 

370 

371 @linstormodifier() 

372 def force_parent(self, path, parentPath, parentRaw=False): 

373 kwargs = { 

374 'parentPath': str(parentPath), 

375 'parentRaw': parentRaw 

376 } 

377 return self._call_method(CowUtil.setParent, 'setParent', path, use_parent=False, **kwargs) 

378 

379 @linstormodifier() 

380 def force_coalesce(self, path): 

381 return int(self._call_method(CowUtil.coalesce, 'coalesce', path, use_parent=True)) 

382 

383 @linstormodifier() 

384 def force_repair(self, path): 

385 return self._call_method(CowUtil.repair, 'repair', path, use_parent=False) 

386 

387 @linstormodifier() 

388 def force_deflate(self, path, newSize, oldSize, zeroize): 

389 kwargs = { 

390 'newSize': newSize, 

391 'oldSize': oldSize, 

392 'zeroize': zeroize 

393 } 

394 return self._call_method('_force_deflate', 'deflate', path, use_parent=False, **kwargs) 

395 

396 def _force_deflate(self, cowutil_inst, path, newSize, oldSize, zeroize): 

397 self.deflate(path, newSize, oldSize, zeroize) 

398 

399 # -------------------------------------------------------------------------- 

400 # Helpers. 

401 # -------------------------------------------------------------------------- 

402 

403 def compute_volume_size(self, virtual_size: int) -> int: 

404 if VdiType.isCowImage(self._vdi_type): 

405 # All LINSTOR VDIs have the metadata area preallocated for 

406 # the maximum possible virtual size (for fast online VDI.resize). 

407 meta_overhead = self._cowutil.calcOverheadEmpty( 

408 max(virtual_size, self._cowutil.getDefaultPreallocationSizeVirt()) 

409 ) 

410 bitmap_overhead = self._cowutil.calcOverheadBitmap(virtual_size) 

411 virtual_size += meta_overhead + bitmap_overhead 

412 else: 

413 raise Exception('Invalid image type: {}'.format(self._vdi_type)) 

414 

415 return LinstorVolumeManager.round_up_volume_size(virtual_size) 

416 

417 def _extract_uuid(self, device_path): 

418 # TODO: Remove new line in the vhdutil module. Not here. 

419 return self._linstor.get_volume_uuid_from_device_path( 

420 device_path.rstrip('\n') 

421 ) 

422 

423 def _get_readonly_host(self, vdi_uuid, device_path, node_names): 

424 """ 

425 When CowUtil is called to fetch VDI info we must find a 

426 diskful DRBD disk to read the data. It's the goal of this function. 

427 Why? Because when a COW image is open in RO mode, the LVM layer is used 

428 directly to bypass DRBD verifications (we can have only one process 

429 that reads/writes to disk with DRBD devices). 

430 """ 

431 

432 if not node_names: 

433 raise xs_errors.XenError( 

434 'VDIUnavailable', 

435 opterr='Unable to find diskful node: {} (path={})' 

436 .format(vdi_uuid, device_path) 

437 ) 

438 

439 hosts = self._session.xenapi.host.get_all_records() 

440 for host_ref, host_record in hosts.items(): 

441 if host_record['hostname'] in node_names: 

442 return host_ref 

443 

444 raise xs_errors.XenError( 

445 'VDIUnavailable', 

446 opterr='Unable to find a valid host from VDI: {} (path={})' 

447 .format(vdi_uuid, device_path) 

448 ) 

449 

450 # -------------------------------------------------------------------------- 

451 

452 def _raise_openers_exception(self, device_path, e): 

453 if isinstance(e, util.CommandException): 

454 e_str = 'cmd: `{}`, code: `{}`, reason: `{}`'.format(e.cmd, e.code, e.reason) 

455 else: 

456 e_str = str(e) 

457 

458 try: 

459 volume_uuid = self._linstor.get_volume_uuid_from_device_path( 

460 device_path 

461 ) 

462 e_wrapper = Exception( 

463 e_str + ' (openers: {})'.format( 

464 self._linstor.get_volume_openers(volume_uuid) 

465 ) 

466 ) 

467 except Exception as illformed_e: 

468 e_wrapper = Exception( 

469 e_str + ' (unable to get openers: {})'.format(illformed_e) 

470 ) 

471 util.SMlog('raise opener exception: {}'.format(e_wrapper)) 

472 raise e_wrapper # pylint: disable = E0702 

473 

474 def _call_local_method(self, local_method, device_path, *args, **kwargs): 

475 if isinstance(local_method, str): 

476 local_method = getattr(self, local_method) 

477 

478 try: 

479 def local_call(): 

480 try: 

481 return local_method(self._cowutil, device_path, *args, **kwargs) 

482 except util.CommandException as e: 

483 if e.code == errno.EROFS or e.code == errno.EMEDIUMTYPE: 

484 raise ErofsLinstorCallException(e) # Break retry calls. 

485 if e.code == errno.ENOENT: 

486 raise NoPathLinstorCallException(e) 

487 raise e 

488 # Retry only locally if it's not an EROFS exception. 

489 return util.retry(local_call, 5, 2, exceptions=[util.CommandException]) 

490 except util.CommandException as e: 

491 util.SMlog('failed to execute locally CowUtil (sys {})'.format(e.code)) 

492 raise e 

493 

494 def _call_local_method_or_fail(self, local_method, device_path, *args, **kwargs): 

495 try: 

496 return self._call_local_method(local_method, device_path, *args, **kwargs) 

497 except ErofsLinstorCallException as e: 

498 # Volume is locked on a host, find openers. 

499 self._raise_openers_exception(device_path, e.cmd_err) 

500 

501 def _call_method(self, local_method, remote_method, device_path, use_parent, *args, **kwargs): 

502 # Note: `use_parent` exists to know if the COW image parent is used by the local/remote method. 

503 # Normally in case of failure, if the parent is unused we try to execute the method on 

504 # another host using the DRBD opener list. In the other case, if the parent is required, 

505 # we must check where this last one is open instead of the child. 

506 

507 if isinstance(local_method, str): 

508 local_method = getattr(self, local_method) 

509 

510 # A. Try to write locally... 

511 try: 

512 return self._call_local_method(local_method, device_path, *args, **kwargs) 

513 except Exception: 

514 pass 

515 

516 util.SMlog('unable to execute `{}` locally, retry using a writable host...'.format(remote_method)) 

517 

518 # B. Execute the command on another host. 

519 # B.1. Get host list. 

520 try: 

521 hosts = self._session.xenapi.host.get_all_records() 

522 except Exception as e: 

523 raise xs_errors.XenError( 

524 'VDIUnavailable', 

525 opterr='Unable to get host list to run CowUtil command `{}` (path={}): {}' 

526 .format(remote_method, device_path, e) 

527 ) 

528 

529 # B.2. Prepare remote args. 

530 remote_args = { 

531 'devicePath': device_path, 

532 'groupName': self._linstor.group_name 

533 } 

534 remote_args.update(**kwargs) 

535 remote_args = {str(key): str(value) for key, value in remote_args.items()} 

536 

537 volume_uuid = self._linstor.get_volume_uuid_from_device_path( 

538 device_path 

539 ) 

540 parent_volume_uuid = None 

541 if use_parent: 

542 parent_volume_uuid = self.get_parent(volume_uuid) 

543 

544 openers_uuid = parent_volume_uuid if use_parent else volume_uuid 

545 

546 # B.3. Call! 

547 def remote_call(): 

548 try: 

549 all_openers = self._linstor.get_volume_openers(openers_uuid) 

550 except Exception as e: 

551 raise xs_errors.XenError( 

552 'VDIUnavailable', 

553 opterr='Unable to get DRBD openers to run CowUtil command `{}` (path={}): {}' 

554 .format(remote_method, device_path, e) 

555 ) 

556 

557 no_host_found = True 

558 for hostname, openers in all_openers.items(): 

559 if not openers: 

560 continue 

561 

562 try: 

563 host_ref = next(ref for ref, rec in hosts.items() if rec['hostname'] == hostname) 

564 except StopIteration: 

565 continue 

566 

567 no_host_found = False 

568 try: 

569 return call_remote_method(self._session, host_ref, remote_method, remote_args) 

570 except Exception: 

571 pass 

572 

573 if no_host_found: 

574 try: 

575 return local_method(self._cowutil, device_path, *args, **kwargs) 

576 except Exception as e: 

577 self._raise_openers_exception(device_path, e) 

578 

579 raise xs_errors.XenError( 

580 'VDIUnavailable', 

581 opterr='No valid host found to run CowUtil command `{}` (path=`{}`, openers=`{}`)' 

582 .format(remote_method, device_path, openers) 

583 ) 

584 return util.retry(remote_call, 5, 2) 

585 

586 def _zeroize(self, path, size): 

587 if not util.zeroOut(path, size, self._cowutil.getFooterSize()): 

588 raise xs_errors.XenError( 

589 'EIO', 

590 opterr='Failed to zero out COW image footer {}'.format(path) 

591 )