Coverage for watcher/decision_engine/sync.py: 90%

1# -*- encoding: utf-8 -*-

4# Licensed under the Apache License, Version 2.0 (the "License");

5# you may not use this file except in compliance with the License.

6# You may obtain a copy of the License at

8# http://www.apache.org/licenses/LICENSE-2.0

10# Unless required by applicable law or agreed to in writing, software

11# distributed under the License is distributed on an "AS IS" BASIS,

12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or

13# implied.

14# See the License for the specific language governing permissions and

15# limitations under the License.

17import ast

18import collections

20from oslo_log import log

22from watcher.common import context

23from watcher.decision_engine.loading import default

24from watcher.decision_engine.scoring import scoring_factory

25from watcher import objects

27LOG = log.getLogger(__name__)

29GoalMapping = collections.namedtuple(

30 'GoalMapping', ['name', 'display_name', 'efficacy_specification'])

31StrategyMapping = collections.namedtuple(

32 'StrategyMapping',

33 ['name', 'goal_name', 'display_name', 'parameters_spec'])

34ScoringEngineMapping = collections.namedtuple(

35 'ScoringEngineMapping',

36 ['name', 'description', 'metainfo'])

38IndicatorSpec = collections.namedtuple(

39 'IndicatorSpec', ['name', 'description', 'unit', 'schema'])

42class Syncer(object):

43 """Syncs all available goals and strategies with the Watcher DB"""

45 def __init__(self):

46 self.ctx = context.make_context()

47 self.discovered_map = None

49 self._available_goals = None

50 self._available_goals_map = None

52 self._available_strategies = None

53 self._available_strategies_map = None

55 self._available_scoringengines = None

56 self._available_scoringengines_map = None

58 # This goal mapping maps stale goal IDs to the synced goal

59 self.goal_mapping = dict()

60 # This strategy mapping maps stale strategy IDs to the synced goal

61 self.strategy_mapping = dict()

62 # Maps stale scoring engine IDs to the synced scoring engines

63 self.se_mapping = dict()

65 self.stale_audit_templates_map = {}

66 self.stale_audits_map = {}

67 self.stale_action_plans_map = {}

69 @property

70 def available_goals(self):

71 """Goals loaded from DB"""

72 if self._available_goals is None:

73 self._available_goals = objects.Goal.list(self.ctx)

74 return self._available_goals

76 @property

77 def available_strategies(self):

78 """Strategies loaded from DB"""

79 if self._available_strategies is None:

80 self._available_strategies = objects.Strategy.list(self.ctx)

81 goal_ids = [g.id for g in self.available_goals]

82 stale_strategies = [s for s in self._available_strategies

83 if s.goal_id not in goal_ids]

84 for s in stale_strategies:

85 LOG.info("Can't find Goal id %d of strategy %s",

86 s.goal_id, s.name)

87 s.soft_delete()

88 self._available_strategies.remove(s)

89 return self._available_strategies

91 @property

92 def available_scoringengines(self):

93 """Scoring Engines loaded from DB"""

94 if self._available_scoringengines is None:

95 self._available_scoringengines = (objects.ScoringEngine

96 .list(self.ctx))

97 return self._available_scoringengines

99 @property

100 def available_goals_map(self):

101 """Mapping of goals loaded from DB"""

102 if self._available_goals_map is None:

103 self._available_goals_map = {

104 GoalMapping(

105 name=g.name,

106 display_name=g.display_name,

107 efficacy_specification=tuple(

108 IndicatorSpec(**item)

109 for item in g.efficacy_specification)): g

110 for g in self.available_goals

111 }

112 return self._available_goals_map

113

114 @property

115 def available_strategies_map(self):

116 if self._available_strategies_map is None:

117 goals_map = {g.id: g.name for g in self.available_goals}

118 self._available_strategies_map = {

119 StrategyMapping(

120 name=s.name, goal_name=goals_map[s.goal_id],

121 display_name=s.display_name,

122 parameters_spec=str(s.parameters_spec)): s

123 for s in self.available_strategies

124 }

125 return self._available_strategies_map

126

127 @property

128 def available_scoringengines_map(self):

129 if self._available_scoringengines_map is None:

130 self._available_scoringengines_map = {

131 ScoringEngineMapping(

132 name=s.id, description=s.description,

133 metainfo=s.metainfo): s

134 for s in self.available_scoringengines

135 }

136 return self._available_scoringengines_map

137

138 def sync(self):

139 self.discovered_map = self._discover()

140 goals_map = self.discovered_map["goals"]

141 strategies_map = self.discovered_map["strategies"]

142 scoringengines_map = self.discovered_map["scoringengines"]

143

144 for goal_name, goal_map in goals_map.items():

145 if goal_map in self.available_goals_map:

146 LOG.info("Goal %s already exists", goal_name)

147 continue

148

149 self.goal_mapping.update(self._sync_goal(goal_map))

150

151 for strategy_name, strategy_map in strategies_map.items():

152 if (strategy_map in self.available_strategies_map and

153 strategy_map.goal_name not in

154 [g.name for g in self.goal_mapping.values()]):

155 LOG.info("Strategy %s already exists", strategy_name)

156 continue

157

158 self.strategy_mapping.update(self._sync_strategy(strategy_map))

159

160 for se_name, se_map in scoringengines_map.items():

161 if se_map in self.available_scoringengines_map: 161 ↛ 162line 161 didn't jump to line 162 because the condition on line 161 was never true

162 LOG.info("Scoring Engine %s already exists",

163 se_name)

164 continue

165

166 self.se_mapping.update(self._sync_scoringengine(se_map))

167

168 self._sync_objects()

169 self._soft_delete_removed_scoringengines()

170

171 def _sync_goal(self, goal_map):

172 goal_name = goal_map.name

173 goal_mapping = dict()

174 # Goals that are matching by name with the given discovered goal name

175 matching_goals = [g for g in self.available_goals

176 if g.name == goal_name]

177 stale_goals = self._soft_delete_stale_goals(goal_map, matching_goals)

178

179 if stale_goals or not matching_goals: 179 ↛ 195line 179 didn't jump to line 195 because the condition on line 179 was always true

180 goal = objects.Goal(self.ctx)

181 goal.name = goal_name

182 goal.display_name = goal_map.display_name

183 goal.efficacy_specification = [

184 indicator._asdict()

185 for indicator in goal_map.efficacy_specification]

186 goal.create()

187 LOG.info("Goal %s created", goal_name)

188

189 # Updating the internal states

190 self.available_goals_map[goal] = goal_map

191 # Map the old goal IDs to the new (equivalent) goal

192 for matching_goal in matching_goals:

193 goal_mapping[matching_goal.id] = goal

194

195 return goal_mapping

196

197 def _sync_strategy(self, strategy_map):

198 strategy_name = strategy_map.name

199 strategy_display_name = strategy_map.display_name

200 goal_name = strategy_map.goal_name

201 parameters_spec = strategy_map.parameters_spec

202 strategy_mapping = dict()

203

204 # Strategies that are matching by name with the given

205 # discovered strategy name

206 matching_strategies = [s for s in self.available_strategies

207 if s.name == strategy_name]

208 stale_strategies = self._soft_delete_stale_strategies(

209 strategy_map, matching_strategies)

210

211 if stale_strategies or not matching_strategies: 211 ↛ 226line 211 didn't jump to line 226 because the condition on line 211 was always true

212 strategy = objects.Strategy(self.ctx)

213 strategy.name = strategy_name

214 strategy.display_name = strategy_display_name

215 strategy.goal_id = objects.Goal.get_by_name(self.ctx, goal_name).id

216 strategy.parameters_spec = parameters_spec

217 strategy.create()

218 LOG.info("Strategy %s created", strategy_name)

219

220 # Updating the internal states

221 self.available_strategies_map[strategy] = strategy_map

222 # Map the old strategy IDs to the new (equivalent) strategy

223 for matching_strategy in matching_strategies:

224 strategy_mapping[matching_strategy.id] = strategy

225

226 return strategy_mapping

227

228 def _sync_scoringengine(self, scoringengine_map):

229 scoringengine_name = scoringengine_map.name

230 se_mapping = dict()

231 # Scoring Engines matching by id with discovered Scoring engine

232 matching_scoringengines = [se for se in self.available_scoringengines

233 if se.name == scoringengine_name]

234 stale_scoringengines = self._soft_delete_stale_scoringengines(

235 scoringengine_map, matching_scoringengines)

236

237 if stale_scoringengines or not matching_scoringengines: 237 ↛ 252line 237 didn't jump to line 252 because the condition on line 237 was always true

238 scoringengine = objects.ScoringEngine(self.ctx)

239 scoringengine.name = scoringengine_name

240 scoringengine.description = scoringengine_map.description

241 scoringengine.metainfo = scoringengine_map.metainfo

242 scoringengine.create()

243 LOG.info("Scoring Engine %s created", scoringengine_name)

244

245 # Updating the internal states

246 self.available_scoringengines_map[scoringengine] = \

247 scoringengine_map

248 # Map the old scoring engine names to the new (equivalent) SE

249 for matching_scoringengine in matching_scoringengines: 249 ↛ 250line 249 didn't jump to line 250 because the loop on line 249 never started

250 se_mapping[matching_scoringengine.name] = scoringengine

251

252 return se_mapping

253

254 def _sync_objects(self):

255 # First we find audit templates, audits and action plans that are stale

256 # because their associated goal or strategy has been modified and we

257 # update them in-memory

258 self._find_stale_audit_templates_due_to_goal()

259 self._find_stale_audit_templates_due_to_strategy()

260

261 self._find_stale_audits_due_to_goal()

262 self._find_stale_audits_due_to_strategy()

263

264 self._find_stale_action_plans_due_to_strategy()

265 self._find_stale_action_plans_due_to_audit()

266

267 # Then we handle the case where an audit template, an audit or an

268 # action plan becomes stale because its related goal does not

269 # exist anymore.

270 self._soft_delete_removed_goals()

271 # Then we handle the case where an audit template, an audit or an

272 # action plan becomes stale because its related strategy does not

273 # exist anymore.

274 self._soft_delete_removed_strategies()

275

276 # Finally, we save into the DB the updated stale audit templates

277 # and soft delete stale audits and action plans

278 for stale_audit_template in self.stale_audit_templates_map.values():

279 stale_audit_template.save()

280 LOG.info("Audit Template '%s' synced",

281 stale_audit_template.name)

282

283 for stale_audit in self.stale_audits_map.values():

284 stale_audit.save()

285 LOG.info("Stale audit '%s' synced and cancelled",

286 stale_audit.uuid)

287

288 for stale_action_plan in self.stale_action_plans_map.values():

289 stale_action_plan.save()

290 LOG.info("Stale action plan '%s' synced and cancelled",

291 stale_action_plan.uuid)

292

293 def _find_stale_audit_templates_due_to_goal(self):

294 for goal_id, synced_goal in self.goal_mapping.items():

295 filters = {"goal_id": goal_id}

296 stale_audit_templates = objects.AuditTemplate.list(

297 self.ctx, filters=filters)

298

299 # Update the goal ID for the stale audit templates (w/o saving)

300 for audit_template in stale_audit_templates:

301 if audit_template.id not in self.stale_audit_templates_map: 301 ↛ 306line 301 didn't jump to line 306 because the condition on line 301 was always true

302 audit_template.goal_id = synced_goal.id

303 self.stale_audit_templates_map[audit_template.id] = (

304 audit_template)

305 else:

306 self.stale_audit_templates_map[

307 audit_template.id].goal_id = synced_goal.id

308

309 def _find_stale_audit_templates_due_to_strategy(self):

310 for strategy_id, synced_strategy in self.strategy_mapping.items():

311 filters = {"strategy_id": strategy_id}

312 stale_audit_templates = objects.AuditTemplate.list(

313 self.ctx, filters=filters)

314

315 # Update strategy IDs for all stale audit templates (w/o saving)

316 for audit_template in stale_audit_templates:

317 if audit_template.id not in self.stale_audit_templates_map:

318 audit_template.strategy_id = synced_strategy.id

319 self.stale_audit_templates_map[audit_template.id] = (

320 audit_template)

321 else:

322 self.stale_audit_templates_map[

323 audit_template.id].strategy_id = synced_strategy.id

324

325 def _find_stale_audits_due_to_goal(self):

326 for goal_id, synced_goal in self.goal_mapping.items():

327 filters = {"goal_id": goal_id}

328 stale_audits = objects.Audit.list(

329 self.ctx, filters=filters, eager=True)

330

331 # Update the goal ID for the stale audits (w/o saving)

332 for audit in stale_audits:

333 if audit.id not in self.stale_audits_map: 333 ↛ 337line 333 didn't jump to line 337 because the condition on line 333 was always true

334 audit.goal_id = synced_goal.id

335 self.stale_audits_map[audit.id] = audit

336 else:

337 self.stale_audits_map[audit.id].goal_id = synced_goal.id

338

339 def _find_stale_audits_due_to_strategy(self):

340 for strategy_id, synced_strategy in self.strategy_mapping.items():

341 filters = {"strategy_id": strategy_id}

342 stale_audits = objects.Audit.list(

343 self.ctx, filters=filters, eager=True)

344 # Update strategy IDs for all stale audits (w/o saving)

345 for audit in stale_audits:

346 if audit.id not in self.stale_audits_map:

347 audit.strategy_id = synced_strategy.id

348 audit.state = objects.audit.State.CANCELLED

349 self.stale_audits_map[audit.id] = audit

350 else:

351 self.stale_audits_map[

352 audit.id].strategy_id = synced_strategy.id

353 self.stale_audits_map[

354 audit.id].state = objects.audit.State.CANCELLED

355

356 def _find_stale_action_plans_due_to_strategy(self):

357 for strategy_id, synced_strategy in self.strategy_mapping.items():

358 filters = {"strategy_id": strategy_id}

359 stale_action_plans = objects.ActionPlan.list(

360 self.ctx, filters=filters, eager=True)

361

362 # Update strategy IDs for all stale action plans (w/o saving)

363 for action_plan in stale_action_plans:

364 if action_plan.id not in self.stale_action_plans_map: 364 ↛ 369line 364 didn't jump to line 369 because the condition on line 364 was always true

365 action_plan.strategy_id = synced_strategy.id

366 action_plan.state = objects.action_plan.State.CANCELLED

367 self.stale_action_plans_map[action_plan.id] = action_plan

368 else:

369 self.stale_action_plans_map[

370 action_plan.id].strategy_id = synced_strategy.id

371 self.stale_action_plans_map[

372 action_plan.id].state = (

373 objects.action_plan.State.CANCELLED)

374

375 def _find_stale_action_plans_due_to_audit(self):

376 for audit_id, synced_audit in self.stale_audits_map.items():

377 filters = {"audit_id": audit_id}

378 stale_action_plans = objects.ActionPlan.list(

379 self.ctx, filters=filters, eager=True)

380

381 # Update audit IDs for all stale action plans (w/o saving)

382 for action_plan in stale_action_plans:

383 if action_plan.id not in self.stale_action_plans_map: 383 ↛ 384line 383 didn't jump to line 384 because the condition on line 383 was never true

384 action_plan.audit_id = synced_audit.id

385 action_plan.state = objects.action_plan.State.CANCELLED

386 self.stale_action_plans_map[action_plan.id] = action_plan

387 else:

388 self.stale_action_plans_map[

389 action_plan.id].audit_id = synced_audit.id

390 self.stale_action_plans_map[

391 action_plan.id].state = (

392 objects.action_plan.State.CANCELLED)

393

394 def _soft_delete_removed_goals(self):

395 removed_goals = [

396 g for g in self.available_goals

397 if g.name not in self.discovered_map['goals']]

398 for removed_goal in removed_goals:

399 removed_goal.soft_delete()

400 filters = {"goal_id": removed_goal.id}

401

402 invalid_ats = objects.AuditTemplate.list(self.ctx, filters=filters)

403 for at in invalid_ats:

404 LOG.warning(

405 "Audit Template '%(audit_template)s' references a "

406 "goal that does not exist", audit_template=at.uuid)

407

408 stale_audits = objects.Audit.list(

409 self.ctx, filters=filters, eager=True)

410 for audit in stale_audits:

411 LOG.warning(

412 "Audit '%(audit)s' references a "

413 "goal that does not exist", audit=audit.uuid)

414 if audit.id not in self.stale_audits_map: 414 ↛ 418line 414 didn't jump to line 418 because the condition on line 414 was always true

415 audit.state = objects.audit.State.CANCELLED

416 self.stale_audits_map[audit.id] = audit

417 else:

418 self.stale_audits_map[

419 audit.id].state = objects.audit.State.CANCELLED

420

421 def _soft_delete_removed_strategies(self):

422 removed_strategies = [

423 s for s in self.available_strategies

424 if s.name not in self.discovered_map['strategies']]

425

426 for removed_strategy in removed_strategies:

427 removed_strategy.soft_delete()

428 filters = {"strategy_id": removed_strategy.id}

429 invalid_ats = objects.AuditTemplate.list(self.ctx, filters=filters)

430 for at in invalid_ats:

431 LOG.info(

432 "Audit Template '%(audit_template)s' references a "

433 "strategy that does not exist",

434 audit_template=at.uuid)

435 # In this case we can reset the strategy ID to None

436 # so the audit template can still achieve the same goal

437 # but with a different strategy

438 if at.id not in self.stale_audit_templates_map: 438 ↛ 442line 438 didn't jump to line 442 because the condition on line 438 was always true

439 at.strategy_id = None

440 self.stale_audit_templates_map[at.id] = at

441 else:

442 self.stale_audit_templates_map[at.id].strategy_id = None

443

444 stale_audits = objects.Audit.list(

445 self.ctx, filters=filters, eager=True)

446 for audit in stale_audits:

447 LOG.warning(

448 "Audit '%(audit)s' references a "

449 "strategy that does not exist", audit=audit.uuid)

450 if audit.id not in self.stale_audits_map: 450 ↛ 451line 450 didn't jump to line 451 because the condition on line 450 was never true

451 audit.state = objects.audit.State.CANCELLED

452 self.stale_audits_map[audit.id] = audit

453 else:

454 self.stale_audits_map[

455 audit.id].state = objects.audit.State.CANCELLED

456

457 stale_action_plans = objects.ActionPlan.list(

458 self.ctx, filters=filters, eager=True)

459 for action_plan in stale_action_plans:

460 LOG.warning(

461 "Action Plan '%(action_plan)s' references a "

462 "strategy that does not exist",

463 action_plan=action_plan.uuid)

464 if action_plan.id not in self.stale_action_plans_map: 464 ↛ 468line 464 didn't jump to line 468 because the condition on line 464 was always true

465 action_plan.state = objects.action_plan.State.CANCELLED

466 self.stale_action_plans_map[action_plan.id] = action_plan

467 else:

468 self.stale_action_plans_map[

469 action_plan.id].state = (

470 objects.action_plan.State.CANCELLED)

471

472 def _soft_delete_removed_scoringengines(self):

473 removed_se = [

474 se for se in self.available_scoringengines

475 if se.name not in self.discovered_map['scoringengines']]

476 for se in removed_se: 476 ↛ 477line 476 didn't jump to line 477 because the loop on line 476 never started

477 LOG.info("Scoring Engine %s removed", se.name)

478 se.soft_delete()

479

480 def _discover(self):

481 strategies_map = {}

482 goals_map = {}

483 scoringengines_map = {}

484 discovered_map = {

485 "goals": goals_map,

486 "strategies": strategies_map,

487 "scoringengines": scoringengines_map}

488 goal_loader = default.DefaultGoalLoader()

489 implemented_goals = goal_loader.list_available()

490

491 strategy_loader = default.DefaultStrategyLoader()

492 implemented_strategies = strategy_loader.list_available()

493

494 for goal_cls in implemented_goals.values():

495 goals_map[goal_cls.get_name()] = GoalMapping(

496 name=goal_cls.get_name(),

497 display_name=goal_cls.get_translatable_display_name(),

498 efficacy_specification=tuple(

499 IndicatorSpec(**indicator.to_dict())

500 for indicator in goal_cls.get_efficacy_specification(

501 ).get_indicators_specifications()))

502

503 for strategy_cls in implemented_strategies.values():

504 strategies_map[strategy_cls.get_name()] = StrategyMapping(

505 name=strategy_cls.get_name(),

506 goal_name=strategy_cls.get_goal_name(),

507 display_name=strategy_cls.get_translatable_display_name(),

508 parameters_spec=str(strategy_cls.get_schema()))

509

510 for se in scoring_factory.get_scoring_engine_list():

511 scoringengines_map[se.get_name()] = ScoringEngineMapping(

512 name=se.get_name(),

513 description=se.get_description(),

514 metainfo=se.get_metainfo())

515

516 return discovered_map

517

518 def _soft_delete_stale_goals(self, goal_map, matching_goals):

519 """Soft delete the stale goals

520

521 :param goal_map: discovered goal map

522 :type goal_map: :py:class:`~.GoalMapping` instance

523 :param matching_goals: list of DB goals matching the goal_map

524 :type matching_goals: list of :py:class:`~.objects.Goal` instances

525 :returns: A list of soft deleted DB goals (subset of matching goals)

526 :rtype: list of :py:class:`~.objects.Goal` instances

527 """

528 goal_display_name = goal_map.display_name

529 goal_name = goal_map.name

530 goal_efficacy_spec = goal_map.efficacy_specification

531

532 stale_goals = []

533 for matching_goal in matching_goals:

534 if (matching_goal.efficacy_specification == goal_efficacy_spec and 534 ↛ 536line 534 didn't jump to line 536 because the condition on line 534 was never true

535 matching_goal.display_name == goal_display_name):

536 LOG.info("Goal %s unchanged", goal_name)

537 else:

538 LOG.info("Goal %s modified", goal_name)

539 matching_goal.soft_delete()

540 stale_goals.append(matching_goal)

541

542 return stale_goals

543

544 def _soft_delete_stale_strategies(self, strategy_map, matching_strategies):

545 strategy_name = strategy_map.name

546 strategy_display_name = strategy_map.display_name

547 parameters_spec = strategy_map.parameters_spec

548

549 stale_strategies = []

550 for matching_strategy in matching_strategies:

551 if (matching_strategy.display_name == strategy_display_name and 551 ↛ 555line 551 didn't jump to line 555 because the condition on line 551 was never true

552 matching_strategy.goal_id not in self.goal_mapping and

553 matching_strategy.parameters_spec ==

554 ast.literal_eval(parameters_spec)):

555 LOG.info("Strategy %s unchanged", strategy_name)

556 else:

557 LOG.info("Strategy %s modified", strategy_name)

558 matching_strategy.soft_delete()

559 stale_strategies.append(matching_strategy)

560

561 return stale_strategies

562

563 def _soft_delete_stale_scoringengines(

564 self, scoringengine_map, matching_scoringengines):

565 se_name = scoringengine_map.name

566 se_description = scoringengine_map.description

567 se_metainfo = scoringengine_map.metainfo

568

569 stale_scoringengines = []

570 for matching_scoringengine in matching_scoringengines: 570 ↛ 571line 570 didn't jump to line 571 because the loop on line 570 never started

571 if (matching_scoringengine.description == se_description and

572 matching_scoringengine.metainfo == se_metainfo):

573 LOG.info("Scoring Engine %s unchanged", se_name)

574 else:

575 LOG.info("Scoring Engine %s modified", se_name)

576 matching_scoringengine.soft_delete()

577 stale_scoringengines.append(matching_scoringengine)

578

579 return stale_scoringengines