Coverage for watcher/db/purge.py: 91%
246 statements
« prev ^ index » next coverage.py v7.8.2, created at 2025-06-17 12:22 +0000
1# -*- encoding: utf-8 -*-
2# Copyright (c) 2016 b<>com
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
13# implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
18import collections
19import datetime
20import itertools
21import sys
23from oslo_log import log
24from oslo_utils import strutils
25import prettytable as ptable
27from watcher._i18n import _
28from watcher._i18n import lazy_translation_enabled
29from watcher.common import context
30from watcher.common import exception
31from watcher.common import utils
32from watcher import objects
34LOG = log.getLogger(__name__)
37class WatcherObjectsMap(object):
38 """Wrapper to deal with watcher objects per type
40 This wrapper object contains a list of watcher objects per type.
41 Its main use is to simplify the merge of watcher objects by avoiding
42 duplicates, but also for representing the relationships between these
43 objects.
44 """
46 # This is for generating the .pot translations
47 keymap = collections.OrderedDict([
48 ("goals", _("Goals")),
49 ("strategies", _("Strategies")),
50 ("audit_templates", _("Audit Templates")),
51 ("audits", _("Audits")),
52 ("action_plans", _("Action Plans")),
53 ("actions", _("Actions")),
54 ])
56 def __init__(self):
57 for attr_name in self.keys():
58 setattr(self, attr_name, [])
60 def values(self):
61 return (getattr(self, key) for key in self.keys())
63 @classmethod
64 def keys(cls):
65 return cls.keymap.keys()
67 def __iter__(self):
68 return itertools.chain(*self.values())
70 def __add__(self, other):
71 new_map = self.__class__()
73 # Merge the 2 items dicts into a new object (and avoid dupes)
74 for attr_name, initials, others in zip(self.keys(), self.values(),
75 other.values()):
76 # Creates a copy
77 merged = initials[:]
78 initials_ids = [item.id for item in initials]
79 non_dupes = [item for item in others
80 if item.id not in initials_ids]
81 merged += non_dupes
83 setattr(new_map, attr_name, merged)
85 return new_map
87 def __str__(self):
88 out = ""
89 for key, vals in zip(self.keys(), self.values()):
90 ids = [val.id for val in vals]
91 out += "%(key)s: %(val)s" % (dict(key=key, val=ids))
92 out += "\n"
93 return out
95 def __len__(self):
96 return sum(len(getattr(self, key)) for key in self.keys())
98 def get_count_table(self):
99 headers = list(self.keymap.values())
100 headers.append(_("Total")) # We also add a total count
101 translated_headers = [
102 h.translate() if lazy_translation_enabled() else h
103 for h in headers
104 ]
106 counters = [len(cat_vals) for cat_vals in self.values()] + [len(self)]
107 table = ptable.PrettyTable(field_names=translated_headers)
108 table.add_row(counters)
109 return table.get_string()
112class PurgeCommand(object):
113 """Purges the DB by removing soft deleted entries
115 The workflow for this purge is the following:
117 # Find soft deleted objects which are expired
118 # Find orphan objects
119 # Find their related objects whether they are expired or not
120 # Merge them together
121 # If it does not exceed the limit, destroy them all
122 """
124 ctx = context.make_context(show_deleted=True)
126 def __init__(self, age_in_days=None, max_number=None,
127 uuid=None, exclude_orphans=False, dry_run=None):
128 self.age_in_days = age_in_days
129 self.max_number = max_number
130 self.uuid = uuid
131 self.exclude_orphans = exclude_orphans
132 self.dry_run = dry_run
134 self._delete_up_to_max = None
135 self._objects_map = WatcherObjectsMap()
137 def get_expiry_date(self):
138 if not self.age_in_days:
139 return None
140 today = datetime.datetime.today()
141 expiry_date = today - datetime.timedelta(days=self.age_in_days)
142 return expiry_date
144 @classmethod
145 def get_goal_uuid(cls, uuid_or_name):
146 if uuid_or_name is None:
147 return
149 query_func = None
150 if not utils.is_uuid_like(uuid_or_name):
151 query_func = objects.Goal.get_by_name
152 else:
153 query_func = objects.Goal.get_by_uuid
155 try:
156 goal = query_func(cls.ctx, uuid_or_name)
157 except Exception as exc:
158 LOG.exception(exc)
159 raise exception.GoalNotFound(goal=uuid_or_name)
161 if not goal.deleted_at:
162 raise exception.NotSoftDeletedStateError(
163 name=_('Goal'), id=uuid_or_name)
165 return goal.uuid
167 def _find_goals(self, filters=None):
168 return objects.Goal.list(self.ctx, filters=filters)
170 def _find_strategies(self, filters=None):
171 return objects.Strategy.list(self.ctx, filters=filters)
173 def _find_audit_templates(self, filters=None):
174 return objects.AuditTemplate.list(self.ctx, filters=filters)
176 def _find_audits(self, filters=None):
177 return objects.Audit.list(self.ctx, filters=filters)
179 def _find_action_plans(self, filters=None):
180 return objects.ActionPlan.list(self.ctx, filters=filters)
182 def _find_actions(self, filters=None):
183 return objects.Action.list(self.ctx, filters=filters)
185 def _find_orphans(self):
186 orphans = WatcherObjectsMap()
188 filters = dict(deleted=False)
189 goals = objects.Goal.list(self.ctx, filters=filters)
190 strategies = objects.Strategy.list(self.ctx, filters=filters)
191 audit_templates = objects.AuditTemplate.list(self.ctx, filters=filters)
192 audits = objects.Audit.list(self.ctx, filters=filters)
193 action_plans = objects.ActionPlan.list(self.ctx, filters=filters)
194 actions = objects.Action.list(self.ctx, filters=filters)
196 goal_ids = set(g.id for g in goals)
197 orphans.strategies = [
198 strategy for strategy in strategies
199 if strategy.goal_id not in goal_ids]
201 strategy_ids = [s.id for s in (s for s in strategies
202 if s not in orphans.strategies)]
203 orphans.audit_templates = [
204 audit_template for audit_template in audit_templates
205 if audit_template.goal_id not in goal_ids or
206 (audit_template.strategy_id and
207 audit_template.strategy_id not in strategy_ids)]
209 orphans.audits = [
210 audit for audit in audits
211 if audit.goal_id not in goal_ids or
212 (audit.strategy_id and
213 audit.strategy_id not in strategy_ids)]
215 # Objects with orphan parents are themselves orphans
216 audit_ids = [audit.id for audit in audits
217 if audit not in orphans.audits]
218 orphans.action_plans = [
219 ap for ap in action_plans
220 if ap.audit_id not in audit_ids or
221 ap.strategy_id not in strategy_ids]
223 # Objects with orphan parents are themselves orphans
224 action_plan_ids = [ap.id for ap in action_plans
225 if ap not in orphans.action_plans]
226 orphans.actions = [
227 action for action in actions
228 if action.action_plan_id not in action_plan_ids]
230 LOG.debug("Orphans found:\n%s", orphans)
231 LOG.info("Orphans found:\n%s", orphans.get_count_table())
233 return orphans
235 def _find_soft_deleted_objects(self):
236 to_be_deleted = WatcherObjectsMap()
237 expiry_date = self.get_expiry_date()
238 filters = dict(deleted=True)
240 if self.uuid:
241 filters["uuid"] = self.uuid
242 if expiry_date:
243 filters.update(dict(deleted_at__lt=expiry_date))
245 to_be_deleted.goals.extend(self._find_goals(filters))
246 to_be_deleted.strategies.extend(self._find_strategies(filters))
247 to_be_deleted.audit_templates.extend(
248 self._find_audit_templates(filters))
249 to_be_deleted.audits.extend(self._find_audits(filters))
250 to_be_deleted.action_plans.extend(
251 self._find_action_plans(filters))
252 to_be_deleted.actions.extend(self._find_actions(filters))
254 soft_deleted_objs = self._find_related_objects(
255 to_be_deleted, base_filters=dict(deleted=True))
257 LOG.debug("Soft deleted objects:\n%s", soft_deleted_objs)
259 return soft_deleted_objs
261 def _find_related_objects(self, objects_map, base_filters=None):
262 base_filters = base_filters or {}
264 for goal in objects_map.goals:
265 filters = {}
266 filters.update(base_filters)
267 filters.update(dict(goal_id=goal.id))
268 related_objs = WatcherObjectsMap()
269 related_objs.strategies = self._find_strategies(filters)
270 related_objs.audit_templates = self._find_audit_templates(filters)
271 related_objs.audits = self._find_audits(filters)
272 objects_map += related_objs
274 for strategy in objects_map.strategies:
275 filters = {}
276 filters.update(base_filters)
277 filters.update(dict(strategy_id=strategy.id))
278 related_objs = WatcherObjectsMap()
279 related_objs.audit_templates = self._find_audit_templates(filters)
280 related_objs.audits = self._find_audits(filters)
281 objects_map += related_objs
283 for audit in objects_map.audits:
284 filters = {}
285 filters.update(base_filters)
286 filters.update(dict(audit_id=audit.id))
287 related_objs = WatcherObjectsMap()
288 related_objs.action_plans = self._find_action_plans(filters)
289 objects_map += related_objs
291 for action_plan in objects_map.action_plans:
292 filters = {}
293 filters.update(base_filters)
294 filters.update(dict(action_plan_id=action_plan.id))
295 related_objs = WatcherObjectsMap()
296 related_objs.actions = self._find_actions(filters)
297 objects_map += related_objs
299 return objects_map
301 def confirmation_prompt(self):
302 print(self._objects_map.get_count_table())
303 raw_val = input(
304 _("There are %(count)d objects set for deletion. "
305 "Continue? [y/N]") % dict(count=len(self._objects_map)))
307 return strutils.bool_from_string(raw_val)
309 def delete_up_to_max_prompt(self, objects_map):
310 print(objects_map.get_count_table())
311 print(_("The number of objects (%(num)s) to delete from the database "
312 "exceeds the maximum number of objects (%(max_number)s) "
313 "specified.") % dict(max_number=self.max_number,
314 num=len(objects_map)))
315 raw_val = input(
316 _("Do you want to delete objects up to the specified maximum "
317 "number? [y/N]"))
319 self._delete_up_to_max = strutils.bool_from_string(raw_val)
321 return self._delete_up_to_max
323 def _aggregate_objects(self):
324 """Objects aggregated on a 'per goal' basis"""
325 # todo: aggregate orphans as well
326 aggregate = []
327 for goal in self._objects_map.goals:
328 related_objs = WatcherObjectsMap()
330 # goals
331 related_objs.goals = [goal]
333 # strategies
334 goal_ids = [goal.id]
335 related_objs.strategies = [
336 strategy for strategy in self._objects_map.strategies
337 if strategy.goal_id in goal_ids
338 ]
340 # audit templates
341 strategy_ids = [
342 strategy.id for strategy in related_objs.strategies]
343 related_objs.audit_templates = [
344 at for at in self._objects_map.audit_templates
345 if at.goal_id in goal_ids or
346 (at.strategy_id and at.strategy_id in strategy_ids)
347 ]
349 # audits
350 related_objs.audits = [
351 audit for audit in self._objects_map.audits
352 if audit.goal_id in goal_ids
353 ]
355 # action plans
356 audit_ids = [audit.id for audit in related_objs.audits]
357 related_objs.action_plans = [
358 action_plan for action_plan in self._objects_map.action_plans
359 if action_plan.audit_id in audit_ids
360 ]
362 # actions
363 action_plan_ids = [
364 action_plan.id for action_plan in related_objs.action_plans
365 ]
366 related_objs.actions = [
367 action for action in self._objects_map.actions
368 if action.action_plan_id in action_plan_ids
369 ]
370 aggregate.append(related_objs)
372 return aggregate
374 def _get_objects_up_to_limit(self):
375 aggregated_objects = self._aggregate_objects()
376 to_be_deleted_subset = WatcherObjectsMap()
378 for aggregate in aggregated_objects: 378 ↛ 384line 378 didn't jump to line 384 because the loop on line 378 didn't complete
379 if len(aggregate) + len(to_be_deleted_subset) <= self.max_number:
380 to_be_deleted_subset += aggregate
381 else:
382 break
384 LOG.debug(to_be_deleted_subset)
385 return to_be_deleted_subset
387 def find_objects_to_delete(self):
388 """Finds all the objects to be purged
390 :returns: A mapping with all the Watcher objects to purged
391 :rtype: :py:class:`~.WatcherObjectsMap` instance
392 """
393 to_be_deleted = self._find_soft_deleted_objects()
395 if not self.exclude_orphans:
396 to_be_deleted += self._find_orphans()
398 LOG.debug("Objects to be deleted:\n%s", to_be_deleted)
400 return to_be_deleted
402 def do_delete(self):
403 LOG.info("Deleting...")
404 # Reversed to avoid errors with foreign keys
405 for entry in reversed(list(self._objects_map)):
406 entry.destroy()
408 def execute(self):
409 LOG.info("Starting purge command")
410 self._objects_map = self.find_objects_to_delete()
412 if (self.max_number is not None and
413 len(self._objects_map) > self.max_number):
414 if self.delete_up_to_max_prompt(self._objects_map): 414 ↛ 417line 414 didn't jump to line 417 because the condition on line 414 was always true
415 self._objects_map = self._get_objects_up_to_limit()
416 else:
417 return
419 _orphans_note = (_(" (orphans excluded)") if self.exclude_orphans
420 else _(" (may include orphans)"))
421 if not self.dry_run and self.confirmation_prompt(): 421 ↛ 426line 421 didn't jump to line 426 because the condition on line 421 was always true
422 self.do_delete()
423 print(_("Purge results summary%s:") % _orphans_note)
424 LOG.info("Purge results summary%s:", _orphans_note)
425 else:
426 LOG.debug(self._objects_map)
427 print(_("Here below is a table containing the objects "
428 "that can be purged%s:") % _orphans_note)
430 LOG.info("\n%s", self._objects_map.get_count_table())
431 print(self._objects_map.get_count_table())
432 LOG.info("Purge process completed")
435def purge(age_in_days, max_number, goal, exclude_orphans, dry_run):
436 """Removes soft deleted objects from the database
438 :param age_in_days: Number of days since deletion (from today)
439 to exclude from the purge. If None, everything will be purged.
440 :type age_in_days: int
441 :param max_number: Max number of objects expected to be deleted.
442 Prevents the deletion if exceeded. No limit if set to None.
443 :type max_number: int
444 :param goal: UUID or name of the goal to purge.
445 :type goal: str
446 :param exclude_orphans: Flag to indicate whether or not you want to
447 exclude orphans from deletion (default: False).
448 :type exclude_orphans: bool
449 :param dry_run: Flag to indicate whether or not you want to perform
450 a dry run (no deletion).
451 :type dry_run: bool
452 """
453 try:
454 if max_number and max_number < 0:
455 raise exception.NegativeLimitError
457 LOG.info("[options] age_in_days = %s", age_in_days)
458 LOG.info("[options] max_number = %s", max_number)
459 LOG.info("[options] goal = %s", goal)
460 LOG.info("[options] exclude_orphans = %s", exclude_orphans)
461 LOG.info("[options] dry_run = %s", dry_run)
463 uuid = PurgeCommand.get_goal_uuid(goal)
465 cmd = PurgeCommand(age_in_days, max_number, uuid,
466 exclude_orphans, dry_run)
468 cmd.execute()
470 except Exception as exc:
471 LOG.exception(exc)
472 print(exc)
473 sys.exit(1)