diff --git a/fm/models/activealarm.py b/fm/models/activealarm.py index 3c9b10b647484f6f669ce1cdb3476fa515284acd..63afc592061a894c88cd6d90e1a63d00f5b9a95e 100644 --- a/fm/models/activealarm.py +++ b/fm/models/activealarm.py @@ -34,8 +34,7 @@ from mongoengine.errors import SaveConditionError from noc.core.mongo.fields import ForeignKeyField, PlainReferenceField from noc.aaa.models.user import User from noc.main.models.style import Style -from noc.main.models.notificationgroup import NotificationGroup -from noc.main.models.template import Template +from noc.main.models.notificationgroup import Notifications from noc.sa.models.managedobject import ManagedObject from noc.sa.models.servicesummary import ServiceSummary, SummaryItem, ObjectSummaryItem from noc.core.datastream.decorator import datastream @@ -123,8 +122,7 @@ class ActiveAlarm(Document): total_services = ListField(EmbeddedDocumentField(SummaryItem)) total_subscribers = ListField(EmbeddedDocumentField(SummaryItem)) # Template and notification group to send close notification - clear_template = ForeignKeyField(Template, required=False) - clear_notification_group = ForeignKeyField(NotificationGroup, required=False) + clear_notifications = ListField(EmbeddedDocumentField(Notifications)) # Paths adm_path = ListField(IntField()) segment_path = ListField(ObjectIdField()) @@ -316,28 +314,40 @@ class ActiveAlarm(Document): self.delete() # Close TT # MUST be after .delete() to prevent race conditions - if a.escalation_tt or self.clear_template: - if self.clear_template: - ctx = {"alarm": a} - subject = self.clear_template.render_subject(**ctx) - body = self.clear_template.render_body(**ctx) + if a.escalation_tt or self.clear_notifications: + subject = "Alarm cleared" + body = "Alarm has been cleared" + if self.clear_notifications: + for cn in self.clear_notifications: + ctx = {"alarm": a} + if "template" in cn: + subject = cn["template"].render_subject(**ctx) + body = cn["template"].render_body(**ctx) + call_later( + "noc.services.escalator.escalation.notify_close", + scheduler="escalator", + pool=self.managed_object.escalator_shard, + max_runs=config.fm.alarm_close_retries, + alarm_id=self.id, + tt_id=self.escalation_tt, + subject=subject, + body=body, + notification_group_id=cn["group"].id if "group" in cn else None, + close_tt=self.close_tt, + ) else: - subject = "Alarm cleared" - body = "Alarm has been cleared" - call_later( - "noc.services.escalator.escalation.notify_close", - scheduler="escalator", - pool=self.managed_object.escalator_shard, - max_runs=config.fm.alarm_close_retries, - alarm_id=self.id, - tt_id=self.escalation_tt, - subject=subject, - body=body, - notification_group_id=self.clear_notification_group.id - if self.clear_notification_group - else None, - close_tt=self.close_tt, - ) + call_later( + "noc.services.escalator.escalation.notify_close", + scheduler="escalator", + pool=self.managed_object.escalator_shard, + max_runs=config.fm.alarm_close_retries, + alarm_id=self.id, + tt_id=self.escalation_tt, + subject=subject, + body=body, + notification_group_id=None, + close_tt=self.close_tt, + ) # Gather diagnostics AlarmDiagnosticConfig.on_clear(a) # Return archived @@ -789,9 +799,8 @@ class ActiveAlarm(Document): {"_id": self.id}, {"$set": {"escalation_ctx": current_context}} ) - def set_clear_notification(self, notification_group, template): - self.clear_notification_group = notification_group - self.clear_template = template + def set_clear_notifications(self, clear_notifications): + self.clear_notifications = clear_notifications self.safe_save(save_condition={"managed_object": {"$exists": True}, "id": self.id}) def iter_consequences(self): diff --git a/main/models/notificationgroup.py b/main/models/notificationgroup.py index 02e90d6a5fc00ec4ac46e305c5c9a90d21d14edd..444b8ba962bf84305d120dba2fc5a92e25d76441 100644 --- a/main/models/notificationgroup.py +++ b/main/models/notificationgroup.py @@ -16,17 +16,21 @@ from threading import Lock # Third-party modules import six from django.db import models +from mongoengine.document import EmbeddedDocument import cachetools # NOC modules from noc.core.model.base import NOCModel from noc.aaa.models.user import User from noc.settings import LANGUAGE_CODE +from noc.lib.nosql import ForeignKeyField from noc.lib.timepattern import TimePatternList from noc.core.service.pub import pub from noc.core.model.decorator import on_delete_check +from noc.main.models.template import Template from .timepattern import TimePattern + id_lock = Lock() logger = logging.getLogger(__name__) @@ -265,3 +269,17 @@ class NotificationGroupOther(NOCModel): self.notification_method, self.params, ) + + +class Notifications(EmbeddedDocument): + meta = { + "strict": False + } + group = ForeignKeyField(NotificationGroup, required=False) + template = ForeignKeyField(Template, required=False) + + def __unicode__(self): + return u"%s : %s" % (self.group, self.template) + + def __eq__(self, other): + return (self.group == other.group and self.template == other.template) diff --git a/services/escalator/escalation.py b/services/escalator/escalation.py index 32e13b7a622e15a6273254e25e31b3f2527f05bf..5c27e6c28ed2af9042ce5ae00eb62d3b9f271a49 100644 --- a/services/escalator/escalation.py +++ b/services/escalator/escalation.py @@ -23,7 +23,7 @@ from noc.sa.models.managedobjectprofile import ManagedObjectProfile from noc.fm.models.activealarm import ActiveAlarm from noc.fm.models.archivedalarm import ArchivedAlarm from noc.core.perf import metrics -from noc.main.models.notificationgroup import NotificationGroup +from noc.main.models.notificationgroup import NotificationGroup, Notifications from noc.maintenance.models.maintenance import Maintenance from noc.config import config from noc.core.tt.error import TTError, TemporaryTTError @@ -87,6 +87,7 @@ def escalate(alarm_id, escalation_id, escalation_delay, *args, **kwargs): else: sample = PARENT_SAMPLE with Span(client="escalator", sample=sample) as ctx: + clear_notification = [] alarm.set_escalation_context() # Evaluate escalation chain mo = alarm.managed_object @@ -269,27 +270,41 @@ def escalate(alarm_id, escalation_id, escalation_delay, *args, **kwargs): logger.debug("[%s] Notification message:\nSubject: %s\n%s", alarm_id, subject, body) log("Sending notification to group %s", a.notification_group.name) a.notification_group.notify(subject, body) - alarm.set_clear_notification(a.notification_group, a.clear_template) + notification = Notifications( + group=a.notification_group if a.notification_group else None, + template=a.clear_template if a.clear_template else None, + ) + clear_notification += [notification] metrics["escalation_notify"] += 1 # if a.stop_processing: logger.debug("Stopping processing") break + alarm.set_clear_notifications(clear_notification) nalarm = get_alarm(alarm_id) if nalarm and nalarm.status == "C" and nalarm.escalation_tt: log("Alarm has been closed during escalation. Try to deescalate") metrics["escalation_closed_while_escalated"] += 1 if not nalarm.escalation_close_ts and not nalarm.escalation_close_error: - notify_close( - alarm_id=alarm_id, - tt_id=nalarm.escalation_tt, - subject="Closing", - body="Closing", - notification_group_id=alarm.clear_notification_group.id - if alarm.clear_notification_group - else None, - close_tt=alarm.close_tt, - ) + if alarm.clear_notifications: + for cn in alarm.clear_notifications: + notify_close( + alarm_id=alarm_id, + tt_id=nalarm.escalation_tt, + subject="Closing", + body="Closing", + notification_group_id=cn["group"].id if "group" in cn else None, + close_tt=alarm.close_tt, + ) + else: + notify_close( + alarm_id=alarm_id, + tt_id=nalarm.escalation_tt, + subject="Closing", + body="Closing", + notification_group_id=None, + close_tt=alarm.close_tt, + ) logger.info("[%s] Escalations loop end", alarm_id)