Update Notification for escalation when use two or more groups notifications

fm/models/activealarm.py

@@ -34,8 +34,7 @@ from mongoengine.errors import SaveConditionError
 from noc.core.mongo.fields import ForeignKeyField, PlainReferenceField
 from noc.aaa.models.user import User
 from noc.main.models.style import Style
-from noc.main.models.notificationgroup import NotificationGroup
-from noc.main.models.template import Template
+from noc.main.models.notificationgroup import Notifications
 from noc.sa.models.managedobject import ManagedObject
 from noc.sa.models.servicesummary import ServiceSummary, SummaryItem, ObjectSummaryItem
 from noc.core.datastream.decorator import datastream
@@ -123,8 +122,7 @@ class ActiveAlarm(Document):
     total_services = ListField(EmbeddedDocumentField(SummaryItem))
     total_subscribers = ListField(EmbeddedDocumentField(SummaryItem))
     # Template and notification group to send close notification
-    clear_template = ForeignKeyField(Template, required=False)
-    clear_notification_group = ForeignKeyField(NotificationGroup, required=False)
+    clear_notifications = ListField(EmbeddedDocumentField(Notifications))
     # Paths
     adm_path = ListField(IntField())
     segment_path = ListField(ObjectIdField())
@@ -316,28 +314,40 @@ class ActiveAlarm(Document):
         self.delete()
         # Close TT
         # MUST be after .delete() to prevent race conditions
-        if a.escalation_tt or self.clear_template:
-            if self.clear_template:
-                ctx = {"alarm": a}
-                subject = self.clear_template.render_subject(**ctx)
-                body = self.clear_template.render_body(**ctx)
+        if a.escalation_tt or self.clear_notifications:
+            subject = "Alarm cleared"
+            body = "Alarm has been cleared"
+            if self.clear_notifications:
+                for cn in self.clear_notifications:
+                    ctx = {"alarm": a}
+                    if "template" in cn:
+                        subject = cn["template"].render_subject(**ctx)
+                        body = cn["template"].render_body(**ctx)
+                    call_later(
+                        "noc.services.escalator.escalation.notify_close",
+                        scheduler="escalator",
+                        pool=self.managed_object.escalator_shard,
+                        max_runs=config.fm.alarm_close_retries,
+                        alarm_id=self.id,
+                        tt_id=self.escalation_tt,
+                        subject=subject,
+                        body=body,
+                        notification_group_id=cn["group"].id if "group" in cn else None,
+                        close_tt=self.close_tt,
+                    )
             else:
-                subject = "Alarm cleared"
-                body = "Alarm has been cleared"
-            call_later(
-                "noc.services.escalator.escalation.notify_close",
-                scheduler="escalator",
-                pool=self.managed_object.escalator_shard,
-                max_runs=config.fm.alarm_close_retries,
-                alarm_id=self.id,
-                tt_id=self.escalation_tt,
-                subject=subject,
-                body=body,
-                notification_group_id=self.clear_notification_group.id
-                if self.clear_notification_group
-                else None,
-                close_tt=self.close_tt,
-            )
+                call_later(
+                    "noc.services.escalator.escalation.notify_close",
+                    scheduler="escalator",
+                    pool=self.managed_object.escalator_shard,
+                    max_runs=config.fm.alarm_close_retries,
+                    alarm_id=self.id,
+                    tt_id=self.escalation_tt,
+                    subject=subject,
+                    body=body,
+                    notification_group_id=None,
+                    close_tt=self.close_tt,
+                )
         # Gather diagnostics
         AlarmDiagnosticConfig.on_clear(a)
         # Return archived
@@ -787,9 +797,8 @@ class ActiveAlarm(Document):
                 {"_id": self.id}, {"$set": {"escalation_ctx": current_context}}
             )
 
-    def set_clear_notification(self, notification_group, template):
-        self.clear_notification_group = notification_group
-        self.clear_template = template
+    def set_clear_notifications(self, clear_notifications):
+        self.clear_notifications = clear_notifications
         self.safe_save(save_condition={"managed_object": {"$exists": True}, "id": self.id})
 
     def iter_consequences(self):

main/models/notificationgroup.py

@@ -16,17 +16,21 @@ from threading import Lock
 # Third-party modules
 import six
 from django.db import models
+from mongoengine.document import EmbeddedDocument
 import cachetools
 
 # NOC modules
 from noc.core.model.base import NOCModel
 from noc.aaa.models.user import User
 from noc.settings import LANGUAGE_CODE
+from noc.lib.nosql import ForeignKeyField
 from noc.lib.timepattern import TimePatternList
 from noc.core.service.pub import pub
 from noc.core.model.decorator import on_delete_check
+from noc.main.models.template import Template
 from .timepattern import TimePattern
 
+
 id_lock = Lock()
 logger = logging.getLogger(__name__)
 
@@ -265,3 +269,17 @@ class NotificationGroupOther(NOCModel):
             self.notification_method,
             self.params,
         )
+
+
+class Notifications(EmbeddedDocument):
+    meta = {
+        "strict": False
+    }
+    group = ForeignKeyField(NotificationGroup, required=False)
+    template = ForeignKeyField(Template, required=False)
+
+    def __unicode__(self):
+        return u"%s : %s" % (self.group, self.template)
+
+    def __eq__(self, other):
+        return (self.group == other.group and self.template == other.template)

services/escalator/escalation.py

@@ -23,7 +23,7 @@ from noc.sa.models.managedobjectprofile import ManagedObjectProfile
 from noc.fm.models.activealarm import ActiveAlarm
 from noc.fm.models.archivedalarm import ArchivedAlarm
 from noc.core.perf import metrics
-from noc.main.models.notificationgroup import NotificationGroup
+from noc.main.models.notificationgroup import NotificationGroup, Notifications
 from noc.maintenance.models.maintenance import Maintenance
 from noc.config import config
 from noc.core.tt.error import TTError, TemporaryTTError
@@ -87,6 +87,7 @@ def escalate(alarm_id, escalation_id, escalation_delay, *args, **kwargs):
     else:
         sample = PARENT_SAMPLE
     with Span(client="escalator", sample=sample) as ctx:
+        clear_notification = []
         alarm.set_escalation_context()
         # Evaluate escalation chain
         mo = alarm.managed_object
@@ -265,27 +266,41 @@ def escalate(alarm_id, escalation_id, escalation_delay, *args, **kwargs):
                 logger.debug("[%s] Notification message:\nSubject: %s\n%s", alarm_id, subject, body)
                 log("Sending notification to group %s", a.notification_group.name)
                 a.notification_group.notify(subject, body)
-                alarm.set_clear_notification(a.notification_group, a.clear_template)
+                notification = Notifications(
+                    group=a.notification_group if a.notification_group else None,
+                    template=a.clear_template if a.clear_template else None,
+                )
+                clear_notification += [notification]
                 metrics["escalation_notify"] += 1
             #
             if a.stop_processing:
                 logger.debug("Stopping processing")
                 break
+        alarm.set_clear_notifications(clear_notification)
         nalarm = get_alarm(alarm_id)
         if nalarm and nalarm.status == "C" and nalarm.escalation_tt:
             log("Alarm has been closed during escalation. Try to deescalate")
             metrics["escalation_closed_while_escalated"] += 1
             if not nalarm.escalation_close_ts and not nalarm.escalation_close_error:
-                notify_close(
-                    alarm_id=alarm_id,
-                    tt_id=nalarm.escalation_tt,
-                    subject="Closing",
-                    body="Closing",
-                    notification_group_id=alarm.clear_notification_group.id
-                    if alarm.clear_notification_group
-                    else None,
-                    close_tt=alarm.close_tt,
-                )
+                if alarm.clear_notifications:
+                    for cn in alarm.clear_notifications:
+                        notify_close(
+                            alarm_id=alarm_id,
+                            tt_id=nalarm.escalation_tt,
+                            subject="Closing",
+                            body="Closing",
+                            notification_group_id=cn["group"].id if "group" in cn else None,
+                            close_tt=alarm.close_tt,
+                        )
+                else:
+                    notify_close(
+                        alarm_id=alarm_id,
+                        tt_id=nalarm.escalation_tt,
+                        subject="Closing",
+                        body="Closing",
+                        notification_group_id=None,
+                        close_tt=alarm.close_tt,
+                    )
         logger.info("[%s] Escalations loop end", alarm_id)