Commit 08dc3eb4 authored by MaksimSmile13's avatar MaksimSmile13 Committed by Andrey Vertiprahov
Browse files

Update Notification for escalation when use two or more groups notifications

parent 62514157
Pipeline #15338 failed with stages
in 1 minute and 55 seconds
......@@ -34,8 +34,7 @@ from mongoengine.errors import SaveConditionError
from noc.core.mongo.fields import ForeignKeyField, PlainReferenceField
from noc.aaa.models.user import User
from noc.main.models.style import Style
from noc.main.models.notificationgroup import NotificationGroup
from noc.main.models.template import Template
from noc.main.models.notificationgroup import Notifications
from noc.sa.models.managedobject import ManagedObject
from noc.sa.models.servicesummary import ServiceSummary, SummaryItem, ObjectSummaryItem
from noc.core.datastream.decorator import datastream
......@@ -123,8 +122,7 @@ class ActiveAlarm(Document):
total_services = ListField(EmbeddedDocumentField(SummaryItem))
total_subscribers = ListField(EmbeddedDocumentField(SummaryItem))
# Template and notification group to send close notification
clear_template = ForeignKeyField(Template, required=False)
clear_notification_group = ForeignKeyField(NotificationGroup, required=False)
clear_notifications = ListField(EmbeddedDocumentField(Notifications))
# Paths
adm_path = ListField(IntField())
segment_path = ListField(ObjectIdField())
......@@ -316,14 +314,28 @@ class ActiveAlarm(Document):
self.delete()
# Close TT
# MUST be after .delete() to prevent race conditions
if a.escalation_tt or self.clear_template:
if self.clear_template:
ctx = {"alarm": a}
subject = self.clear_template.render_subject(**ctx)
body = self.clear_template.render_body(**ctx)
else:
if a.escalation_tt or self.clear_notifications:
subject = "Alarm cleared"
body = "Alarm has been cleared"
if self.clear_notifications:
for cn in self.clear_notifications:
ctx = {"alarm": a}
if "template" in cn:
subject = cn["template"].render_subject(**ctx)
body = cn["template"].render_body(**ctx)
call_later(
"noc.services.escalator.escalation.notify_close",
scheduler="escalator",
pool=self.managed_object.escalator_shard,
max_runs=config.fm.alarm_close_retries,
alarm_id=self.id,
tt_id=self.escalation_tt,
subject=subject,
body=body,
notification_group_id=cn["group"].id if "group" in cn else None,
close_tt=self.close_tt,
)
else:
call_later(
"noc.services.escalator.escalation.notify_close",
scheduler="escalator",
......@@ -333,9 +345,7 @@ class ActiveAlarm(Document):
tt_id=self.escalation_tt,
subject=subject,
body=body,
notification_group_id=self.clear_notification_group.id
if self.clear_notification_group
else None,
notification_group_id=None,
close_tt=self.close_tt,
)
# Gather diagnostics
......@@ -789,9 +799,8 @@ class ActiveAlarm(Document):
{"_id": self.id}, {"$set": {"escalation_ctx": current_context}}
)
def set_clear_notification(self, notification_group, template):
self.clear_notification_group = notification_group
self.clear_template = template
def set_clear_notifications(self, clear_notifications):
self.clear_notifications = clear_notifications
self.safe_save(save_condition={"managed_object": {"$exists": True}, "id": self.id})
def iter_consequences(self):
......
......@@ -16,17 +16,21 @@ from threading import Lock
# Third-party modules
import six
from django.db import models
from mongoengine.document import EmbeddedDocument
import cachetools
# NOC modules
from noc.core.model.base import NOCModel
from noc.aaa.models.user import User
from noc.settings import LANGUAGE_CODE
from noc.lib.nosql import ForeignKeyField
from noc.lib.timepattern import TimePatternList
from noc.core.service.pub import pub
from noc.core.model.decorator import on_delete_check
from noc.main.models.template import Template
from .timepattern import TimePattern
id_lock = Lock()
logger = logging.getLogger(__name__)
......@@ -265,3 +269,17 @@ class NotificationGroupOther(NOCModel):
self.notification_method,
self.params,
)
class Notifications(EmbeddedDocument):
meta = {
"strict": False
}
group = ForeignKeyField(NotificationGroup, required=False)
template = ForeignKeyField(Template, required=False)
def __unicode__(self):
return u"%s : %s" % (self.group, self.template)
def __eq__(self, other):
return (self.group == other.group and self.template == other.template)
......@@ -23,7 +23,7 @@ from noc.sa.models.managedobjectprofile import ManagedObjectProfile
from noc.fm.models.activealarm import ActiveAlarm
from noc.fm.models.archivedalarm import ArchivedAlarm
from noc.core.perf import metrics
from noc.main.models.notificationgroup import NotificationGroup
from noc.main.models.notificationgroup import NotificationGroup, Notifications
from noc.maintenance.models.maintenance import Maintenance
from noc.config import config
from noc.core.tt.error import TTError, TemporaryTTError
......@@ -87,6 +87,7 @@ def escalate(alarm_id, escalation_id, escalation_delay, *args, **kwargs):
else:
sample = PARENT_SAMPLE
with Span(client="escalator", sample=sample) as ctx:
clear_notification = []
alarm.set_escalation_context()
# Evaluate escalation chain
mo = alarm.managed_object
......@@ -269,25 +270,39 @@ def escalate(alarm_id, escalation_id, escalation_delay, *args, **kwargs):
logger.debug("[%s] Notification message:\nSubject: %s\n%s", alarm_id, subject, body)
log("Sending notification to group %s", a.notification_group.name)
a.notification_group.notify(subject, body)
alarm.set_clear_notification(a.notification_group, a.clear_template)
notification = Notifications(
group=a.notification_group if a.notification_group else None,
template=a.clear_template if a.clear_template else None,
)
clear_notification += [notification]
metrics["escalation_notify"] += 1
#
if a.stop_processing:
logger.debug("Stopping processing")
break
alarm.set_clear_notifications(clear_notification)
nalarm = get_alarm(alarm_id)
if nalarm and nalarm.status == "C" and nalarm.escalation_tt:
log("Alarm has been closed during escalation. Try to deescalate")
metrics["escalation_closed_while_escalated"] += 1
if not nalarm.escalation_close_ts and not nalarm.escalation_close_error:
if alarm.clear_notifications:
for cn in alarm.clear_notifications:
notify_close(
alarm_id=alarm_id,
tt_id=nalarm.escalation_tt,
subject="Closing",
body="Closing",
notification_group_id=cn["group"].id if "group" in cn else None,
close_tt=alarm.close_tt,
)
else:
notify_close(
alarm_id=alarm_id,
tt_id=nalarm.escalation_tt,
subject="Closing",
body="Closing",
notification_group_id=alarm.clear_notification_group.id
if alarm.clear_notification_group
else None,
notification_group_id=None,
close_tt=alarm.close_tt,
)
logger.info("[%s] Escalations loop end", alarm_id)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment