Commit 08dc3eb4 authored by MaksimSmile13's avatar MaksimSmile13 Committed by Andrey Vertiprahov
Browse files

Update Notification for escalation when use two or more groups notifications

parent 62514157
Pipeline #15338 failed with stages
in 1 minute and 55 seconds
...@@ -34,8 +34,7 @@ from mongoengine.errors import SaveConditionError ...@@ -34,8 +34,7 @@ from mongoengine.errors import SaveConditionError
from noc.core.mongo.fields import ForeignKeyField, PlainReferenceField from noc.core.mongo.fields import ForeignKeyField, PlainReferenceField
from noc.aaa.models.user import User from noc.aaa.models.user import User
from noc.main.models.style import Style from noc.main.models.style import Style
from noc.main.models.notificationgroup import NotificationGroup from noc.main.models.notificationgroup import Notifications
from noc.main.models.template import Template
from noc.sa.models.managedobject import ManagedObject from noc.sa.models.managedobject import ManagedObject
from noc.sa.models.servicesummary import ServiceSummary, SummaryItem, ObjectSummaryItem from noc.sa.models.servicesummary import ServiceSummary, SummaryItem, ObjectSummaryItem
from noc.core.datastream.decorator import datastream from noc.core.datastream.decorator import datastream
...@@ -123,8 +122,7 @@ class ActiveAlarm(Document): ...@@ -123,8 +122,7 @@ class ActiveAlarm(Document):
total_services = ListField(EmbeddedDocumentField(SummaryItem)) total_services = ListField(EmbeddedDocumentField(SummaryItem))
total_subscribers = ListField(EmbeddedDocumentField(SummaryItem)) total_subscribers = ListField(EmbeddedDocumentField(SummaryItem))
# Template and notification group to send close notification # Template and notification group to send close notification
clear_template = ForeignKeyField(Template, required=False) clear_notifications = ListField(EmbeddedDocumentField(Notifications))
clear_notification_group = ForeignKeyField(NotificationGroup, required=False)
# Paths # Paths
adm_path = ListField(IntField()) adm_path = ListField(IntField())
segment_path = ListField(ObjectIdField()) segment_path = ListField(ObjectIdField())
...@@ -316,28 +314,40 @@ class ActiveAlarm(Document): ...@@ -316,28 +314,40 @@ class ActiveAlarm(Document):
self.delete() self.delete()
# Close TT # Close TT
# MUST be after .delete() to prevent race conditions # MUST be after .delete() to prevent race conditions
if a.escalation_tt or self.clear_template: if a.escalation_tt or self.clear_notifications:
if self.clear_template: subject = "Alarm cleared"
ctx = {"alarm": a} body = "Alarm has been cleared"
subject = self.clear_template.render_subject(**ctx) if self.clear_notifications:
body = self.clear_template.render_body(**ctx) for cn in self.clear_notifications:
ctx = {"alarm": a}
if "template" in cn:
subject = cn["template"].render_subject(**ctx)
body = cn["template"].render_body(**ctx)
call_later(
"noc.services.escalator.escalation.notify_close",
scheduler="escalator",
pool=self.managed_object.escalator_shard,
max_runs=config.fm.alarm_close_retries,
alarm_id=self.id,
tt_id=self.escalation_tt,
subject=subject,
body=body,
notification_group_id=cn["group"].id if "group" in cn else None,
close_tt=self.close_tt,
)
else: else:
subject = "Alarm cleared" call_later(
body = "Alarm has been cleared" "noc.services.escalator.escalation.notify_close",
call_later( scheduler="escalator",
"noc.services.escalator.escalation.notify_close", pool=self.managed_object.escalator_shard,
scheduler="escalator", max_runs=config.fm.alarm_close_retries,
pool=self.managed_object.escalator_shard, alarm_id=self.id,
max_runs=config.fm.alarm_close_retries, tt_id=self.escalation_tt,
alarm_id=self.id, subject=subject,
tt_id=self.escalation_tt, body=body,
subject=subject, notification_group_id=None,
body=body, close_tt=self.close_tt,
notification_group_id=self.clear_notification_group.id )
if self.clear_notification_group
else None,
close_tt=self.close_tt,
)
# Gather diagnostics # Gather diagnostics
AlarmDiagnosticConfig.on_clear(a) AlarmDiagnosticConfig.on_clear(a)
# Return archived # Return archived
...@@ -789,9 +799,8 @@ class ActiveAlarm(Document): ...@@ -789,9 +799,8 @@ class ActiveAlarm(Document):
{"_id": self.id}, {"$set": {"escalation_ctx": current_context}} {"_id": self.id}, {"$set": {"escalation_ctx": current_context}}
) )
def set_clear_notification(self, notification_group, template): def set_clear_notifications(self, clear_notifications):
self.clear_notification_group = notification_group self.clear_notifications = clear_notifications
self.clear_template = template
self.safe_save(save_condition={"managed_object": {"$exists": True}, "id": self.id}) self.safe_save(save_condition={"managed_object": {"$exists": True}, "id": self.id})
def iter_consequences(self): def iter_consequences(self):
......
...@@ -16,17 +16,21 @@ from threading import Lock ...@@ -16,17 +16,21 @@ from threading import Lock
# Third-party modules # Third-party modules
import six import six
from django.db import models from django.db import models
from mongoengine.document import EmbeddedDocument
import cachetools import cachetools
# NOC modules # NOC modules
from noc.core.model.base import NOCModel from noc.core.model.base import NOCModel
from noc.aaa.models.user import User from noc.aaa.models.user import User
from noc.settings import LANGUAGE_CODE from noc.settings import LANGUAGE_CODE
from noc.lib.nosql import ForeignKeyField
from noc.lib.timepattern import TimePatternList from noc.lib.timepattern import TimePatternList
from noc.core.service.pub import pub from noc.core.service.pub import pub
from noc.core.model.decorator import on_delete_check from noc.core.model.decorator import on_delete_check
from noc.main.models.template import Template
from .timepattern import TimePattern from .timepattern import TimePattern
id_lock = Lock() id_lock = Lock()
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
...@@ -265,3 +269,17 @@ class NotificationGroupOther(NOCModel): ...@@ -265,3 +269,17 @@ class NotificationGroupOther(NOCModel):
self.notification_method, self.notification_method,
self.params, self.params,
) )
class Notifications(EmbeddedDocument):
meta = {
"strict": False
}
group = ForeignKeyField(NotificationGroup, required=False)
template = ForeignKeyField(Template, required=False)
def __unicode__(self):
return u"%s : %s" % (self.group, self.template)
def __eq__(self, other):
return (self.group == other.group and self.template == other.template)
...@@ -23,7 +23,7 @@ from noc.sa.models.managedobjectprofile import ManagedObjectProfile ...@@ -23,7 +23,7 @@ from noc.sa.models.managedobjectprofile import ManagedObjectProfile
from noc.fm.models.activealarm import ActiveAlarm from noc.fm.models.activealarm import ActiveAlarm
from noc.fm.models.archivedalarm import ArchivedAlarm from noc.fm.models.archivedalarm import ArchivedAlarm
from noc.core.perf import metrics from noc.core.perf import metrics
from noc.main.models.notificationgroup import NotificationGroup from noc.main.models.notificationgroup import NotificationGroup, Notifications
from noc.maintenance.models.maintenance import Maintenance from noc.maintenance.models.maintenance import Maintenance
from noc.config import config from noc.config import config
from noc.core.tt.error import TTError, TemporaryTTError from noc.core.tt.error import TTError, TemporaryTTError
...@@ -87,6 +87,7 @@ def escalate(alarm_id, escalation_id, escalation_delay, *args, **kwargs): ...@@ -87,6 +87,7 @@ def escalate(alarm_id, escalation_id, escalation_delay, *args, **kwargs):
else: else:
sample = PARENT_SAMPLE sample = PARENT_SAMPLE
with Span(client="escalator", sample=sample) as ctx: with Span(client="escalator", sample=sample) as ctx:
clear_notification = []
alarm.set_escalation_context() alarm.set_escalation_context()
# Evaluate escalation chain # Evaluate escalation chain
mo = alarm.managed_object mo = alarm.managed_object
...@@ -269,27 +270,41 @@ def escalate(alarm_id, escalation_id, escalation_delay, *args, **kwargs): ...@@ -269,27 +270,41 @@ def escalate(alarm_id, escalation_id, escalation_delay, *args, **kwargs):
logger.debug("[%s] Notification message:\nSubject: %s\n%s", alarm_id, subject, body) logger.debug("[%s] Notification message:\nSubject: %s\n%s", alarm_id, subject, body)
log("Sending notification to group %s", a.notification_group.name) log("Sending notification to group %s", a.notification_group.name)
a.notification_group.notify(subject, body) a.notification_group.notify(subject, body)
alarm.set_clear_notification(a.notification_group, a.clear_template) notification = Notifications(
group=a.notification_group if a.notification_group else None,
template=a.clear_template if a.clear_template else None,
)
clear_notification += [notification]
metrics["escalation_notify"] += 1 metrics["escalation_notify"] += 1
# #
if a.stop_processing: if a.stop_processing:
logger.debug("Stopping processing") logger.debug("Stopping processing")
break break
alarm.set_clear_notifications(clear_notification)
nalarm = get_alarm(alarm_id) nalarm = get_alarm(alarm_id)
if nalarm and nalarm.status == "C" and nalarm.escalation_tt: if nalarm and nalarm.status == "C" and nalarm.escalation_tt:
log("Alarm has been closed during escalation. Try to deescalate") log("Alarm has been closed during escalation. Try to deescalate")
metrics["escalation_closed_while_escalated"] += 1 metrics["escalation_closed_while_escalated"] += 1
if not nalarm.escalation_close_ts and not nalarm.escalation_close_error: if not nalarm.escalation_close_ts and not nalarm.escalation_close_error:
notify_close( if alarm.clear_notifications:
alarm_id=alarm_id, for cn in alarm.clear_notifications:
tt_id=nalarm.escalation_tt, notify_close(
subject="Closing", alarm_id=alarm_id,
body="Closing", tt_id=nalarm.escalation_tt,
notification_group_id=alarm.clear_notification_group.id subject="Closing",
if alarm.clear_notification_group body="Closing",
else None, notification_group_id=cn["group"].id if "group" in cn else None,
close_tt=alarm.close_tt, close_tt=alarm.close_tt,
) )
else:
notify_close(
alarm_id=alarm_id,
tt_id=nalarm.escalation_tt,
subject="Closing",
body="Closing",
notification_group_id=None,
close_tt=alarm.close_tt,
)
logger.info("[%s] Escalations loop end", alarm_id) logger.info("[%s] Escalations loop end", alarm_id)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment