maintenance.py 13.7 KB
Newer Older
Dmitry Lukhtionov's avatar
Dmitry Lukhtionov committed
1
# ---------------------------------------------------------------------
2
# Maintenance
Dmitry Lukhtionov's avatar
Dmitry Lukhtionov committed
3
# ---------------------------------------------------------------------
4
# Copyright (C) 2007-2022 The NOC Project
Dmitry Lukhtionov's avatar
Dmitry Lukhtionov committed
5
6
# See LICENSE for details
# ---------------------------------------------------------------------
Dmitry Volodin's avatar
Dmitry Volodin committed
7

Dmitry Lukhtionov's avatar
Dmitry Lukhtionov committed
8
# Python
9
import datetime
Dmitry Volodin's avatar
Dmitry Volodin committed
10
import operator
MaksimSmile13's avatar
MaksimSmile13 committed
11
import re
Dmitry Volodin's avatar
Dmitry Volodin committed
12
from threading import Lock
13
from typing import Optional, List, Set
kk's avatar
kk committed
14

Dmitry Lukhtionov's avatar
Dmitry Lukhtionov committed
15
# Third-party modules
16
from django.db import connection as pg_connection
Dmitry Volodin's avatar
Dmitry Volodin committed
17
from mongoengine.document import Document, EmbeddedDocument
18
from mongoengine.fields import (
kk's avatar
kk committed
19
20
21
22
23
24
    StringField,
    BooleanField,
    ReferenceField,
    DateTimeField,
    ListField,
    EmbeddedDocumentField,
25
)
Dmitry Volodin's avatar
Dmitry Volodin committed
26
import cachetools
27
import orjson
kk's avatar
kk committed
28

Dmitry Lukhtionov's avatar
Dmitry Lukhtionov committed
29
# NOC modules
30
from .maintenancetype import MaintenanceType
31
from mongoengine.errors import ValidationError
Dmitry Volodin's avatar
Dmitry Volodin committed
32
from noc.sa.models.managedobject import ManagedObject
33
from noc.inv.models.networksegment import NetworkSegment
34
35
from noc.core.mongo.fields import ForeignKeyField
from noc.core.model.decorator import on_save, on_delete
36
from noc.main.models.timepattern import TimePattern
MaksimSmile13's avatar
MaksimSmile13 committed
37
from noc.main.models.template import Template
38
from noc.core.defer import call_later
MaksimSmile13's avatar
MaksimSmile13 committed
39
from noc.sa.models.administrativedomain import AdministrativeDomain
40
from noc.main.models.notificationgroup import NotificationGroup
Dmitry Volodin's avatar
Dmitry Volodin committed
41

Dmitry Volodin's avatar
Dmitry Volodin committed
42
43
id_lock = Lock()

44
45
46
47
48
49
50
# Query for remove maintenance from affected structure
SQL_REMOVE = """
  UPDATE sa_managedobject
  SET affected_maintenances = affected_maintenances - %s
  WHERE affected_maintenances ? %s
"""

Dmitry Volodin's avatar
Dmitry Volodin committed
51

52
class MaintenanceObject(EmbeddedDocument):
Dmitry Volodin's avatar
Dmitry Volodin committed
53
54
55
    object = ForeignKeyField(ManagedObject)


56
class MaintenanceSegment(EmbeddedDocument):
57
58
59
    segment = ReferenceField(NetworkSegment)


Dmitry Volodin's avatar
Dmitry Volodin committed
60
@on_save
61
@on_delete
62
class Maintenance(Document):
Dmitry Volodin's avatar
Dmitry Volodin committed
63
    meta = {
64
        "collection": "noc.maintenance",
65
        "strict": False,
66
        "auto_create_index": False,
MaksimSmile13's avatar
MaksimSmile13 committed
67
        "indexes": ["start", "stop", ("start", "is_completed"), "administrative_domain"],
kk's avatar
kk committed
68
        "legacy_collections": ["noc.maintainance"],
Dmitry Volodin's avatar
Dmitry Volodin committed
69
70
    }

71
    type = ReferenceField(MaintenanceType)
Dmitry Volodin's avatar
Dmitry Volodin committed
72
73
74
75
76
    subject = StringField(required=True)
    description = StringField()
    start = DateTimeField()
    stop = DateTimeField()
    is_completed = BooleanField(default=False)
MaksimSmile13's avatar
MaksimSmile13 committed
77
    auto_confirm = BooleanField(default=True)
MaksimSmile13's avatar
MaksimSmile13 committed
78
    template = ForeignKeyField(Template)
Dmitry Volodin's avatar
Dmitry Volodin committed
79
80
    contacts = StringField()
    suppress_alarms = BooleanField()
81
82
83
    # Escalate TT during maintenance
    escalate_managed_object = ForeignKeyField(ManagedObject)
    # Time pattern when maintenance is active
84
85
    # None - active all the time
    time_pattern = ForeignKeyField(TimePattern)
86
    # Objects declared to be affected by maintenance
87
    direct_objects = ListField(EmbeddedDocumentField(MaintenanceObject))
88
    # Segments declared to be affected by maintenance
89
    direct_segments = ListField(EmbeddedDocumentField(MaintenanceSegment))
MaksimSmile13's avatar
MaksimSmile13 committed
90
91
    # All Administrative Domain for all affected objects
    administrative_domain = ListField(ForeignKeyField(AdministrativeDomain))
92
93
94
    # Escalated TT ID in form
    # <external system name>:<external tt id>
    escalation_tt = StringField(required=False)
Dmitry Volodin's avatar
Dmitry Volodin committed
95
96
    # @todo: Attachments

Dmitry Volodin's avatar
Dmitry Volodin committed
97
98
99
100
    _id_cache = cachetools.TTLCache(maxsize=100, ttl=60)

    @classmethod
    @cachetools.cachedmethod(operator.attrgetter("_id_cache"), lock=lambda _: id_lock)
101
    def get_by_id(cls, id) -> "Maintenance":
102
        return Maintenance.objects.filter(id=id).first()
Dmitry Volodin's avatar
Dmitry Volodin committed
103

MaksimSmile13's avatar
MaksimSmile13 committed
104
105
106
107
108
    def update_affected_objects_maintenance(self):
        call_later(
            "noc.maintenance.models.maintenance.update_affected_objects",
            60,
            maintenance_id=self.id,
109
110
            start=self.start,
            stop=self.stop if self.auto_confirm else None,
MaksimSmile13's avatar
MaksimSmile13 committed
111
112
113
        )

    def auto_confirm_maintenance(self):
114
115
116
117
        st = str(self.stop)
        if "T" in st:
            st = st.replace("T", " ")
        stop = datetime.datetime.strptime(st, "%Y-%m-%d %H:%M:%S")
MaksimSmile13's avatar
MaksimSmile13 committed
118
119
120
121
122
        now = datetime.datetime.now()
        if stop > now:
            delay = (stop - now).total_seconds()
            call_later("noc.maintenance.models.maintenance.stop", delay, maintenance_id=self.id)

123
    def save(self, *args, **kwargs):
MaksimSmile13's avatar
MaksimSmile13 committed
124
125
126
        created = False
        if self._created:
            created = self._created
127
128
129
130
131
132
133
134
135
        if self.direct_objects:
            if any(o_elem.object is None for o_elem in self.direct_objects):
                raise ValidationError("Object line is Empty")
        if self.direct_segments:
            for elem in self.direct_segments:
                try:
                    elem.segment = elem.segment
                except Exception:
                    raise ValidationError("Segment line is Empty")
Dmitry Volodin's avatar
Dmitry Volodin committed
136
        super().save(*args, **kwargs)
MaksimSmile13's avatar
MaksimSmile13 committed
137
        if created and (self.direct_objects or self.direct_segments):
MaksimSmile13's avatar
MaksimSmile13 committed
138
            self.update_affected_objects_maintenance()
MaksimSmile13's avatar
MaksimSmile13 committed
139
        if self.auto_confirm:
MaksimSmile13's avatar
MaksimSmile13 committed
140
            self.auto_confirm_maintenance()
141

Dmitry Volodin's avatar
Dmitry Volodin committed
142
    def on_save(self):
143
144
145
146
147
        changed_fields = set()
        if hasattr(self, "_changed_fields"):
            changed_fields = set(self._changed_fields)
        if changed_fields.intersection(
            {"direct_objects", "direct_segments", "stop", "start", "time_pattern"}
MaksimSmile13's avatar
MaksimSmile13 committed
148
        ):
MaksimSmile13's avatar
MaksimSmile13 committed
149
            self.update_affected_objects_maintenance()
150
        if "stop" in changed_fields:
MaksimSmile13's avatar
MaksimSmile13 committed
151
            if not self.is_completed and self.auto_confirm:
MaksimSmile13's avatar
MaksimSmile13 committed
152
                self.auto_confirm_maintenance()
153
        if "is_completed" in changed_fields:
154
            self.remove_maintenance()
MaksimSmile13's avatar
MaksimSmile13 committed
155

156
        if self.escalate_managed_object:
MaksimSmile13's avatar
MaksimSmile13 committed
157
            if not self.is_completed and self.auto_confirm:
158
159
                call_later(
                    "noc.services.escalator.maintenance.start_maintenance",
160
                    delay=max(
161
                        (self.start - datetime.datetime.now()).total_seconds(),
162
163
                        60,
                    ),
164
                    scheduler="escalator",
165
                    pool=self.escalate_managed_object.escalator_shard,
kk's avatar
kk committed
166
                    maintenance_id=self.id,
167
                )
MaksimSmile13's avatar
MaksimSmile13 committed
168
169
170
171
                if self.auto_confirm:
                    call_later(
                        "noc.services.escalator.maintenance.close_maintenance",
                        delay=max(
172
                            (self.stop - datetime.datetime.now()).total_seconds(),
MaksimSmile13's avatar
MaksimSmile13 committed
173
174
175
176
177
178
179
                            60,
                        ),
                        scheduler="escalator",
                        pool=self.escalate_managed_object.escalator_shard,
                        maintenance_id=self.id,
                    )
            if self.is_completed and not self.auto_confirm:
180
181
182
                call_later(
                    "noc.services.escalator.maintenance.close_maintenance",
                    scheduler="escalator",
183
                    pool=self.escalate_managed_object.escalator_shard,
kk's avatar
kk committed
184
                    maintenance_id=self.id,
185
                )
Dmitry Volodin's avatar
Dmitry Volodin committed
186

187
188
189
190
191
    def on_delete(self):
        self.remove_maintenance()

    def remove_maintenance(self):
        with pg_connection.cursor() as cursor:
192
            cursor.execute(SQL_REMOVE, [str(self.id), str(self.id)])
193

194
    @classmethod
195
    def currently_affected(cls, objects: Optional[List[int]] = None) -> List[int]:
196
197
198
        """
        Returns a list of currently affected object ids
        """
199
        data = []
200
        now = datetime.datetime.now()
201
        for d in Maintenance._get_collection().find(
MaksimSmile13's avatar
MaksimSmile13 committed
202
203
            {"start": {"$lte": now}, "stop": {"$gte": now}, "is_completed": False},
            {"_id": 1, "time_pattern": 1},
kk's avatar
kk committed
204
        ):
205
206
207
208
209
            if d.get("time_pattern"):
                # Restrict to time pattern
                tp = TimePattern.get_by_id(d["time_pattern"])
                if tp and not tp.match(now):
                    continue
210
211
212
213
214
215
216
217
218
            data.append(str(d["_id"]))
        affected = list(
            ManagedObject.objects.filter(
                is_managed=True, affected_maintenances__has_any_keys=data
            ).values_list("id", flat=True)
        )
        if objects:
            affected = list(set(affected) & set(objects))
        return affected
MaksimSmile13's avatar
MaksimSmile13 committed
219
220


221
222
223
def update_affected_objects(
    maintenance_id, start: datetime.datetime, stop: Optional[datetime.datetime] = None
):
MaksimSmile13's avatar
MaksimSmile13 committed
224
225
226
227
    """
    Calculate and fill affected objects
    """

228
    # All affected maintenance objects
229
    mai_objects: List[int] = list(
230
231
232
233
234
        ManagedObject.objects.filter(
            is_managed=True, affected_maintenances__has_key=str(maintenance_id)
        ).values_list("id", flat=True)
    )

235
    def get_downlinks(objects: Set[int]):
MaksimSmile13's avatar
MaksimSmile13 committed
236
        # Get all additional objects which may be affected
237
238
        r = {
            mo_id
239
240
241
            for mo_id in ManagedObject.objects.filter(
                is_managed=True, uplinks__overlap=list(objects)
            ).values_list("id", flat=True)
242
243
            if mo_id not in objects
        }
MaksimSmile13's avatar
MaksimSmile13 committed
244
245
246
247
        if not r:
            return r
        # Leave only objects with all uplinks affected
        rr = set()
248
249
250
        for mo_id, uplinks in ManagedObject.objects.filter(
            is_managed=True, id__in=list(r)
        ).values_list("id", "uplinks"):
251
252
            if len([1 for u in uplinks if u in objects]) == len(uplinks):
                rr.add(mo_id)
MaksimSmile13's avatar
MaksimSmile13 committed
253
254
255
256
        return rr

    def get_segment_objects(segment):
        # Get objects belonging to segment
257
258
259
260
261
        so = set(
            ManagedObject.objects.filter(is_managed=True, segment=segment).values_list(
                "id", flat=True
            )
        )
MaksimSmile13's avatar
MaksimSmile13 committed
262
263
264
265
266
267
268
        # Get objects from underlying segments
        for ns in NetworkSegment._get_collection().find({"parent": segment}, {"_id": 1}):
            so |= get_segment_objects(ns["_id"])
        return so

    data = Maintenance.get_by_id(maintenance_id)
    # Calculate affected objects
269
    affected: Set[int] = set(o.object.id for o in data.direct_objects if o.object)
MaksimSmile13's avatar
MaksimSmile13 committed
270
271
272
273
274
275
276
277
278
279
280
    for o in data.direct_segments:
        if o.segment:
            affected |= get_segment_objects(o.segment.id)
    while True:
        r = get_downlinks(affected)
        if not r:
            break
        affected |= r
    # Calculate affected administrative_domain
    affected_ad = list(
        set(
281
            ManagedObject.objects.filter(is_managed=True, id__in=list(affected)).values_list(
MaksimSmile13's avatar
MaksimSmile13 committed
282
283
284
285
286
287
                "administrative_domain__id", flat=True
            )
        )
    )

    # @todo: Calculate affected objects considering topology
288
289
    Maintenance._get_collection().update_one(
        {"_id": data.id},
290
291
        {"$set": {"administrative_domain": affected_ad}},
    )
292
293
294
    affected_data = {"start": start, "stop": stop}
    if data.time_pattern:
        affected_data["time_pattern"] = data.time_pattern.id
295
296
297
298
299
300
301
302
303
304
    with pg_connection.cursor() as cursor:
        # Cleanup Maintenance objects
        cursor.execute(SQL_REMOVE, [str(maintenance_id), str(maintenance_id)])
        # Add Maintenance objects
        SQL_ADD = """UPDATE sa_managedobject
        SET affected_maintenances = affected_maintenances || %s::jsonb
        WHERE id = ANY(%s::int[])"""
        cursor.execute(
            SQL_ADD,
            [
305
                orjson.dumps({str(maintenance_id): affected_data}).decode("utf-8"),
306
307
308
309
310
311
                list(affected),
            ],
        )
    # Clear cache
    for mo_id in set(mai_objects).union(affected):
        ManagedObject._reset_caches(mo_id)
312
    # Check id objects not in affected
313
    # nin_mai = set(affected).difference(set(mai_objects))
314
    # Check id objects for delete
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
    # in_mai = set(mai_objects).difference(set(affected))

    # if len(nin_mai) != 0 or len(in_mai) != 0:
    #     with pg_connection.cursor() as cursor:
    #         # Add Maintenance objects
    #         if len(nin_mai) != 0:
    #             SQL_ADD = """UPDATE sa_managedobject
    #             SET affected_maintenances = jsonb_insert(affected_maintenances,
    #             '{"%s"}', '{"start": "%s", "stop": "%s"}'::jsonb)
    #             WHERE id IN %s;""" % (
    #                 str(maintenance_id),
    #                 start,
    #                 stop,
    #                 "(%s)" % ", ".join(map(repr, nin_mai)),
    #             )
    #             cursor.execute(SQL_ADD)
    #         # Delete Maintenance objects
    #         if len(in_mai) != 0:
    #             SQL_REMOVE = """UPDATE sa_managedobject
    #                  SET affected_maintenances = affected_maintenances #- '{%s}'
    #                  WHERE id IN %s AND affected_maintenances @> '{"%s": {}}';""" % (
    #                 str(maintenance_id),
    #                 "(%s)" % ", ".join(map(repr, in_mai)),
    #                 str(maintenance_id),
    #             )
    #             cursor.execute(SQL_REMOVE)
MaksimSmile13's avatar
MaksimSmile13 committed
341
342
343
344
345
346


def stop(maintenance_id):
    rx_mail = re.compile(r"(?P<mail>[A-Za-z0-9\.\_\-]+\@[A-Za-z0-9\@\.\_\-]+)", re.MULTILINE)
    # Find Active Maintenance
    mai = Maintenance.get_by_id(maintenance_id)
347
348
    if not mai:
        return
MaksimSmile13's avatar
MaksimSmile13 committed
349
350
351
352
353
354
355
356
357
358
    mai.is_completed = True
    # Find email addresses on Maintenance Contacts
    if mai.template:
        ctx = {"maintenance": mai}
        contacts = rx_mail.findall(mai.contacts)
        if contacts:
            # Create message
            subject = mai.template.render_subject(**ctx)
            body = mai.template.render_body(**ctx)
            for mail in contacts:
359
360
361
362
363
364
365
                nf = NotificationGroup()
                nf.send_notification(
                    "mail",
                    mail,
                    subject,
                    body,
                )
MaksimSmile13's avatar
MaksimSmile13 committed
366
    Maintenance._get_collection().update({"_id": maintenance_id}, {"$set": {"is_completed": True}})
367
368
369
370
371
    mai_objects: List[int] = list(
        ManagedObject.objects.filter(
            is_managed=True, affected_maintenances__has_key=str(maintenance_id)
        ).values_list("id", flat=True)
    )
372
    with pg_connection.cursor() as cursor:
373
        cursor.execute(SQL_REMOVE, [str(maintenance_id), str(maintenance_id)])
374
375
376
    # Clear cache
    for mo_id in mai_objects:
        ManagedObject._reset_caches(mo_id)