discoveryid.py 11.1 KB
Newer Older
Dmitry Lukhtionov's avatar
Dmitry Lukhtionov committed
1
2
3
# ---------------------------------------------------------------------
# Discovery id
# ---------------------------------------------------------------------
4
# Copyright (C) 2007-2020 The NOC Project
Dmitry Lukhtionov's avatar
Dmitry Lukhtionov committed
5
6
# See LICENSE for details
# ---------------------------------------------------------------------
Dmitry Volodin's avatar
Dmitry Volodin committed
7

Dmitry Lukhtionov's avatar
Dmitry Lukhtionov committed
8
# Python modules
9
10
import operator
from threading import Lock
11
import bisect
Dmitry Volodin's avatar
Dmitry Volodin committed
12

Dmitry Lukhtionov's avatar
Dmitry Lukhtionov committed
13
# Third-party modules
14
import cachetools
Dmitry Volodin's avatar
Dmitry Volodin committed
15
from mongoengine.document import Document, EmbeddedDocument
Dmitry Volodin's avatar
Dmitry Volodin committed
16
from mongoengine.fields import StringField, ListField, LongField, EmbeddedDocumentField
17
from pymongo import ReadPreference
Dmitry Volodin's avatar
Dmitry Volodin committed
18

Dmitry Lukhtionov's avatar
Dmitry Lukhtionov committed
19
# NOC modules
kk's avatar
kk committed
20
from noc.core.mongo.fields import ForeignKeyField
Dmitry Volodin's avatar
Dmitry Volodin committed
21
from noc.sa.models.managedobject import ManagedObject
22
from noc.inv.models.interface import Interface
23
from noc.inv.models.subinterface import SubInterface
Dmitry Volodin's avatar
Dmitry Volodin committed
24
from noc.core.perf import metrics
25
26
from noc.core.cache.decorator import cachedmethod
from noc.core.cache.base import cache
27
from noc.core.mac import MAC
28
from noc.core.model.decorator import on_delete
29
30

mac_lock = Lock()
Dmitry Volodin's avatar
Dmitry Volodin committed
31
32


33
class MACRange(EmbeddedDocument):
Dmitry Volodin's avatar
Dmitry Volodin committed
34
    meta = {"strict": False, "auto_create_index": False}
35
36
37
    first_mac = StringField()
    last_mac = StringField()

Dmitry Volodin's avatar
Dmitry Volodin committed
38
    def __str__(self):
Dmitry Volodin's avatar
Dmitry Volodin committed
39
        return "%s - %s" % (self.first_mac, self.last_mac)
40
41


42
@on_delete
Dmitry Volodin's avatar
Dmitry Volodin committed
43
44
45
46
class DiscoveryID(Document):
    """
    Managed Object's discovery identity
    """
Dmitry Volodin's avatar
Dmitry Volodin committed
47

Dmitry Volodin's avatar
Dmitry Volodin committed
48
49
    meta = {
        "collection": "noc.inv.discovery_id",
50
        "strict": False,
51
        "auto_create_index": False,
52
        "indexes": ["object", "hostname", "hostname_id", "udld_id", "macs"],
Dmitry Volodin's avatar
Dmitry Volodin committed
53
54
    }
    object = ForeignKeyField(ManagedObject)
55
    chassis_mac = ListField(EmbeddedDocumentField(MACRange))
Dmitry Volodin's avatar
Dmitry Volodin committed
56
    hostname = StringField()
57
    hostname_id = StringField()
Dmitry Volodin's avatar
Dmitry Volodin committed
58
    router_id = StringField()
Dmitry Volodin's avatar
Dmitry Volodin committed
59
    udld_id = StringField()  # UDLD local identifier
60
61
    #
    macs = ListField(LongField())
Dmitry Volodin's avatar
Dmitry Volodin committed
62

63
    _mac_cache = cachetools.TTLCache(maxsize=10000, ttl=60)
64
    _udld_cache = cachetools.TTLCache(maxsize=1000, ttl=60)
65

Dmitry Volodin's avatar
Dmitry Volodin committed
66
    def __str__(self):
Dmitry Volodin's avatar
Dmitry Volodin committed
67
68
        return self.object.name

69
70
71
72
73
74
75
76
77
78
79
80
81
    @staticmethod
    def _macs_as_ints(ranges=None, additional=None):
        """
        Get all MAC addresses within ranges as integers
        :param ranges: list of dicts {first_chassis_mac: ..., last_chassis_mac: ...}
        :param additional: Optional list of additional macs
        :return: List of integers
        """
        ranges = ranges or []
        additional = additional or []
        # Apply ranges
        macs = set()
        for r in ranges:
Andrey Vertiprahov's avatar
Andrey Vertiprahov committed
82
83
            if not r:
                continue
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
            first = MAC(r["first_chassis_mac"])
            last = MAC(r["last_chassis_mac"])
            macs.update(m for m in range(int(first), int(last) + 1))
        # Append additional macs
        macs.update(int(MAC(m)) for m in additional)
        return sorted(macs)

    @staticmethod
    def _macs_to_ranges(macs):
        """
        Convert list of macs (as integers) to MACRange
        :param macs: List of integer
        :return: List of MACRange
        """
        ranges = []
        for mi in macs:
            if ranges and mi - ranges[-1][1] == 1:
                # Extend last range
                ranges[-1][1] = mi
            else:
                # New range
                ranges += [[mi, mi]]
        return [MACRange(first_mac=str(MAC(r[0])), last_mac=str(MAC(r[1]))) for r in ranges]

Dmitry Volodin's avatar
Dmitry Volodin committed
108
    @classmethod
Dmitry Volodin's avatar
Dmitry Volodin committed
109
    def submit(cls, object, chassis_mac=None, hostname=None, router_id=None, additional_macs=None):
110
111
112
113
        # Process ranges
        macs = cls._macs_as_ints(chassis_mac, additional_macs)
        ranges = cls._macs_to_ranges(macs)
        # Update database
Dmitry Volodin's avatar
Dmitry Volodin committed
114
115
        o = cls.objects.filter(object=object.id).first()
        if o:
116
            old_macs = set(m.first_mac for m in o.chassis_mac)
117
            o.chassis_mac = ranges
Dmitry Volodin's avatar
Dmitry Volodin committed
118
            o.hostname = hostname
119
            o.hostname_id = hostname.lower() if hostname else None
Dmitry Volodin's avatar
Dmitry Volodin committed
120
            o.router_id = router_id
121
122
123
            old_macs -= set(m.first_mac for m in o.chassis_mac)
            if old_macs:
                cache.delete_many(["discoveryid-mac-%s" % m for m in old_macs])
124
125
            # MAC index
            o.macs = macs
Andrey Vertiprahov's avatar
Andrey Vertiprahov committed
126
            o.save()
Dmitry Volodin's avatar
Dmitry Volodin committed
127
        else:
Dmitry Volodin's avatar
Dmitry Volodin committed
128
            cls(
129
130
131
132
133
134
                object=object,
                chassis_mac=ranges,
                hostname=hostname,
                hostname_id=hostname.lower(),
                router_id=router_id,
                macs=macs,
Dmitry Volodin's avatar
Dmitry Volodin committed
135
            ).save()
136
137

    @classmethod
Dmitry Volodin's avatar
Dmitry Volodin committed
138
139
140
    @cachedmethod(
        operator.attrgetter("_mac_cache"), key="discoveryid-mac-%s", lock=lambda _: mac_lock
    )
141
    def get_by_mac(cls, mac):
Dmitry Volodin's avatar
Dmitry Volodin committed
142
        return cls._get_collection().find_one({"macs": int(MAC(mac))}, {"_id": 0, "object": 1})
143

144
    @classmethod
Dmitry Volodin's avatar
Dmitry Volodin committed
145
    @cachetools.cachedmethod(operator.attrgetter("_udld_cache"), lock=lambda _: mac_lock)
146
    def get_by_udld_id(cls, device_id):
Dmitry Volodin's avatar
Dmitry Volodin committed
147
        return cls._get_collection().find_one({"udld_id": device_id}, {"_id": 0, "object": 1})
148

149
    @classmethod
150
    def find_object(cls, mac=None, ipv4_address=None):
151
152
        """
        Find managed object
153
154
        :param mac:
        :param ipv4_address:
155
156
157
        :param cls:
        :return: Managed object instance or None
        """
Dmitry Volodin's avatar
Dmitry Volodin committed
158

159
160
161
162
163
164
165
        def has_ip(ip, addresses):
            x = ip + "/"
            for a in addresses:
                if a.startswith(x):
                    return True
            return False

166
167
        # Find by mac
        if mac:
Dmitry Volodin's avatar
Dmitry Volodin committed
168
            metrics["discoveryid_mac_requests"] += 1
169
            r = cls.get_by_mac(mac)
170
            if r:
171
172
                return ManagedObject.get_by_id(r["object"])
        if ipv4_address:
Dmitry Volodin's avatar
Dmitry Volodin committed
173
            metrics["discoveryid_ip_requests"] += 1
174
175
176
            # Try router_id
            d = DiscoveryID.objects.filter(router_id=ipv4_address).first()
            if d:
Dmitry Volodin's avatar
Dmitry Volodin committed
177
                metrics["discoveryid_ip_routerid"] += 1
178
179
180
181
                return d.object
            # Fallback to interface addresses
            o = set(
                d["managed_object"]
Dmitry Volodin's avatar
Dmitry Volodin committed
182
183
184
185
186
187
                for d in SubInterface._get_collection()
                .with_options(read_preference=ReadPreference.SECONDARY_PREFERRED)
                .find(
                    {"ipv4_addresses": {"$gt": ipv4_address + "/", "$lt": ipv4_address + "/99"}},
                    {"_id": 0, "managed_object": 1, "ipv4_addresses": 1},
                )
188
                if has_ip(ipv4_address, d["ipv4_addresses"])
Dmitry Volodin's avatar
PEP8    
Dmitry Volodin committed
189
            )
190
            if len(o) == 1:
Dmitry Volodin's avatar
Dmitry Volodin committed
191
                metrics["discoveryid_ip_interface"] += 1
192
                return ManagedObject.get_by_id(list(o)[0])
Dmitry Volodin's avatar
Dmitry Volodin committed
193
            metrics["discoveryid_ip_failed"] += 1
194
        return None
195
196
197
198
199
200
201
202
203

    @classmethod
    def macs_for_object(cls, object):
        """
        Get MAC addresses for object
        :param cls:
        :param object:
        :return: list of (fist_mac, last_mac)
        """
204
205
206
207
208
209
210
        # Get discovered chassis id range
        o = DiscoveryID.objects.filter(object=object.id).first()
        if o and o.chassis_mac:
            c_macs = [(r.first_mac, r.last_mac) for r in o.chassis_mac]
        else:
            c_macs = []
        # Get interface macs
Dmitry Volodin's avatar
Dmitry Volodin committed
211
212
213
214
215
216
        i_macs = set(
            i.mac
            for i in Interface.objects.filter(managed_object=object.id, mac__exists=True).only(
                "mac"
            )
            if i.mac
217
218
        )
        # Enrich discovered macs with additional interface's ones
Dmitry Volodin's avatar
Dmitry Volodin committed
219
        c_macs += [(m, m) for m in i_macs if not any(1 for f, t in c_macs if f <= m <= t)]
220
        return c_macs
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245

    @classmethod
    def macs_for_objects(cls, objects_ids):
        """
        Get MAC addresses for object
        :param cls:
        :param objects_ids: Lis IDs of Managed Object Instance
        :type: list
        :return: Dictionary mac: objects
        :rtype: dict
        """
        if not objects_ids:
            return None
        if isinstance(objects_ids, list):
            objects = objects_ids
        else:
            objects = list(objects_ids)

        os = cls.objects.filter(object__in=objects)
        if not os:
            return None
        # Discovered chassis id range
        c_macs = {int(did[0][0]): did[1] for did in os.scalar("macs", "object") if did[0]}
        # c_macs = [r.macs for r in os]
        # Other interface macs
Dmitry Volodin's avatar
Dmitry Volodin committed
246
247
248
249
250
251
252
        i_macs = {
            int(MAC(i[0])): i[1]
            for i in Interface.objects.filter(managed_object__in=objects, mac__exists=True).scalar(
                "mac", "managed_object"
            )
            if i[0]
        }
253
        # Other subinterface macs (actual for DSLAM)
Dmitry Volodin's avatar
Dmitry Volodin committed
254
255
256
257
258
259
260
        si_macs = {
            int(MAC(i[0])): i[1]
            for i in SubInterface.objects.filter(
                managed_object__in=objects, mac__exists=True
            ).scalar("mac", "managed_object")
            if i[0]
        }
261
        c_macs.update(i_macs)
262
        c_macs.update(si_macs)
263
264

        return c_macs
265
266
267
268
269
270
271
272
273
274
275
276
277

    def on_delete(self):
        # Reset cache
        macs = set(m.first_mac for m in self.chassis_mac)
        if macs:
            cache.delete_many(["discoveryid-mac-%s" % m for m in macs])

    @classmethod
    def clean_for_object(cls, mo):
        if hasattr(mo, "id"):
            mo = mo.id
        for d in DiscoveryID.objects.filter(object=mo):
            d.delete()
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292

    @classmethod
    def find_objects(cls, macs):
        """
        Find objects for list of macs
        :param macs: List of MAC addresses
        :return: dict of MAC -> ManagedObject for resolved MACs
        """
        r = {}
        if not macs:
            return r
        # Build list of macs to search
        mlist = sorted(int(MAC(m)) for m in macs)
        # Search for macs
        obj_ranges = {}  # (first, last) -> mo
Dmitry Volodin's avatar
Dmitry Volodin committed
293
294
295
        for d in DiscoveryID._get_collection().find(
            {"macs": {"$in": mlist}}, {"_id": 0, "object": 1, "chassis_mac": 1}
        ):
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
            mo = ManagedObject.get_by_id(d["object"])
            if mo:
                for dd in d.get("chassis_mac", []):
                    obj_ranges[int(MAC(dd["first_mac"])), int(MAC(dd["last_mac"]))] = mo
        n = 1
        for s, e in obj_ranges:
            n += 1
        # Resolve ranges
        start = 0
        ll = len(mlist)
        for s, e in sorted(obj_ranges):
            mo = obj_ranges[s, e]
            start = bisect.bisect_left(mlist, s, start, ll)
            while start < ll and s <= mlist[start] <= e:
                r[MAC(mlist[start])] = mo
                start += 1
        return r
313

314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
    @classmethod
    def find_all_objects(cls, mac):
        """
        Find objects for mac
        :return: dict of ManagedObjects ID for resolved MAC
        """
        r = []
        if not mac:
            return r
        metrics["discoveryid_mac_requests"] += 1
        for d in DiscoveryID._get_collection().find(
            {"macs": int(MAC(mac))}, {"_id": 0, "object": 1, "chassis_mac": 1}
        ):
            mo = ManagedObject.get_by_id(d["object"])
            if mo:
                r.append(mo.id)
        return r

332
333
334
335
    @classmethod
    def update_udld_id(cls, object, local_id):
        """
        Update UDLD id if necessary
336
337
        :param object: Object for set
        :param local_id: Local UDLD id
338
339
        :return:
        """
340
        DiscoveryID._get_collection().update_one(
Dmitry Volodin's avatar
Dmitry Volodin committed
341
342
            {"object": object.id}, {"$set": {"udld_id": local_id}}, upsert=True
        )