asset.py 34.2 KB
Newer Older
Dmitry Lukhtionov's avatar
Dmitry Lukhtionov committed
1
2
3
# ---------------------------------------------------------------------
# Asset check
# ---------------------------------------------------------------------
4
# Copyright (C) 2007-2020 The NOC Project
Dmitry Lukhtionov's avatar
Dmitry Lukhtionov committed
5
6
# See LICENSE for details
# ---------------------------------------------------------------------
Dmitry Volodin's avatar
Dmitry Volodin committed
7

Dmitry Lukhtionov's avatar
Dmitry Lukhtionov committed
8
# Python modules
Dmitry Volodin's avatar
Dmitry Volodin committed
9
10
11
from collections import defaultdict
import hashlib
import base64
12
13
14
from threading import Lock
import operator
import re
Dmitry Volodin's avatar
Dmitry Volodin committed
15
from typing import Optional, List, Dict, Set, Tuple, Iterable, Any, Union
Dmitry Volodin's avatar
Dmitry Volodin committed
16

Dmitry Volodin's avatar
Dmitry Volodin committed
17
# Third-party modules
18
import cachetools
Dmitry Volodin's avatar
Dmitry Volodin committed
19

Dmitry Lukhtionov's avatar
Dmitry Lukhtionov committed
20
# NOC modules
Dmitry Volodin's avatar
Dmitry Volodin committed
21
from noc.services.discovery.jobs.base import DiscoveryCheck
22
from noc.main.models.label import Label
Dmitry Volodin's avatar
Dmitry Volodin committed
23
24
from noc.inv.models.objectmodel import ObjectModel, ConnectionRule
from noc.inv.models.object import Object, ObjectAttr
Dmitry Volodin's avatar
Dmitry Volodin committed
25
26
27
28
from noc.inv.models.vendor import Vendor
from noc.inv.models.unknownmodel import UnknownModel
from noc.inv.models.modelmapping import ModelMapping
from noc.inv.models.error import ConnectionError
29
30
31
from noc.inv.models.sensor import Sensor
from noc.inv.models.sensorprofile import SensorProfile
from noc.pm.models.measurementunits import MeasurementUnits, DEFAULT_UNITS_NAME
kk's avatar
kk committed
32
from noc.core.text import str_dict
33
from noc.core.comp import smart_bytes, smart_text
Dmitry Volodin's avatar
Dmitry Volodin committed
34
35
36
37
38
39


class AssetCheck(DiscoveryCheck):
    """
    Version discovery
    """
Dmitry Volodin's avatar
Dmitry Volodin committed
40

Dmitry Volodin's avatar
Dmitry Volodin committed
41
42
43
    name = "asset"
    required_script = "get_inventory"

44
45
46
    _serial_masks = {}
    _serial_masks_lock = Lock()

Dmitry Volodin's avatar
Dmitry Volodin committed
47
    def __init__(self, *args, **kwargs):
Dmitry Volodin's avatar
Dmitry Volodin committed
48
        super().__init__(*args, **kwargs)
Dmitry Volodin's avatar
Dmitry Volodin committed
49
50
51
52
53
54
        self.unknown_part_no: Dict[str, Set[str]] = {}  # part_no -> list of variants
        self.pn_description: Dict[str, str] = {}  # part_no -> Description
        self.vendors: Dict[str, Vendor] = {}  # code -> Vendor instance
        self.objects: List[
            Tuple[str, Union[Object, str], Dict[str, Union[int, str]], Optional[str]]
        ] = []  # [(type, object, context, serial)]
55
56
57
        self.sensors: Dict[
            Tuple[Optional[Object], str] : Dict[str, Any]
        ] = {}  # object, sensor -> sensor data
Dmitry Volodin's avatar
Dmitry Volodin committed
58
59
60
61
62
63
        self.to_disconnect: Set[
            Tuple[Object, str, Object, str]
        ] = set()  # Save processed connection. [(in_connection, object, out_connection), ... ]
        self.rule: Dict[str, List[ConnectionRule]] = defaultdict(
            list
        )  # Connection rule. type -> [rule1, ruleN]
Dmitry Volodin's avatar
Dmitry Volodin committed
64
        self.rule_context = {}
Dmitry Volodin's avatar
Dmitry Volodin committed
65
66
67
68
        self.ctx: Dict[str, Union[int, str]] = {}
        self.stack_member: Dict["Object", str] = {}  # object -> stack member numbers
        self.managed: Set[str] = set()  # Object ids
        self.unk_model: Dict[str, ObjectModel] = {}  # name -> model
Dmitry Volodin's avatar
Dmitry Volodin committed
69
70
71
72
73
74
75
76
        self.lost_and_found = self.get_lost_and_found(self.object)

    def handler(self):
        self.logger.info("Checking assets")
        result = self.object.scripts.get_inventory()
        self.find_managed()
        # Submit objects
        for o in result:
77
            self.logger.info("Submit %s", str_dict(o))
Dmitry Volodin's avatar
Dmitry Volodin committed
78
            self.submit(
Dmitry Volodin's avatar
Dmitry Volodin committed
79
80
                type=o["type"],
                number=o.get("number"),
Dmitry Volodin's avatar
Dmitry Volodin committed
81
                builtin=o["builtin"],
Dmitry Volodin's avatar
Dmitry Volodin committed
82
83
84
85
                vendor=o.get("vendor"),
                part_no=o["part_no"],
                revision=o.get("revision"),
                serial=o.get("serial"),
Dmitry Volodin's avatar
Dmitry Volodin committed
86
                mfg_date=o.get("mfg_date"),
Dmitry Volodin's avatar
Dmitry Volodin committed
87
                description=o.get("description"),
88
                sensors=o.get("sensors"),
Dmitry Volodin's avatar
Dmitry Volodin committed
89
90
91
92
93
94
95
            )
        # Assign stack members
        self.submit_stack_members()
        #
        self.submit_connections()
        #
        self.check_management()
96
97
        #
        self.disconnect_connections()
98
99
        #
        self.sync_sensors()
Dmitry Volodin's avatar
Dmitry Volodin committed
100

Dmitry Volodin's avatar
Dmitry Volodin committed
101
102
    def submit(
        self,
Dmitry Volodin's avatar
Dmitry Volodin committed
103
104
105
106
107
108
109
110
111
        type: str,
        part_no: List[str],
        number: Optional[str] = None,
        builtin: bool = False,
        vendor: Optional[str] = None,
        revision: Optional[str] = None,
        serial: Optional[str] = None,
        mfg_date: Optional[str] = None,
        description: Optional[str] = None,
112
        sensors: List[Dict[str, Any]] = None,
Dmitry Volodin's avatar
Dmitry Volodin committed
113
    ):
Dmitry Volodin's avatar
Dmitry Volodin committed
114
115
116
        # Check the vendor and the serial are sane
        # OEM transceivers return binary trash often
        if vendor:
117
            # Possible dead code
Dmitry Volodin's avatar
Dmitry Volodin committed
118
119
120
            try:
                vendor.encode("utf-8")
            except UnicodeDecodeError:
Dmitry Volodin's avatar
Dmitry Volodin committed
121
                self.logger.info("Trash submited as vendor id: %s", vendor.encode("hex"))
Dmitry Volodin's avatar
Dmitry Volodin committed
122
123
                return
        if serial:
124
            # Possible dead code
Dmitry Volodin's avatar
Dmitry Volodin committed
125
126
127
            try:
                serial.encode("utf-8")
            except UnicodeDecodeError:
Dmitry Volodin's avatar
Dmitry Volodin committed
128
                self.logger.info("Trash submited as serial: %s", serial.encode("hex"))
Dmitry Volodin's avatar
Dmitry Volodin committed
129
130
                return
        #
Dmitry Volodin's avatar
Dmitry Volodin committed
131
        is_unknown_xcvr = not builtin and part_no[0].startswith("Unknown | Transceiver | ")
Dmitry Volodin's avatar
Dmitry Volodin committed
132
133
134
135
136
137
138
139
140
        if not type and is_unknown_xcvr:
            type = "XCVR"
        # Skip builtin modules
        if builtin:
            # Adjust context anyway
            self.prepare_context(type, number)
            return  # Builtin must aways have type set
        #
        if is_unknown_xcvr:
141
            self.logger.info("%s S/N %s should be resolved later", part_no[0], serial)
Dmitry Volodin's avatar
Dmitry Volodin committed
142
            self.prepare_context(type, number)
Dmitry Volodin's avatar
Dmitry Volodin committed
143
            self.objects += [("XCVR", part_no[0], self.ctx.copy(), serial)]
Dmitry Volodin's avatar
Dmitry Volodin committed
144
145
146
147
148
149
150
151
152
153
154
155
            return
        # Cache description
        if description:
            for p in part_no:
                if p not in self.pn_description:
                    self.pn_description[p] = description
        # Find vendor
        vnd = self.get_vendor(vendor)
        if not vnd:
            # Try to resolve via model map
            m = self.get_model_map(vendor, part_no, serial)
            if not m:
Dmitry Volodin's avatar
Dmitry Volodin committed
156
                self.logger.error(
Dmitry Volodin's avatar
Dmitry Volodin committed
157
158
                    "Unknown vendor '%s' for S/N %s (%s)", vendor, serial, description
                )
Dmitry Volodin's avatar
Dmitry Volodin committed
159
160
161
162
163
164
165
166
                return
        else:
            # Find model
            m = ObjectModel.get_model(vnd, part_no)
            if not m:
                # Try to resolve via model map
                m = self.get_model_map(vendor, part_no, serial)
                if not m:
167
                    self.logger.info(
Dmitry Volodin's avatar
Dmitry Volodin committed
168
169
170
                        "Unknown model: vendor=%s, part_no=%s (%s). " "Skipping",
                        vnd.name,
                        part_no,
171
                        description,
Dmitry Volodin's avatar
Dmitry Volodin committed
172
                    )
Dmitry Volodin's avatar
Dmitry Volodin committed
173
                    self.register_unknown_part_no(vnd, part_no, description)
Dmitry Volodin's avatar
Dmitry Volodin committed
174
                    return
175
176
177
        # Sanitize serial number against the model
        serial = self.clean_serial(m, number, serial)
        #
Dmitry Volodin's avatar
Dmitry Volodin committed
178
179
        if m.cr_context and type != m.cr_context:
            # Override type with object mode's one
180
            self.logger.info("Model changes type to '%s'", m.cr_context)
Dmitry Volodin's avatar
Dmitry Volodin committed
181
182
            type = m.cr_context
        if not type:
183
            self.logger.info(
Dmitry Volodin's avatar
Dmitry Volodin committed
184
185
186
187
                "Cannot resolve type for: vendor=%s, part_no=%s (%s). " "Skipping",
                vnd.name,
                description,
                part_no,
Dmitry Volodin's avatar
Dmitry Volodin committed
188
189
190
191
192
193
194
195
196
197
198
199
            )
            return
        self.prepare_context(type, number)
        # Get connection rule
        if not self.rule and m.connection_rule:
            self.set_rule(m.connection_rule)
            # Set initial context
            if type in self.rule_context:
                scope = self.rule_context[type][0]
                if scope:
                    self.set_context(scope, number)
        # Find existing object or create new
Dmitry Volodin's avatar
Dmitry Volodin committed
200
201
202
        o: Optional["Object"] = Object.objects.filter(
            model=m.id, data__match={"interface": "asset", "attr": "serial", "value": serial}
        ).first()
Dmitry Volodin's avatar
Dmitry Volodin committed
203
204
        if not o:
            # Create new object
Dmitry Volodin's avatar
Dmitry Volodin committed
205
            self.logger.info("Creating new object. model='%s', serial='%s'", m.name, serial)
Dmitry Volodin's avatar
Dmitry Volodin committed
206
            data = [ObjectAttr(scope="", interface="asset", attr="serial", value=serial)]
Dmitry Volodin's avatar
Dmitry Volodin committed
207
            if revision:
Dmitry Volodin's avatar
Dmitry Volodin committed
208
                data += [ObjectAttr(scope="", interface="asset", attr="revision", value=revision)]
Dmitry Volodin's avatar
Dmitry Volodin committed
209
            if mfg_date:
Dmitry Volodin's avatar
Dmitry Volodin committed
210
                data += [ObjectAttr(scope="", interface="asset", attr="mfg_date", value=mfg_date)]
211
212
213
214
            if self.object.container:
                container = self.object.container.id
            else:
                container = self.lost_and_found
Dmitry Volodin's avatar
Dmitry Volodin committed
215
            o = Object(model=m, data=data, container=container)
Dmitry Volodin's avatar
Dmitry Volodin committed
216
217
218
            o.save()
            o.log(
                "Created by asset_discovery",
Dmitry Volodin's avatar
Dmitry Volodin committed
219
220
221
                system="DISCOVERY",
                managed_object=self.object,
                op="CREATE",
Dmitry Volodin's avatar
Dmitry Volodin committed
222
            )
223
224
225
226
227
        else:
            # Add all connection to disconnect list
            self.to_disconnect.update(
                set((o, c[0], c[1], c[2]) for c in o.iter_inner_connections())
            )
Dmitry Volodin's avatar
Dmitry Volodin committed
228
229
230
231
232
        # Check revision
        if o.get_data("asset", "revision") != revision:
            # Update revision
            self.logger.info(
                "Object revision changed [%s %s] %s -> %s",
Dmitry Volodin's avatar
Dmitry Volodin committed
233
234
235
236
                m.name,
                o.id,
                o.get_data("asset", "revision"),
                revision,
Dmitry Volodin's avatar
Dmitry Volodin committed
237
238
239
240
            )
            o.set_data("asset", "revision", revision)
            o.save()
            o.log(
Dmitry Volodin's avatar
Dmitry Volodin committed
241
242
243
244
                "Object revision changed: %s -> %s" % (o.get_data("asset", "revision"), revision),
                system="DISCOVERY",
                managed_object=self.object,
                op="CHANGE",
Dmitry Volodin's avatar
Dmitry Volodin committed
245
246
247
248
249
250
            )
        # Check manufacturing date
        if mfg_date and o.get_data("asset", "revision") != revision:
            # Update revision
            self.logger.info(
                "Object manufacturing date changed [%s %s] %s -> %s",
Dmitry Volodin's avatar
Dmitry Volodin committed
251
252
253
254
                m.name,
                o.id,
                o.get_data("asset", "mfg_date"),
                mfg_date,
Dmitry Volodin's avatar
Dmitry Volodin committed
255
256
257
258
            )
            o.set_data("asset", "mfg_date", mfg_date)
            o.save()
            o.log(
Dmitry Volodin's avatar
Dmitry Volodin committed
259
260
261
262
                "Object manufacturing date: %s -> %s" % (o.get_data("asset", "mfg_date"), mfg_date),
                system="DISCOVERY",
                managed_object=self.object,
                op="CHANGE",
Dmitry Volodin's avatar
Dmitry Volodin committed
263
264
265
266
            )
        # Check management
        if o.get_data("management", "managed"):
            if o.get_data("management", "managed_object") != self.object.id:
Dmitry Volodin's avatar
Dmitry Volodin committed
267
268
                self.logger.info("Changing object management to '%s'", self.object.name)
                o.set_data("management", "managed_object", self.object.id)
Dmitry Volodin's avatar
Dmitry Volodin committed
269
270
271
                o.save()
                o.log(
                    "Management granted",
Dmitry Volodin's avatar
Dmitry Volodin committed
272
273
274
                    system="DISCOVERY",
                    managed_object=self.object,
                    op="CHANGE",
Dmitry Volodin's avatar
Dmitry Volodin committed
275
276
277
278
279
                )
            self.update_name(o)
            if o.id in self.managed:
                self.managed.remove(o.id)
        self.objects += [(type, o, self.ctx.copy(), serial)]
280
281
282
283
        # Collect sensors
        if sensors:
            for s in sensors:
                self.sensors[(o, s["name"])] = s
Dmitry Volodin's avatar
Dmitry Volodin committed
284
285
286
287
        # Collect stack members
        if number and o.get_data("stack", "stackable"):
            self.stack_member[o] = number

Dmitry Volodin's avatar
Dmitry Volodin committed
288
    def prepare_context(self, type: str, number: Optional[str]):
Dmitry Volodin's avatar
Dmitry Volodin committed
289
290
291
292
293
294
295
296
        self.set_context("N", number)
        if type and type in self.rule_context:
            scope, reset_scopes = self.rule_context[type]
            if scope:
                self.set_context(scope, number)
            if reset_scopes:
                self.reset_context(reset_scopes)

Dmitry Volodin's avatar
Dmitry Volodin committed
297
    def update_name(self, object: Object):
Dmitry Volodin's avatar
Dmitry Volodin committed
298
299
300
301
302
        n = self.get_name(object, self.object)
        if n and n != object.name:
            object.name = n
            self.logger.info("Changing name to '%s'", n)
            object.save()
Dmitry Volodin's avatar
Dmitry Volodin committed
303
304
305
306
307
308
            object.log(
                "Change name to '%s'" % n,
                system="DISCOVERY",
                managed_object=self.object,
                op="CHANGE",
            )
Dmitry Volodin's avatar
Dmitry Volodin committed
309

Dmitry Volodin's avatar
Dmitry Volodin committed
310
311
312
    def iter_object(
        self, i: int, scope: str, value: int, target_type: str, fwd: bool
    ) -> Iterable[Tuple[str, Union[Object, str], Dict[str, Union[int, str]]]]:
Dmitry Volodin's avatar
Dmitry Volodin committed
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
        # Search backwards
        if not fwd:
            for j in range(i - 1, -1, -1):
                type, object, ctx, _ = self.objects[j]
                if scope in ctx and ctx[scope] == value:
                    if target_type == type:
                        yield type, object, ctx
                else:
                    break
        # Search forward
        if fwd:
            for j in range(i + 1, len(self.objects)):
                type, object, ctx, _ = self.objects[j]
                if scope in ctx and ctx[scope] == value:
                    if target_type == type:
                        yield type, object, ctx
                else:
Dmitry Volodin's avatar
Dmitry Volodin committed
330
                    return
Dmitry Volodin's avatar
Dmitry Volodin committed
331

Dmitry Volodin's avatar
Dmitry Volodin committed
332
    def expand_context(self, s: str, ctx: Dict[str, int]) -> str:
Dmitry Volodin's avatar
Dmitry Volodin committed
333
334
335
336
337
338
339
340
341
342
343
344
345
346
        """
        Replace values in context
        """
        s = s or ""
        for c in ctx:
            s = s.replace("{%s}" % c, str(ctx[c]))
        return s

    def submit_connections(self):
        # Check connection rule is set
        if not self.rule:
            return
        for i, o in enumerate(self.objects):
            type, object, context, serial = o
347
            self.logger.info("Trying to connect #%d. %s (%s)", i, type, str_dict(context))
Dmitry Volodin's avatar
Dmitry Volodin committed
348
349
350
351
352
353
354
355
356
357
358
359
360
            if type not in self.rule:
                continue
            # Find applicable rule
            for r in self.rule[type]:
                found = False
                t_n = self.expand_context(r.target_number, context)
                if r.scope.startswith("-"):
                    scope = r.scope[1:]
                    fwd = True
                else:
                    scope = r.scope
                    fwd = False
                for t_type, t_object, t_ctx in self.iter_object(
Dmitry Volodin's avatar
Dmitry Volodin committed
361
362
                    i, scope, context.get(scope), r.target_type, fwd=fwd
                ):
363
                    if isinstance(t_object, str):
Dmitry Volodin's avatar
Dmitry Volodin committed
364
365
366
                        continue
                    if not t_n or t_n == t_ctx["N"]:
                        # Check target object has proper connection
Dmitry Volodin's avatar
Dmitry Volodin committed
367
                        t_c = self.expand_context(r.target_connection, context)
Dmitry Volodin's avatar
Dmitry Volodin committed
368
369
                        if not t_object.has_connection(t_c):
                            continue
Dmitry Volodin's avatar
Dmitry Volodin committed
370
                        # Check source object has proper connection
Dmitry Volodin's avatar
Dmitry Volodin committed
371
                        m_c = self.expand_context(r.match_connection, context)
372
                        if isinstance(object, str):
Dmitry Volodin's avatar
Dmitry Volodin committed
373
                            # Resolving unknown object
Dmitry Volodin's avatar
Dmitry Volodin committed
374
                            o = self.resolve_object(object, m_c, t_object, t_c, serial)
Dmitry Volodin's avatar
Dmitry Volodin committed
375
376
377
378
379
380
381
382
                            if not o:
                                continue
                            object = o
                        if not object.has_connection(m_c):
                            continue
                        # Connect
                        self.logger.info(
                            "Connecting %s %s:%s -> %s %s:%s",
Dmitry Volodin's avatar
Dmitry Volodin committed
383
384
385
386
387
388
                            type,
                            context["N"],
                            m_c,
                            t_type,
                            t_ctx["N"],
                            t_c,
Dmitry Volodin's avatar
Dmitry Volodin committed
389
                        )
Dmitry Volodin's avatar
Dmitry Volodin committed
390
391
392
                        if object.get_data("twinax", "twinax") and m_c == object.get_data(
                            "twinax", "alias"
                        ):
Dmitry Volodin's avatar
Dmitry Volodin committed
393
394
395
396
397
398
399
400
                            self.connect_twinax(object, m_c, t_object, t_c)
                        else:
                            self.connect_p2p(object, m_c, t_object, t_c)
                        found = True
                        break
                if found:
                    break

Dmitry Volodin's avatar
Dmitry Volodin committed
401
    def connect_p2p(self, o1: Object, c1: str, o2: Object, c2: str):
Dmitry Volodin's avatar
Dmitry Volodin committed
402
403
404
405
406
407
408
409
410
411
        """
        Create P2P connection o1:c1 - o2:c2
        """
        try:
            cn = o1.connect_p2p(c1, o2, c2, {}, reconnect=True)
            if cn:
                o1.log(
                    "Connect %s -> %s:%s" % (c1, o2, c2),
                    system="DISCOVERY",
                    managed_object=self.object,
Dmitry Volodin's avatar
Dmitry Volodin committed
412
                    op="CONNECT",
Dmitry Volodin's avatar
Dmitry Volodin committed
413
414
415
416
417
                )
                o2.log(
                    "Connect %s -> %s:%s" % (c2, o1, c1),
                    system="DISCOVERY",
                    managed_object=self.object,
Dmitry Volodin's avatar
Dmitry Volodin committed
418
                    op="CONNECT",
Dmitry Volodin's avatar
Dmitry Volodin committed
419
                )
420
421
422
423
            c_name = o2.model.get_model_connection(c2)  # If internal_name use
            if (o2, c_name.name, o1, c1) in self.to_disconnect:
                # Remove if connection on system
                self.to_disconnect.remove((o2, c_name.name, o1, c1))
Dmitry Volodin's avatar
Dmitry Volodin committed
424
425
        except ConnectionError as e:
            self.logger.error("Failed to connect: %s", e)
Dmitry Volodin's avatar
Dmitry Volodin committed
426

Dmitry Volodin's avatar
Dmitry Volodin committed
427
    def connect_twinax(self, o1: Object, c1: str, o2: Object, c2: str):
Dmitry Volodin's avatar
Dmitry Volodin committed
428
429
430
431
432
        """
        Connect twinax object o1 and virtual connection c1 to o2:c2
        """
        free_connections = []
        # Resolve virtual name c1 to real connection
Dmitry Volodin's avatar
Dmitry Volodin committed
433
        r_names = [o1.get_data("twinax", "connection%d" % i) for i in range(1, 3)]
Dmitry Volodin's avatar
Dmitry Volodin committed
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
        # Check connection is already exists
        for n in r_names:
            cn, o, c = o1.get_p2p_connection(n)
            if not cn:
                free_connections += [n]
                continue
            if o.id == o2.id and c == c2:
                # Already connected
                return
        # Check twinax has free connection
        if not free_connections:
            self.logger.error("Twinax has no free connections")
            return
        # Connect first free to o2:c2
        c = free_connections[0]
449
        self.logger.info("Using twinax connection '%s' instead of '%s'", c, c1)
Dmitry Volodin's avatar
Dmitry Volodin committed
450
451
        self.connect_p2p(o1, c, o2, c2)

452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
    def sync_sensors(self):
        obj_sensors: Dict[Tuple[Optional[Object], str], Sensor] = {
            (s.object, s.local_id): s
            for s in Sensor.objects.filter(object__in=[s[0] for s in self.sensors])
        }
        for obj, sn in obj_sensors:
            si = obj_sensors[(obj, sn)]
            # @todo rename sensors, need sensor_num for deduplicate
            sf = self.sensors.get((obj, sn))
            if sf:
                # Exist
                self.update_sensor(
                    si,
                    status=sf["status"],
                    units=sf["measurement"],
                    label=sf.get("description"),
                    snmp_oid=sf.get("snmp_oid"),
                    ipmi_id=sf.get("ipmi_id"),
470
                    labels=sf.get("labels"),
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
                )
                del self.sensors[(obj, sn)]
            else:
                # Missed sensors
                si.unseen(source="asset")
        # Create new sensors
        for obj, sn in self.sensors:
            si = self.sensors[(obj, sn)]
            self.submit_sensor(
                obj=obj,
                name=sn,
                status=si["status"],
                units=si["measurement"],
                label=si.get("description"),
                snmp_oid=si.get("snmp_oid"),
                ipmi_id=si.get("ipmi_id"),
487
                labels=si.get("labels"),
488
489
490
491
492
493
494
            )

    def submit_sensor(
        self,
        obj: Object,
        name: str,
        status: bool = True,
495
        units: Optional[str] = "Scalar",
496
497
498
        label: Optional[str] = None,
        snmp_oid: Optional[str] = None,
        ipmi_id: Optional[str] = None,
499
        labels: List[str] = None,
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
    ):
        self.logger.info("[%s|%s] Creating new sensor '%s'", obj.name if obj else "-", "-", name)
        s = Sensor(
            profile=SensorProfile.get_default_profile(),
            object=obj,
            managed_object=None if obj else self.object,
            label=label,
            local_id=name,
            units=self.normalize_sensor_units(units),
        )
        # Get sensor protocol
        if snmp_oid:
            s.protocol = "snmp"
            s.snmp_oid = snmp_oid
        elif ipmi_id:
            s.protocol = "ipmi"
            s.ipmi_id = ipmi_id
        else:
            self.logger.info(
                "[%s|%s] Unknown sensor protocol '%s'",
                obj.name if obj else "-",
                "-",
                name,
            )
524
525
526
527
528
        if labels is not None:
            for ll in labels:
                Label.ensure_label(ll)
            s.labels = [ll for ll in labels if Sensor.can_set_label(ll)]
            s.extra_labels = {"sa": s.labels}
529
530
531
532
533
534
535
        s.save()
        s.seen(source="asset")

    def update_sensor(
        self,
        sensor: Sensor,
        status: bool = True,
536
        units: Optional[str] = "Scalar",
537
538
539
        label: Optional[str] = None,
        snmp_oid: Optional[str] = None,
        ipmi_id: Optional[str] = None,
540
        labels: Optional[List[str]] = None,
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
    ):
        sensor.seen(source="asset")
        if not status:
            sensor.fire_event("down")
        else:
            sensor.fire_event("up")
        units = self.normalize_sensor_units(units)
        if sensor.units != units:
            sensor.units = units
        if label and sensor.label != label:
            sensor.label = label
        # Get sensor protocol
        if snmp_oid and snmp_oid != sensor.snmp_oid:
            sensor.protocol = "snmp"
            sensor.snmp_oid = snmp_oid
        elif ipmi_id and sensor.ipmi_id != ipmi_id:
            sensor.protocol = "ipmi"
            sensor.ipmi_id = ipmi_id
559
560
561
562
563
564
565
566
567
568
569
570
        sa_labels = sensor.extra_labels.get("sa", [])
        labels = labels or []
        for ll in labels:
            if ll in sa_labels:
                continue
            self.logger.info("[%s] Ensure Sensor label: %s", sensor.id, ll)
            Label.ensure_label(ll, enable_sensor=True)
        if labels != sa_labels:
            remove_labels = set(sa_labels).difference(set(labels))
            if remove_labels:
                sensor.labels = [ll for ll in sensor.labels if ll not in remove_labels]
            sensor.extra_labels["sa"] = labels
571
572
573
574
575
576
577
578
        sensor.save()

    def normalize_sensor_units(self, units: str) -> MeasurementUnits:
        units = MeasurementUnits.get_by_name(units)
        if not units:
            units = MeasurementUnits.get_by_name(DEFAULT_UNITS_NAME)
        return units

Dmitry Volodin's avatar
Dmitry Volodin committed
579
580
581
582
583
584
585
586
587
    def submit_stack_members(self):
        if len(self.stack_member) < 2:
            return
        for o in self.stack_member:
            m = self.stack_member[o]
            if o.get_data("stack", "member") != m:
                self.logger.info("Setting stack member %s", m)
                o.set_data("stack", "member", m)
                o.save()
Dmitry Volodin's avatar
Dmitry Volodin committed
588
589
590
591
592
593
                o.log(
                    "Setting stack member %s" % m,
                    system="DISCOVERY",
                    managed_object=self.object,
                    op="CHANGE",
                )
Dmitry Volodin's avatar
Dmitry Volodin committed
594
595
596
597
598
599
600
601
602
603
604
605
606
607
                self.update_name(o)

    def send(self):
        if self.unknown_part_no:
            platform = self.object.platform
            upn = self.get_unknown_part_no()
            for pns in upn:
                # Find description
                description = "no description"
                for p in pns:
                    if p in self.pn_description:
                        description = self.pn_description[p]
                        break
                # Report error
Dmitry Volodin's avatar
Dmitry Volodin committed
608
609
610
                self.logger.error(
                    "Unknown part number for %s: %s (%s)", platform, ", ".join(pns), description
                )
Dmitry Volodin's avatar
Dmitry Volodin committed
611

Dmitry Volodin's avatar
Dmitry Volodin committed
612
613
614
    def register_unknown_part_no(
        self, vendor: "Vendor", part_no: Union[List[str], str], descripton: Optional[str]
    ):
Dmitry Volodin's avatar
Dmitry Volodin committed
615
616
617
        """
        Register missed part number
        """
Dmitry Volodin's avatar
Dmitry Volodin committed
618
        if not isinstance(part_no, list):
Dmitry Volodin's avatar
Dmitry Volodin committed
619
620
621
622
623
624
            part_no = [part_no]
        for p in part_no:
            if p not in self.unknown_part_no:
                self.unknown_part_no[p] = set()
            for pp in part_no:
                self.unknown_part_no[p].add(pp)
625
            UnknownModel.mark_unknown(vendor.code[0], self.object, p, descripton)
Dmitry Volodin's avatar
Dmitry Volodin committed
626

Dmitry Volodin's avatar
Dmitry Volodin committed
627
    def get_unknown_part_no(self) -> List[List[str]]:
Dmitry Volodin's avatar
Dmitry Volodin committed
628
629
630
631
632
633
634
635
636
637
        """
        Get list of missed part number variants
        """
        r = []
        for p in self.unknown_part_no:
            n = sorted(self.unknown_part_no[p])
            if n not in r:
                r += [n]
        return r

Dmitry Volodin's avatar
Dmitry Volodin committed
638
    def get_vendor(self, v: Optional[str]) -> Optional["Vendor"]:
Dmitry Volodin's avatar
Dmitry Volodin committed
639
640
641
        """
        Get vendor instance or None
        """
Dmitry Lukhtionov's avatar
Fix    
Dmitry Lukhtionov committed
642
        if v is None or v.startswith("OEM") or v == "None":
Dmitry Volodin's avatar
Dmitry Volodin committed
643
644
645
646
647
648
649
            v = "NONAME"
        v = v.upper()
        if v in self.vendors:
            return self.vendors[v]
        # Temporary fix
        if v == "D-LINK":
            v = "DLINK"
Dmitry Lukhtionov's avatar
Dmitry Lukhtionov committed
650
        if "INTEL" in v:
Dmitry Lukhtionov's avatar
Dmitry Lukhtionov committed
651
            v = "INTEL"
Dmitry Lukhtionov's avatar
Dmitry Lukhtionov committed
652
653
        if "FINISAR" in v:
            v = "FINISAR"
Dmitry Volodin's avatar
Dmitry Volodin committed
654
655
656
657
658
659
660
661
        o = Vendor.objects.filter(code=v).first()
        if o:
            self.vendors[v] = o
            return o
        else:
            self.vendors[v] = None
            return None

Dmitry Volodin's avatar
Dmitry Volodin committed
662
    def set_rule(self, rule: "ConnectionRule"):
Dmitry Volodin's avatar
Dmitry Volodin committed
663
664
665
666
667
668
669
670
671
672
        self.logger.debug("Setting connection rule '%s'", rule.name)
        # Compile context mappings
        self.rule_context = {}
        for ctx in rule.context:
            self.rule_context[ctx.type] = (ctx.scope, ctx.reset_scopes)
        self.logger.debug("Context mappings: %s", self.rule_context)
        # Compile rules
        for r in rule.rules:
            self.rule[r.match_type] += [r]

Dmitry Volodin's avatar
Dmitry Volodin committed
673
    def set_context(self, name: str, value: Optional[str]):
Dmitry Volodin's avatar
Dmitry Volodin committed
674
675
676
677
678
679
        self.ctx[name] = value
        n = "N%s" % name
        if n not in self.ctx:
            self.ctx[n] = 0
        else:
            self.ctx[n] += 1
Dmitry Volodin's avatar
Dmitry Volodin committed
680
        self.logger.debug("Set context %s = %s -> %s", name, value, str_dict(self.ctx))
Dmitry Volodin's avatar
Dmitry Volodin committed
681

Dmitry Volodin's avatar
Dmitry Volodin committed
682
    def reset_context(self, names: List[str]):
Dmitry Volodin's avatar
Dmitry Volodin committed
683
684
685
686
687
688
        for n in names:
            if n in self.ctx:
                del self.ctx[n]
            m = "N%s" % n
            if m in self.ctx:
                del self.ctx[m]
Dmitry Volodin's avatar
Dmitry Volodin committed
689
        self.logger.debug("Reset context scopes %s -> %s", ", ".join(names), str_dict(self.ctx))
Dmitry Volodin's avatar
Dmitry Volodin committed
690
691
692
693
694

    def find_managed(self):
        """
        Get all objects managed by managed object
        """
Dmitry Volodin's avatar
Dmitry Volodin committed
695
        self.managed = set(
Dmitry Volodin's avatar
Dmitry Volodin committed
696
697
698
699
700
701
702
            Object.objects.filter(
                data__match={
                    "interface": "management",
                    "attr": "managed_object",
                    "value": self.object.id,
                }
            ).values_list("id")
Dmitry Volodin's avatar
Dmitry Volodin committed
703
        )
Dmitry Volodin's avatar
Dmitry Volodin committed
704
705
706
707
708
709
710
711

    def check_management(self):
        """
        Unmanage all left objects
        """
        for oid in self.managed:
            o = Object.objects.filter(id=oid).first()
            if o:
Dmitry Volodin's avatar
Dmitry Volodin committed
712
                self.logger.info("Revoking management from %s %s", o.model.name, o.id)
Dmitry Volodin's avatar
Dmitry Volodin committed
713
714
715
716
                o.reset_data("management", "managed_object")
                o.save()
                o.log(
                    "Management revoked",
Dmitry Volodin's avatar
Dmitry Volodin committed
717
718
719
                    system="DISCOVERY",
                    managed_object=self.object,
                    op="CHANGE",
Dmitry Volodin's avatar
Dmitry Volodin committed
720
721
                )

Dmitry Volodin's avatar
Dmitry Volodin committed
722
723
724
    def resolve_object(
        self, name: str, m_c: str, t_object: Object, t_c: str, serial: str
    ) -> Optional["Object"]:
Dmitry Volodin's avatar
Dmitry Volodin committed
725
726
727
728
729
730
        """
        Resolve object type
        """
        # Check object is already exists
        c, object, c_name = t_object.get_p2p_connection(t_c)
        if c is not None:
Dmitry Volodin's avatar
Dmitry Volodin committed
731
            if c_name == m_c and object.get_data("asset", "serial") == serial:
Dmitry Volodin's avatar
Dmitry Volodin committed
732
733
734
735
736
737
738
739
                # Object with same serial number exists
                return object
            else:
                # Serial number/connection mismatch
                return None
        # Check connection type
        c = t_object.model.get_model_connection(t_c)
        if c is None:
Dmitry Volodin's avatar
Dmitry Volodin committed
740
            self.logger.error("Connection violation for %s SN %s", name, serial)
Dmitry Volodin's avatar
Dmitry Volodin committed
741
742
743
744
745
746
747
748
749
            return None  # ERROR
        # Transceiver formfactor
        tp = c.type.name.split(" | ")
        ff = tp[1]
        m = "NoName | Transceiver | Unknown %s" % ff
        if name != "Unknown | Transceiver | Unknown":
            mtype = name[24:].upper().replace("-", "")
            if "BASE" in mtype:
                speed, ot = mtype.split("BASE", 1)
750
                spd = {"100": "100M", "1000": "1G", "10/100/1000": "1G", "10G": "10G"}.get(speed)
Dmitry Volodin's avatar
Dmitry Volodin committed
751
752
753
                if spd:
                    m = "NoName | Transceiver | %s | %s %s" % (spd, ff, ot)
                else:
Dmitry Volodin's avatar
Dmitry Volodin committed
754
                    self.logger.error("Unknown transceiver speed: %s", speed)
Dmitry Volodin's avatar
Dmitry Volodin committed
755
756
757
758
759
760
761
762
763
764
765
766
767
768
                    m = name
            else:
                m = name
        # Add vendor suffix when necessary
        if len(tp) == 3:
            m += " | %s" % tp[2]
        #
        if m in self.unk_model:
            model = self.unk_model[m]
        else:
            model = ObjectModel.objects.filter(name=m).first()
            self.unk_model[m] = model
        if not model:
            self.logger.error("Unknown model '%s'", m)
Dmitry Volodin's avatar
Dmitry Volodin committed
769
            self.register_unknown_part_no(self.get_vendor("NONAME"), m, "%s -> %s" % (name, m))
Dmitry Volodin's avatar
Dmitry Volodin committed
770
771
            return None
        # Create object
Dmitry Volodin's avatar
Dmitry Volodin committed
772
        self.logger.info("Creating new object. model='%s', serial='%s'", m, serial)
773
774
775
776
        if self.object.container:
            container = self.object.container.id
        else:
            container = self.lost_and_found
Dmitry Volodin's avatar
Dmitry Volodin committed
777
778
779
780
781
        o = Object(
            model=model,
            data=[ObjectAttr(scope="", interface="asset", attr="serial", value=serial)],
            container=container,
        )
Dmitry Volodin's avatar
Dmitry Volodin committed
782
        o.save()
Dmitry Volodin's avatar
Dmitry Volodin committed
783
784
785
786
787
788
        o.log(
            "Created by asset_discovery",
            system="DISCOVERY",
            managed_object=self.object,
            op="CREATE",
        )
Dmitry Volodin's avatar
Dmitry Volodin committed
789
790
        return o

Dmitry Volodin's avatar
Dmitry Volodin committed
791
792
793
    def get_model_map(
        self, vendor: str, part_no: Union[List[str], str], serial: Optional[str]
    ) -> Optional["ObjectModel"]:
Dmitry Volodin's avatar
Dmitry Volodin committed
794
795
796
797
        """
        Try to resolve using model map
        """
        # Process list of part no
Dmitry Volodin's avatar
Dmitry Volodin committed
798
        if isinstance(part_no, list):
Dmitry Volodin's avatar
Dmitry Volodin committed
799
800
801
802
803
            for p in part_no:
                m = self.get_model_map(vendor, p, serial)
                if m:
                    return m
            return None
Dmitry Volodin's avatar
Dmitry Volodin committed
804
        for mm in ModelMapping.objects.filter(vendor=vendor, is_active=True):
Dmitry Volodin's avatar
Dmitry Volodin committed
805
806
807
808
            if mm.part_no and mm.part_no != part_no:
                continue
            if mm.from_serial and mm.to_serial:
                if mm.from_serial <= serial and serial <= mm.to_serial:
Dmitry Volodin's avatar
Dmitry Volodin committed
809
                    return mm.model
Dmitry Volodin's avatar
Dmitry Volodin committed
810
            else:
Dmitry Volodin's avatar
Dmitry Volodin committed
811
                self.logger.debug("Mapping %s %s %s to %s", vendor, part_no, serial, mm.model.name)
Dmitry Volodin's avatar
Dmitry Volodin committed
812
813
814
815
816
817
818
819
820
821
822
823
824
825
                return mm.model
        return None

    def get_lost_and_found(self, object):
        lfm = ObjectModel.objects.filter(name="Lost&Found").first()
        if not lfm:
            self.logger.error("Lost&Found model not found")
            return None
        lf = Object.objects.filter(model=lfm.id).first()
        if not lf:
            self.logger.error("Lost&Found not found")
            return None
        return lf.id

Dmitry Volodin's avatar
Dmitry Volodin committed
826
    def generate_serial(self, model: ObjectModel, number: Optional[str]) -> str:
Dmitry Volodin's avatar
Dmitry Volodin committed
827
828
829
        """
        Generate virtual serial number
        """
Dmitry Volodin's avatar
Dmitry Volodin committed
830
        seed = [str(self.object.id), str(model.uuid), str(number)]
Dmitry Volodin's avatar
Dmitry Volodin committed
831
832
        for k in sorted(x for x in self.ctx if not x.startswith("N")):
            seed += [k, str(self.ctx[k])]
Dmitry Volodin's avatar
Dmitry Volodin committed
833
        h = hashlib.sha256(smart_bytes(":".join(seed)))
834
        return "NOC%s" % smart_text(base64.b32encode(h.digest())[:7])
835
836

    @staticmethod
Dmitry Volodin's avatar
Dmitry Volodin committed
837
    def get_name(obj: Object, managed_object: Optional[Any] = None) -> str:
838
839
840
841
842
843
844
845
846
847
848
        """
        Generate discovered object's name
        """
        name = None
        if managed_object:
            name = managed_object.name
            sm = obj.get_data("stack", "member")
            if sm is not None:
                # Stack member
                name += "#%s" % sm
        return name
849
850
851
852
853
854

    def disconnect_connections(self):
        for o1, c1, o2, c2 in self.to_disconnect:
            self.logger.info("Disconnect: %s:%s ->X<- %s:%s", o1, c1, c2, o2)
            self.disconnect_p2p(o1, c1, c2, o2)

Dmitry Volodin's avatar
Dmitry Volodin committed
855
    def disconnect_p2p(self, o1: "Object", c1: str, c2: str, o2: "Object"):
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
        """
        Disconnect P2P connection o1:c1 - o2:c2
        """
        try:
            cn = o1.get_p2p_connection(c1)[0]
            if cn:
                o1.log(
                    "Disconnect %s -> %s:%s" % (c1, o2, c2),
                    system="DISCOVERY",
                    managed_object=self.object,
                    op="DISCONNECT",
                )
                o2.log(
                    "Disconnect %s -> %s:%s" % (c2, o1, c1),
                    system="DISCOVERY",
                    managed_object=self.object,
                    op="DISCONNECT",
                )
                cn.delete()
        except ConnectionError as e:
            self.logger.error("Failed to disconnect: %s", e)
877

Dmitry Volodin's avatar
Dmitry Volodin committed
878
    def clean_serial(self, model: "ObjectModel", number: Optional[str], serial: Optional[str]):
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
        # Empty value
        if not serial or serial == "None":
            new_serial = self.generate_serial(model, number)
            self.logger.info("Empty serial number. Generating virtual serial %s", new_serial)
            return new_serial
        # Too short value
        slen = len(serial)
        min_serial_size = model.get_data("asset", "min_serial_size")
        if min_serial_size is not None and slen < min_serial_size:
            new_serial = self.generate_serial(model, number)
            self.logger.info(
                "Invalid serial number '%s': Too short, must be %d symbols or more. "
                "Replacing with virtual serial %s",
                serial,
                min_serial_size,
                new_serial,
            )
            return new_serial
        # Too long value
        max_serial_size = model.get_data("asset", "max_serial_size")
899
        if max_serial_size is not None and slen > max_serial_size:
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
            new_serial = self.generate_serial(model, number)
            self.logger.info(
                "Invalid serial number '%s': Too long, must be %d symbols or less. "
                "Replacing with virtual serial %s",
                serial,
                max_serial_size,
                new_serial,
            )
            return new_serial
        # Regular expression
        serial_mask = model.get_data("asset", "serial_mask")
        if serial_mask:
            rx = self.get_serial_mask(serial_mask)
            if not rx.match(serial):
                new_serial = self.generate_serial(model, number)
                self.logger.info(
                    "Invalid serial number '%s': Must match mask '%s'. "
                    "Replacing with virtual serial %s",
                    serial,
                    serial_mask,
                    new_serial,
                )
                return new_serial
        return serial

    @cachetools.cachedmethod(
        operator.attrgetter("_serial_masks"), lock=operator.attrgetter("_serial_masks_lock")
    )
    def get_serial_mask(self, mask):
        """
        Compile serial mask and cache value
        :param mask:
        :return:
        """
        return re.compile(mask)