Merge 'upstream/master' into arthur-zzz-master-patch-74217

collections/fm.alarmclasses/Multicast/Frequency/Overflow.json

+{
+    "name": "Multicast | Frequency | Overflow",
+    "$collection": "fm.alarmclasses",
+    "uuid": "88d36558-6e26-4b29-a67c-2f07f75ff6dd",
+    "is_unique": true,
+    "discriminator": ["interface"],
+    "user_clearable": true,
+    "default_severity__name": "MINOR",
+    "datasources": [
+        {
+            "name": "InterfaceDS",
+            "datasource": "inv.InterfaceDS",
+            "search": {
+                "interface": "interface",
+                "managed_object": "managed_object"
+            }
+        }
+    ],
+    "vars": [
+        {
+            "name": "interface",
+            "description": "interface name"
+        },
+        {
+            "name": "description",
+            "description": "Interface description",
+            "default": "=InterfaceDS.description"
+        }
+    ],
+    "subject_template": "RF {{ description }} Frequency {{interface}}MHz Bit rate  Overflow",
+    "body_template": "RF {{ description }} Frequency {{interface}}MHz Bit rate  Overflow",
+    "symptoms": "",
+    "probable_causes": "",
+    "recommended_actions": "",
+    "recover_time": 300
+}

collections/fm.eventclasses/Multicast/Frequency/Overflow.json

+{
+    "name": "Multicast | Frequency | Overflow",
+    "$collection": "fm.eventclasses",
+    "uuid": "4c298520-d276-4d46-adda-47ba790f9b43",
+    "description": "Frequency Overflow",
+    "action": "A",
+    "vars": [
+        {
+            "name": "interface",
+            "description": "Interface name",
+            "type": "interface_name",
+            "required": true,
+            "match_suppress": true
+        }
+    ],
+    "link_event": true,
+    "deduplication_window": 3,
+    "suppression_window": 0,
+    "ttl": 86400,
+    "subject_template": "Frequency {{interface}}MHz Bit rate  Overflow",
+    "body_template": "Frequency {{interface}}MHz Bit rate  Overflow",
+    "symptoms": "",
+    "probable_causes": "",
+    "recommended_actions": "",
+    "disposition": [
+        {
+            "name": "dispose",
+            "condition": "True",
+            "action": "raise",
+            "alarm_class__name": "Multicast | Frequency | Overflow"
+        }
+    ]
+}

collections/fm.eventclasses/Multicast/Frequency/Overflow_Resume.json

+{
+    "name": "Multicast | Frequency | Overflow Resume",
+    "$collection": "fm.eventclasses",
+    "uuid": "64643558-f3c1-4519-a5b9-4f0189c05956",
+    "description": "Frequency Overflow",
+    "action": "A",
+    "vars": [
+        {
+            "name": "interface",
+            "description": "Interface name",
+            "type": "interface_name",
+            "required": true,
+            "match_suppress": true
+        }
+    ],
+    "link_event": true,
+    "deduplication_window": 3,
+    "suppression_window": 0,
+    "ttl": 86400,
+    "subject_template": "Frequency {{interface}}MHz Bit rate Overflow Resume",
+    "body_template": "Frequency {{interface}}MHz Bit rate Overflow Resume",
+    "symptoms": "",
+    "probable_causes": "",
+    "recommended_actions": "",
+    "disposition": [
+        {
+            "name": "dispose",
+            "condition": "True",
+            "action": "clear",
+            "alarm_class__name": "Multicast | Frequency | Overflow"
+        }
+    ]
+}

collections/fm.eventclassificationrules/Sumavision/IPQAM/Multicast/Frequency/Overflow_Resume_SNMP.json

+{
+    "name": "Sumavision | IPQAM | Multicast | Frequency | Overflow Resume(SNMP)",
+    "$collection": "fm.eventclassificationrules",
+    "uuid": "40435316-32bb-4402-bbf9-fde6d82823cd",
+    "description": "SNMPv2-SMI::enterprises.32285.2.2.2",
+    "event_class__name": "Multicast | Frequency | Overflow Resume",
+    "preference": 1000,
+    "vars": [
+        {
+            "name": "interface",
+            "value": "='%s.%s' % (f1, f2)"
+        }
+    ],
+    "patterns": [
+        {
+            "key_re": "^1.3.6.1.4.1.32285.2.1.2.1.3.\\d+$",
+            "value_re": "^MainBoard RF 1 Frequency (?P<f1>\\d+)\\s+.\\s+(?P<f2>\\d+)MHz Bit rate Overflow Resume$"
+        },
+        {
+            "key_re": "^SNMPv2-MIB::snmpTrapOID.0$",
+            "value_re": "^SNMPv2-SMI::enterprises.32285.2.2.2$"
+        },
+        {
+            "key_re": "^profile$",
+            "value_re": "^Sumavision.IPQAM$"
+        },
+        {
+            "key_re": "^source$",
+            "value_re": "^SNMP Trap$"
+        }
+    ]
+}
\ No newline at end of file

collections/fm.eventclassificationrules/Sumavision/IPQAM/Multicast/Frequency/Overflow_SNMP.json

+{
+    "name": "Sumavision | IPQAM | Multicast | Frequency | Overflow (SNMP)",
+    "$collection": "fm.eventclassificationrules",
+    "uuid": "99c5c748-e05f-4e0b-839e-2637042492fe",
+    "description": "SNMPv2-SMI::enterprises.32285.2.2.2",
+    "event_class__name": "Multicast | Frequency | Overflow",
+    "preference": 1000,
+    "vars": [
+        {
+            "name": "interface",
+            "value": "='%s.%s' % (f1, f2)"
+        }
+    ],
+    "patterns": [
+        {
+            "key_re": "^1.3.6.1.4.1.32285.2.1.2.1.3.\\d+$",
+            "value_re": "^MainBoard RF 1 Frequency (?P<f1>\\d+)\\s+.\\s+(?P<f2>\\d+)MHz Bit rate Overflow=\\d+$"
+        },
+        {
+            "key_re": "^SNMPv2-MIB::snmpTrapOID.0$",
+            "value_re": "^SNMPv2-SMI::enterprises.32285.2.2.2$"
+        },
+        {
+            "key_re": "^profile$",
+            "value_re": "^Sumavision.IPQAM$"
+        },
+        {
+            "key_re": "^source$",
+            "value_re": "^SNMP Trap$"
+        }
+    ]
+}
\ No newline at end of file

collections/fm.eventclassificationrules/Sumavision/IPQAM/Network/Link/Link_Down_SNMP.json

+{
+    "name": "Sumavision | IPQAM | Network | Link | Link Down (SNMP)",
+    "$collection": "fm.eventclassificationrules",
+    "uuid": "2462d758-5389-461d-8cdd-aa26f302ae51",
+    "description": "SNMPv2-SMI::enterprises.32285.2.2.2",
+    "event_class__name": "Network | Link | Link Down",
+    "preference": 1000,
+    "vars": [
+        {
+            "name": "interface",
+            "value": "='%s%s' % (if1, if2)"
+        }
+    ],
+    "patterns": [
+        {
+            "key_re": "^source$",
+            "value_re": "^SNMP Trap$"
+        },
+        {
+            "key_re": "^profile$",
+            "value_re": "^Sumavision.IPQAM$"
+        },
+        {
+            "key_re": "^SNMPv2-MIB::snmpTrapOID.0$",
+            "value_re": "^SNMPv2-SMI::enterprises.32285.2.2.2$"
+        },
+        {
+            "key_re": "^1.3.6.1.4.1.32285.2.1.2.1.3.\\d+$",
+            "value_re": "^MainBoard (?P<if1>\\S+)\\s+(?P<if2>\\d+)\\s+SFP Not Found=\\d+$"
+        }
+    ]
+}
\ No newline at end of file

collections/fm.eventclassificationrules/Sumavision/IPQAM/Network/Link/Link_Up_SNMP.json

+{
+    "name": "Sumavision | IPQAM | Network | Link | Link Up (SNMP)",
+    "$collection": "fm.eventclassificationrules",
+    "uuid": "f65e1a04-84b0-47be-83f6-faba469bbcd0",
+    "description": "SNMPv2-SMI::enterprises.32285.2.2.2",
+    "event_class__name": "Network | Link | Link Up",
+    "preference": 1000,
+    "vars": [
+        {
+            "name": "interface",
+            "value": "='%s%s' % (if1, if2)"
+        }
+    ],
+    "patterns": [
+        {
+            "key_re": "^source$",
+            "value_re": "^SNMP Trap$"
+        },
+        {
+            "key_re": "^profile$",
+            "value_re": "^Sumavision.IPQAM$"
+        },
+        {
+            "key_re": "^SNMPv2-MIB::snmpTrapOID.0$",
+            "value_re": "^SNMPv2-SMI::enterprises.32285.2.2.2$"
+        },
+        {
+            "key_re": "^1.3.6.1.4.1.32285.2.1.2.1.3.\\d+$",
+            "value_re": "^MainBoard (?P<if1>\\S+)\\s+(?P<if2>\\d+)\\s+SFP Resume\\d+$"
+        }
+    ]
+}
\ No newline at end of file

collections/inv.objectmodels/Huawei/CloudEngine/PAC600S12-CB.json

+{
+    "name": "Huawei | CloudEngine | PAC600S12-CB",
+    "$collection": "inv.objectmodels",
+    "uuid": "1c3a4369-a6b7-4798-a0fd-1a7f325167ab",
+    "vendor__code": "HUAWEI",
+    "description": "Huawei CloudEngine AC Power Supply Unit PAC600S12-CB",
+    "cr_context": "PWR",
+    "tags": ["psu"],
+    "connections": [
+        {
+            "description": "PSU slot",
+            "direction": "o",
+            "gender": "m",
+            "name": "in",
+            "type__name": "Huawei | CloudEngine | PSU"
+        },
+        {
+            "description": "Power inlet",
+            "direction": "s",
+            "gender": "m",
+            "name": "power",
+            "protocols": [">220VAC", ">110VAC"],
+            "type__name": "Power | IEC 60320 C14"
+        }
+    ],
+    "data": {
+        "asset": {
+            "part_no": ["PAC600S12-CB"]
+        },
+        "power": {
+            "power": 600.0
+        }
+    }
+}

commands/inject-event.py

@@ -12,13 +12,13 @@ import json
 import argparse
 
 # Third-party modules
-import bson
+import orjson
 
 # NOC modules
 from noc.core.management.base import BaseCommand
 from noc.core.mongo.connection import connect
 from noc.sa.models.managedobject import ManagedObject
-from noc.core.nsq.pub import nsq_pub
+from noc.core.service.pub import publish
 
 
 class Command(BaseCommand):
@@ -62,8 +62,8 @@ class Command(BaseCommand):
                 data = json.load(f)
             except ValueError as e:
                 self.die('Failed to decode JSON file "%s": %s' % (path, str(e)))
+        stream, partition = obj.events_stream_and_partition
         # Load events
-        topic = "events.%s" % obj.pool.name
         for e in data:
             if e["profile"] != obj.profile.name:
                 self.stdout.write(
@@ -71,21 +71,20 @@ class Command(BaseCommand):
                     % (path, obj.profile.name, e["profile"], e)
                 )
                 continue
+            raw_vars = {"collector": obj.pool.name}
+            raw_vars.update(e["raw_vars"])
             msg = {
-                "id": str(bson.ObjectId()),
                 "ts": time.time(),
                 "object": obj.id,
-                "data": e["raw_vars"],
+                "data": raw_vars,
             }
-            nsq_pub(topic, msg)
-            self.stdout.write(msg["id"])
+            publish(orjson.dumps(msg), stream, partition=partition)
 
     def syslog_message(self, obj, msg):
-        topic = "events.%s" % obj.pool.name
+        stream, partition = obj.events_stream_and_partition
         raw_vars = {"source": "syslog", "facility": "23", "severity": "6", "message": msg}
-        msg = {"id": str(bson.ObjectId()), "ts": time.time(), "object": obj.id, "data": raw_vars}
-        nsq_pub(topic, msg)
-        self.stdout.write(msg["id"])
+        msg = {"ts": time.time(), "object": obj.id, "data": raw_vars}
+        publish(orjson.dumps(msg), stream, partition=partition)
 
 
 if __name__ == "__main__":

core/etl/loader/loader.py

@@ -12,6 +12,7 @@ import threading
 
 # NOC modules
 from .base import BaseLoader
+from noc.config import config
 
 logger = logging.getLogger(__name__)
 
@@ -31,22 +32,26 @@ class LoaderLoader(object):
             loader = self.loaders.get(name)
             if not loader:
                 logger.info("Loading loader %s", name)
-                module_name = "noc.core.etl.loader.%s" % name
-                try:
-                    sm = __import__(module_name, {}, {}, "*")
-                    for n in dir(sm):
-                        o = getattr(sm, n)
-                        if (
-                            inspect.isclass(o)
-                            and issubclass(o, BaseLoader)
-                            and o.__module__ == sm.__name__
-                        ):
-                            loader = o
-                            break
-                        logger.error("Loader not found: %s", name)
-                except Exception as e:
-                    logger.error("Failed to load loader %s: %s", name, e)
-                    loader = None
+                for p in config.get_customized_paths("", prefer_custom=True):
+                    base = "noc.custom" if p else "noc.core"
+                    module_name = "%s.etl.loader.%s" % (base, name)
+                    try:
+                        sm = __import__(module_name, {}, {}, "*")
+                        for n in dir(sm):
+                            o = getattr(sm, n)
+                            if (
+                                inspect.isclass(o)
+                                and issubclass(o, BaseLoader)
+                                and o.__module__ == sm.__name__
+                            ):
+                                loader = o
+                                break
+                            logger.error("Loader not found: %s", name)
+                    except Exception as e:
+                        logger.error("Failed to load loader %s: %s", name, e)
+                        loader = None
+                    if loader:
+                        break
                 self.loaders[name] = loader
             return loader
 

docker/docker-compose-infra.yml

@@ -112,7 +112,7 @@ services:
         -storageDataPath=/storage
         -httpListenAddr=:8428
         -envflag.enable
-        -envflag.COMPOSEPREFIX=vm_
+        -envflag.prefix=vm_
     logging:
       driver: ${COMPOSE_LOG_DRIVER}
       options:
@@ -136,7 +136,7 @@ services:
       -datasource.url=http://vm:8428
       -notifier.url=http://alertmanager:9093
       -remoteWrite.url=http://vm:8428
-      -envflag.COMPOSEPREFIX=vmalert_
+      -envflag.prefix=vmalert_
       -envflag.enable
     logging:
       driver: ${COMPOSE_LOG_DRIVER}
@@ -160,7 +160,7 @@ services:
         -promscrape.config=/etc/prometheus/prometheus.yml
         -envflag.enable
         -remoteWrite.url=http://vm:8428/api/v1/write
-        -envflag.COMPOSEPREFIX=vmagent_
+        -envflag.prefix=vmagent_
     logging:
       driver: ${COMPOSE_LOG_DRIVER}
       options:

sa/profiles/Eltex/MES/confdb/normalizer.py

@@ -147,11 +147,10 @@ class MESNormalizer(BaseNormalizer):
     def normalize_switchport_untagged(self, tokens):
         if_name = self.interface_name(tokens[1])
         untagged = tokens[7]
-        if "," in tokens[7]:
-            # QinQ
-            untagged = tokens[7].split(",")[0]
-        elif "-" in tokens[7]:
-            untagged = tokens[7].split("-")[0]
+        if "," in tokens[7] or "-" in tokens[7]:
+            # QinQ "861-871,986-994"
+            untagged = tokens[7].replace("-", ",").split(",")[0]
+
         yield self.make_switchport_untagged(interface=if_name, unit=if_name, vlan_filter=untagged)
 
     @match("interface", ANY, "switchport", ANY, "native", "vlan", ANY)

sa/profiles/Eltex/TAU/get_capabilities.py

 # ---------------------------------------------------------------------
 # Eltex.TAU.get_capabilities
 # ---------------------------------------------------------------------
-# Copyright (C) 2007-2020 The NOC Project
+# Copyright (C) 2007-2021 The NOC Project
 # See LICENSE for details
 # ---------------------------------------------------------------------
 
 # NOC modules
 from noc.sa.profiles.Generic.get_capabilities import Script as BaseScript
+from noc.sa.profiles.Generic.get_capabilities import false_on_cli_error
 
 
 class Script(BaseScript):
     name = "Eltex.TAU.get_capabilities"
 
     CHECK_SNMP_GET = {"SNMP | OID | fxsMonitoring": "1.3.6.1.4.1.35265.1.9.10.5.0"}
+
+    @false_on_cli_error
+    def has_lldp_cli(self):
+        """
+        Check box has lldp enabled
+        """
+        cmd = self.cli("lldpctl", cached=True)
+        return "Unable to connect to lldpd daemon" not in cmd

sa/profiles/Eltex/TAU/get_lldp_neighbors.py

+# ---------------------------------------------------------------------
+# Eltex.TAU.get_lldp_neighbors
+# ---------------------------------------------------------------------
+# Copyright (C) 2007-2021 The NOC Project
+# See LICENSE for details
+# ---------------------------------------------------------------------
+
+# NOC modules
+from noc.core.script.base import BaseScript
+from noc.sa.interfaces.igetlldpneighbors import IGetLLDPNeighbors
+from noc.core.text import parse_kv
+from noc.core.lldp import (
+    LLDP_CHASSIS_SUBTYPE_MAC,
+    LLDP_CHASSIS_SUBTYPE_INTERFACE_NAME,
+    LLDP_CHASSIS_SUBTYPE_LOCAL,
+    LLDP_PORT_SUBTYPE_MAC,
+    LLDP_PORT_SUBTYPE_NAME,
+    LLDP_PORT_SUBTYPE_LOCAL,
+)
+
+
+class Script(BaseScript):
+    name = "Eltex.TAU.get_lldp_neighbors"
+    interface = IGetLLDPNeighbors
+
+    lldp_map = {
+        "interface": "local_interface",
+        "chassisid": "remote_chassis_id",
+        "sysname": "remote_system_name",
+        "sysdescr": "remote_system_description",
+        "portid": "remote_port",
+        "portdescr": "remote_port_description",
+        "capability": "remote_capabilities",
+    }
+
+    def execute_cli(self, **kwargs):
+        r = []
+        try:
+            lldp = self.cli("lldpctl", cached=True)
+        except self.CLISyntaxError:
+            return []
+
+        for block in lldp.split("----"):
+            neighbors = []
+            if not block:
+                continue
+            n = parse_kv(self.lldp_map, block)
+            if not n:
+                continue
+            n["local_interface"] = n["local_interface"].split(",")[0]
+            loca_iface = n.pop("local_interface")
+            if n["remote_chassis_id"].startswith("mac"):
+                n["remote_chassis_id_subtype"] = LLDP_CHASSIS_SUBTYPE_MAC
+            elif n["remote_chassis_id"].startswith("ifname"):
+                n["remote_chassis_id_subtype"] = LLDP_CHASSIS_SUBTYPE_INTERFACE_NAME
+            else:
+                n["remote_chassis_id_subtype"] = LLDP_CHASSIS_SUBTYPE_LOCAL
+            n["remote_chassis_id"] = n["remote_chassis_id"].split(" ")[1]
+            if n["remote_port"].startswith("mac"):
+                n["remote_port_subtype"] = LLDP_PORT_SUBTYPE_MAC
+            elif n["remote_port"].startswith("ifname"):
+                n["remote_port_subtype"] = LLDP_PORT_SUBTYPE_NAME
+            else:
+                n["remote_port_subtype"] = LLDP_PORT_SUBTYPE_LOCAL
+            n["remote_capabilities"] = 0
+            n["remote_port"] = n["remote_port"].split(" ")[1]
+            neighbors += [n]
+            if neighbors:
+                r += [{"local_interface": loca_iface, "neighbors": neighbors}]
+        return r

sa/profiles/Eltex/TAU/profile.py

@@ -2,7 +2,7 @@
 # Vendor: Eltex
 # OS:     TAU
 # ---------------------------------------------------------------------
-# Copyright (C) 2007-2020 The NOC Project
+# Copyright (C) 2007-2021 The NOC Project
 # See LICENSE for details
 # ---------------------------------------------------------------------
 
@@ -36,6 +36,8 @@ class Profile(BaseProfile):
         "is_tau36": {"platform": {"$regex": r"^TAU-36"}},
     }
 
+    already_in_shell = False
+
     def setup_session(self, script):
         try:
             script.cli("show hwaddr", cached=True)

sa/profiles/Generic/get_interfaces.py

@@ -90,7 +90,11 @@ class Script(BaseScript):
                 )
                 continue
             o = int(oid.split(".")[-1])
-            result[pid_ifindex_mappings[o]]["untagged_vlan"] = pvid
+            if o in pid_ifindex_mappings:
+                o = pid_ifindex_mappings[o]
+            else:
+                self.logger.warning("PortID %s not in ifindex mapping. Use as is", o)
+            result[o]["untagged_vlan"] = pvid
         for oid, ports_mask in self.snmp.getnext(
             mib["Q-BRIDGE-MIB::dot1qVlanCurrentEgressPorts"],
             max_repetitions=self.get_max_repetitions(),

sa/profiles/Generic/get_lldp_neighbors.py

@@ -91,7 +91,6 @@ class Script(BaseScript):
                     mib["LLDP-MIB::lldpRemPortDesc"],
                     mib["LLDP-MIB::lldpRemSysName"],
                 ],
-                bulk=True,
                 max_retries=1,
                 display_hints={
                     "1.0.8802.1.1.2.1.4.1.1.7": render_bin,

sa/profiles/H3C/VRP/profile.py

@@ -24,10 +24,10 @@ class Profile(BaseProfile):
     pattern_syntax_error = r"% Wrong parameter|% Unrecognized command found at"
 
     matchers = {
-        "is_old_version": {"version": {"$regex": r"3.02.*"}},
-        "is_310_version": {"version": {"$regex": r"3.10.*"}},
-        "is_52_version": {"version": {"$regex": r"5.2\S+"}},
-        "is_53_version": {"version": {"$regex": r"5.3\S+"}},
+        "is_old_version": {"version": {"$regex": r"3\.02.*"}},
+        "is_310_version": {"version": {"$regex": r"3\.10.*"}},
+        "is_52_version": {"version": {"$regex": r"5\.2\S+"}},
+        "is_53_version": {"version": {"$regex": r"5\.3\S+"}},
         "is_S3600_platform": {"platform": {"$regex": r".*S3600.*"}},
     }