From bc91f8a77019b471ba2a5940c6f9db9d984af324 Mon Sep 17 00:00:00 2001 From: Andrey Vertiprahov Date: Thu, 20 Jan 2022 10:30:48 +0500 Subject: [PATCH 1/5] Add Labels fields to SA Interface. --- main/models/label.py | 2 ++ sa/interfaces/igetinterfaces.py | 13 ++++++++++++- sa/interfaces/igetinventory.py | 3 +++ sa/interfaces/igetslaprobes.py | 3 ++- 4 files changed, 19 insertions(+), 2 deletions(-) diff --git a/main/models/label.py b/main/models/label.py index bce73aacda..c961742720 100644 --- a/main/models/label.py +++ b/main/models/label.py @@ -102,6 +102,8 @@ class Label(Document): is_regex = BooleanField(default=False) # For scoped - to propagating settings on own labels propagate = BooleanField(default=False) + # Allow SA interface labels control + allow_sa_script = BooleanField(default=False) # Label scope enable_agent = BooleanField(default=False) enable_service = BooleanField(default=False) diff --git a/sa/interfaces/igetinterfaces.py b/sa/interfaces/igetinterfaces.py index 17213947c2..00db94c52d 100644 --- a/sa/interfaces/igetinterfaces.py +++ b/sa/interfaces/igetinterfaces.py @@ -23,6 +23,7 @@ from .base import ( StringParameter, BooleanParameter, IntParameter, + LabelListParameter, ) @@ -368,7 +369,17 @@ class IGetInterfaces(BaseInterface): "description": StringParameter(required=False), "mac": MACAddressParameter(required=False), "snmp_ifindex": IntParameter(required=False), - "hints": StringListParameter(choices=["uplink", "uni", "nni"], required=False), + # noc::interface::role::uni/nni + # noc::topology::direction::uplink + "hints": LabelListParameter( + choices=["uplink", "uni", "nni"], + required=False, + allowed_scopes=[ + "noc::topology::direction", + "noc::interface::role", + "noc::interface::hints", + ], + ), "subinterfaces": ListOfParameter( element=DictParameter( attrs={ diff --git a/sa/interfaces/igetinventory.py b/sa/interfaces/igetinventory.py index a927e76433..0fb1e27fa1 100644 --- a/sa/interfaces/igetinventory.py +++ b/sa/interfaces/igetinventory.py @@ -15,6 +15,7 @@ from .base import ( StringListParameter, REStringParameter, OIDParameter, + LabelListParameter, ) @@ -69,6 +70,8 @@ class IGetInventory(BaseInterface): "status": BooleanParameter(default=True), # Optional description "description": StringParameter(required=False), + # + "labels": LabelListParameter(required=False), # MeasurementUnit Name "measurement": StringParameter(default="Scalar"), # Collected hints diff --git a/sa/interfaces/igetslaprobes.py b/sa/interfaces/igetslaprobes.py index d40e642c91..17a8cf915c 100644 --- a/sa/interfaces/igetslaprobes.py +++ b/sa/interfaces/igetslaprobes.py @@ -13,6 +13,7 @@ from .base import ( StringListParameter, BooleanParameter, IntParameter, + LabelListParameter, ) @@ -58,6 +59,6 @@ class IGetSLAProbes(BaseInterface): "target": StringParameter(), "hw_timestamp": BooleanParameter(default=False), # Custom field - "tags": StringListParameter(required=False), + "tags": LabelListParameter(required=False, default_scope="noc::sla::tag"), } ) -- GitLab From 898245be71281d88f86ab71a2281ab05fe97fb4b Mon Sep 17 00:00:00 2001 From: Andrey Vertiprahov Date: Thu, 20 Jan 2022 15:51:41 +0500 Subject: [PATCH 2/5] Add extra_labels to SLAProbe. --- main/models/label.py | 2 -- sa/interfaces/igetslaprobes.py | 1 - services/discovery/jobs/base.py | 11 +++++++++++ services/discovery/jobs/box/sla.py | 12 ++++++++---- sla/models/slaprobe.py | 6 ++++++ 5 files changed, 25 insertions(+), 7 deletions(-) diff --git a/main/models/label.py b/main/models/label.py index c961742720..bce73aacda 100644 --- a/main/models/label.py +++ b/main/models/label.py @@ -102,8 +102,6 @@ class Label(Document): is_regex = BooleanField(default=False) # For scoped - to propagating settings on own labels propagate = BooleanField(default=False) - # Allow SA interface labels control - allow_sa_script = BooleanField(default=False) # Label scope enable_agent = BooleanField(default=False) enable_service = BooleanField(default=False) diff --git a/sa/interfaces/igetslaprobes.py b/sa/interfaces/igetslaprobes.py index 17a8cf915c..bb1ab4551e 100644 --- a/sa/interfaces/igetslaprobes.py +++ b/sa/interfaces/igetslaprobes.py @@ -10,7 +10,6 @@ from noc.core.interface.base import BaseInterface from .base import ( DictListParameter, StringParameter, - StringListParameter, BooleanParameter, IntParameter, LabelListParameter, diff --git a/services/discovery/jobs/base.py b/services/discovery/jobs/base.py index f841a80f10..5a1dfc4566 100644 --- a/services/discovery/jobs/base.py +++ b/services/discovery/jobs/base.py @@ -586,6 +586,17 @@ class DiscoveryCheck(object): ignore_empty = ignore_empty or [] for k, v in values.items(): vv = getattr(obj, k) + if hasattr(obj, "extra_labels") and k == "extra_labels": + # Processed extra_labels + sa_labels = obj.extra_labels.get("sa", []) + if v != sa_labels: + remove_labels = set(sa_labels).difference(v) + if remove_labels: + obj.labels = [ll for ll in obj.labels if ll not in remove_labels] + changes += [("labels", obj.labels)] + obj.extra_labels["sa"] = v + changes += [("extra_labels", {"sa": sa_labels})] + continue if v != vv: if not isinstance(v, int) or not hasattr(vv, "id") or v != vv.id: if k in ignore_empty and (v is None or v == ""): diff --git a/services/discovery/jobs/box/sla.py b/services/discovery/jobs/box/sla.py index f34dd8e623..1de52ecdbb 100644 --- a/services/discovery/jobs/box/sla.py +++ b/services/discovery/jobs/box/sla.py @@ -63,6 +63,12 @@ class SLACheck(DiscoveryCheck): self.logger.info("[%s|%s] Removing probe", group, p.name) p.fire_event("missed") continue + extra_labels = set(p.extra_labels.get("sa", [])) + for ll in new_data.get("tags", []): + if ll in extra_labels: + continue + self.logger.info("[%s] Ensure SLA label: %s", p.id, ll) + Label.ensure_label(ll, enable_slaprobe=True) self.update_if_changed( p, { @@ -71,10 +77,8 @@ class SLACheck(DiscoveryCheck): "tos": new_data.get("tos", 0), "target": new_data["target"], "hw_timestamp": new_data.get("hw_timestamp", False), - "labels": [ - ll - for ll in new_data.get("tags", []) - if Label.get_effective_setting(ll, "enable_slaprobe") + "extra_labels": [ + ll for ll in new_data.get("tags", []) if SLAProbe.can_set_label(ll) ], }, ) diff --git a/sla/models/slaprobe.py b/sla/models/slaprobe.py index e805cd3746..74d887ba82 100644 --- a/sla/models/slaprobe.py +++ b/sla/models/slaprobe.py @@ -21,6 +21,7 @@ from mongoengine.fields import ( LongField, IntField, ReferenceField, + DictField, ) import cachetools @@ -83,6 +84,7 @@ class SLAProbe(Document): # Labels labels = ListField(StringField()) effective_labels = ListField(StringField()) + extra_labels = DictField() # service = ReferenceField(Service) @@ -102,6 +104,10 @@ class SLAProbe(Document): def get_by_bi_id(cls, id): return SLAProbe.objects.filter(bi_id=id).first() + def clean(self): + if self.extra_labels: + self.labels += [ll for ll in Label.merge_labels(self.extra_labels.values()) if SLAProbe.can_set_label(ll)] + @cachetools.cached(_target_cache, key=lambda x: str(x.id), lock=id_lock) def get_target(self): address = self.target -- GitLab From 0c1557033e30b93a4ca20decce4288687ce5c7b2 Mon Sep 17 00:00:00 2001 From: Andrey Vertiprahov Date: Thu, 20 Jan 2022 17:28:29 +0500 Subject: [PATCH 3/5] Add extra_labels to Sensor model. --- inv/models/sensor.py | 11 ++++++++++- main/models/label.py | 4 ++++ services/discovery/jobs/box/asset.py | 21 +++++++++++++++++++++ sla/models/slaprobe.py | 6 +++++- 4 files changed, 40 insertions(+), 2 deletions(-) diff --git a/inv/models/sensor.py b/inv/models/sensor.py index 5fa30a5d12..48fb1ba514 100644 --- a/inv/models/sensor.py +++ b/inv/models/sensor.py @@ -14,7 +14,7 @@ from typing import Dict, Optional, Iterable, List # Third-party modules from mongoengine.document import Document -from mongoengine.fields import StringField, IntField, LongField, ListField, DateTimeField +from mongoengine.fields import StringField, IntField, LongField, ListField, DateTimeField, DictField import cachetools # NOC modules @@ -83,6 +83,7 @@ class Sensor(Document): # Labels labels = ListField(StringField()) effective_labels = ListField(StringField()) + extra_labels = DictField() _id_cache = cachetools.TTLCache(maxsize=100, ttl=60) _bi_id_cache = cachetools.TTLCache(maxsize=100, ttl=60) @@ -104,6 +105,14 @@ class Sensor(Document): def get_by_bi_id(cls, id): return Sensor.objects.filter(bi_id=id).first() + def clean(self): + if self.extra_labels: + self.labels += [ + ll + for ll in Label.merge_labels(self.extra_labels.values()) + if Sensor.can_set_label(ll) + ] + @property def munits(self) -> MeasurementUnits: """ diff --git a/main/models/label.py b/main/models/label.py index bce73aacda..d4fbff4678 100644 --- a/main/models/label.py +++ b/main/models/label.py @@ -414,6 +414,7 @@ class Label(Document): enable_managedobject=False, enable_slaprobe=False, enable_interface=False, + enable_sensor=False, bg_color1=0xFFFFFF, fg_color1=0x000000, bg_color2=0xFFFFFF, @@ -430,6 +431,7 @@ class Label(Document): :param enable_managedobject: :param enable_slaprobe: :param enable_interface: + :param enable_sensor: :param bg_color1: :param fg_color1: :param bg_color2: @@ -455,6 +457,8 @@ class Label(Document): settings["enable_slaprobe"] = enable_slaprobe if enable_interface: settings["enable_interface"] = enable_interface + if enable_sensor: + settings["enable_sensor"] = enable_sensor if bg_color1 and bg_color1 != 0xFFFFFF: settings["bg_color1"] = bg_color1 if fg_color1: diff --git a/services/discovery/jobs/box/asset.py b/services/discovery/jobs/box/asset.py index 4b632b5b90..cbbe48cadb 100644 --- a/services/discovery/jobs/box/asset.py +++ b/services/discovery/jobs/box/asset.py @@ -19,6 +19,7 @@ import cachetools # NOC modules from noc.services.discovery.jobs.base import DiscoveryCheck +from noc.main.models.label import Label from noc.inv.models.objectmodel import ObjectModel, ConnectionRule from noc.inv.models.object import Object, ObjectAttr from noc.inv.models.vendor import Vendor @@ -466,6 +467,7 @@ class AssetCheck(DiscoveryCheck): label=sf.get("description"), snmp_oid=sf.get("snmp_oid"), ipmi_id=sf.get("ipmi_id"), + labels=si.get("labels"), ) del self.sensors[(obj, sn)] else: @@ -482,6 +484,7 @@ class AssetCheck(DiscoveryCheck): label=si.get("description"), snmp_oid=si.get("snmp_oid"), ipmi_id=si.get("ipmi_id"), + labels=si.get("labels"), ) def submit_sensor( @@ -493,6 +496,7 @@ class AssetCheck(DiscoveryCheck): label: Optional[str] = None, snmp_oid: Optional[str] = None, ipmi_id: Optional[str] = None, + labels: List[str] = None, ): self.logger.info("[%s|%s] Creating new sensor '%s'", obj.name if obj else "-", "-", name) s = Sensor( @@ -517,6 +521,11 @@ class AssetCheck(DiscoveryCheck): "-", name, ) + if labels is not None: + for ll in labels: + Label.ensure_label(ll) + s.labels = [ll for ll in labels if Sensor.can_set_label(ll)] + s.extra_labels = {"sa": s.labels} s.save() s.seen(source="asset") @@ -528,6 +537,7 @@ class AssetCheck(DiscoveryCheck): label: Optional[str] = None, snmp_oid: Optional[str] = None, ipmi_id: Optional[str] = None, + labels: Optional[List[str]] = None, ): sensor.seen(source="asset") if not status: @@ -546,6 +556,17 @@ class AssetCheck(DiscoveryCheck): elif ipmi_id and sensor.ipmi_id != ipmi_id: sensor.protocol = "ipmi" sensor.ipmi_id = ipmi_id + sa_labels = sensor.extra_labels.get("sa", []) + for ll in labels or []: + if ll in sa_labels: + continue + self.logger.info("[%s] Ensure Sensor label: %s", sensor.id, ll) + Label.ensure_label(ll, enable_sensor=True) + if labels != sa_labels: + remove_labels = set(sa_labels).difference(labels) + if remove_labels: + sensor.labels = [ll for ll in sensor.labels if ll not in remove_labels] + sensor.extra_labels["sa"] = labels sensor.save() def normalize_sensor_units(self, units: str) -> MeasurementUnits: diff --git a/sla/models/slaprobe.py b/sla/models/slaprobe.py index 74d887ba82..dba92bfd5d 100644 --- a/sla/models/slaprobe.py +++ b/sla/models/slaprobe.py @@ -106,7 +106,11 @@ class SLAProbe(Document): def clean(self): if self.extra_labels: - self.labels += [ll for ll in Label.merge_labels(self.extra_labels.values()) if SLAProbe.can_set_label(ll)] + self.labels += [ + ll + for ll in Label.merge_labels(self.extra_labels.values()) + if SLAProbe.can_set_label(ll) + ] @cachetools.cached(_target_cache, key=lambda x: str(x.id), lock=id_lock) def get_target(self): -- GitLab From a7585357b160c5386887c9c7a61cb46f49c80071 Mon Sep 17 00:00:00 2001 From: Andrey Vertiprahov Date: Thu, 20 Jan 2022 18:03:29 +0500 Subject: [PATCH 4/5] Fix typo. --- services/discovery/jobs/box/asset.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/services/discovery/jobs/box/asset.py b/services/discovery/jobs/box/asset.py index cbbe48cadb..083b4fe4f3 100644 --- a/services/discovery/jobs/box/asset.py +++ b/services/discovery/jobs/box/asset.py @@ -467,7 +467,7 @@ class AssetCheck(DiscoveryCheck): label=sf.get("description"), snmp_oid=sf.get("snmp_oid"), ipmi_id=sf.get("ipmi_id"), - labels=si.get("labels"), + labels=sf.get("labels"), ) del self.sensors[(obj, sn)] else: @@ -557,13 +557,14 @@ class AssetCheck(DiscoveryCheck): sensor.protocol = "ipmi" sensor.ipmi_id = ipmi_id sa_labels = sensor.extra_labels.get("sa", []) - for ll in labels or []: + labels = labels or [] + for ll in labels: if ll in sa_labels: continue self.logger.info("[%s] Ensure Sensor label: %s", sensor.id, ll) Label.ensure_label(ll, enable_sensor=True) if labels != sa_labels: - remove_labels = set(sa_labels).difference(labels) + remove_labels = set(sa_labels).difference(set(labels)) if remove_labels: sensor.labels = [ll for ll in sensor.labels if ll not in remove_labels] sensor.extra_labels["sa"] = labels -- GitLab From 5a097e6f1479654e90a1a535e17ea98ef1c4befa Mon Sep 17 00:00:00 2001 From: Andrey Vertiprahov Date: Thu, 20 Jan 2022 18:17:23 +0500 Subject: [PATCH 5/5] Add extra_labels to Interface Model. --- inv/models/interface.py | 10 ++++++++++ services/discovery/jobs/box/interface.py | 14 +++++++++++--- services/discovery/jobs/box/sla.py | 4 +++- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/inv/models/interface.py b/inv/models/interface.py index 246221bdc3..dc8a506e17 100644 --- a/inv/models/interface.py +++ b/inv/models/interface.py @@ -19,6 +19,7 @@ from mongoengine.fields import ( DateTimeField, ReferenceField, ObjectIdField, + DictField, ) from pymongo import ReadPreference from typing import Optional, Iterable, List @@ -130,6 +131,7 @@ class Interface(Document): # Labels labels = ListField(StringField()) effective_labels = ListField(StringField()) + extra_labels = DictField() PROFILE_LINK = "profile" @@ -140,6 +142,14 @@ class Interface(Document): def get_by_id(cls, id) -> Optional["Interface"]: return Interface.objects.filter(id=id).first() + def clean(self): + if self.extra_labels: + self.labels += [ + ll + for ll in Label.merge_labels(self.extra_labels.values()) + if Interface.can_set_label(ll) + ] + @classmethod def get_component( cls, managed_object: "ManagedObject", interface=None, ifindex=None, **kwargs diff --git a/services/discovery/jobs/box/interface.py b/services/discovery/jobs/box/interface.py index 814d0b9dba..bcbbcdddab 100644 --- a/services/discovery/jobs/box/interface.py +++ b/services/discovery/jobs/box/interface.py @@ -127,7 +127,7 @@ class InterfaceCheck(PolicyDiscoveryCheck): aggregated_interface=agg, enabled_protocols=i.get("enabled_protocols", []), ifindex=i.get("snmp_ifindex"), - hints=i.get("hints", []), + labels=i.get("hints", []), ) icache[i["name"]] = iface # Submit subinterfaces @@ -252,10 +252,14 @@ class InterfaceCheck(PolicyDiscoveryCheck): aggregated_interface=None, enabled_protocols: List[str] = None, ifindex: Optional[int] = None, - hints: List[str] = None, + labels: List[str] = None, ): enabled_protocols = enabled_protocols or [] iface = self.get_interface_by_name(name) + labels = labels or [] + for ll in labels: + if Interface.can_set_label(ll): + Label.ensure_label(ll, enable_interface=True) if iface: ignore_empty = ["ifindex"] if self.is_confdb_source: @@ -271,7 +275,8 @@ class InterfaceCheck(PolicyDiscoveryCheck): "aggregated_interface": aggregated_interface, "enabled_protocols": enabled_protocols, "ifindex": ifindex, - "hints": hints or [], + "hints": labels or [], + "external_labels": [ll for ll in labels if Interface.can_set_label(ll)], }, ignore_empty=ignore_empty, ) @@ -289,6 +294,9 @@ class InterfaceCheck(PolicyDiscoveryCheck): enabled_protocols=enabled_protocols, ifindex=ifindex, ) + if labels: + iface.labels = [ll for ll in labels if Interface.can_set_label(ll)] + iface.extra_labels["sa"] = labels iface.save() self.set_interface(name, iface) if mac: diff --git a/services/discovery/jobs/box/sla.py b/services/discovery/jobs/box/sla.py index 1de52ecdbb..580a2aa11a 100644 --- a/services/discovery/jobs/box/sla.py +++ b/services/discovery/jobs/box/sla.py @@ -103,8 +103,10 @@ class SLACheck(DiscoveryCheck): tos=new_data.get("tos", 0), target=new_data["target"], hw_timestamp=new_data.get("hw_timestamp", False), - labels=new_data.get("tags", []), ) + if new_data.get("tags"): + probe.labels = [ll for ll in new_data["tags"] if SLAProbe.can_set_label(ll)] + probe.extra_labels["sa"] = new_data["tags"] probe.save() if not new_data["status"]: probe.fire_event("down") -- GitLab