diff --git a/inv/models/interface.py b/inv/models/interface.py index 246221bdc338ecd4114180e66bc3798ac2933787..dc8a506e17df0b53b0fc1cd8ee93b971e3712a3a 100644 --- a/inv/models/interface.py +++ b/inv/models/interface.py @@ -19,6 +19,7 @@ from mongoengine.fields import ( DateTimeField, ReferenceField, ObjectIdField, + DictField, ) from pymongo import ReadPreference from typing import Optional, Iterable, List @@ -130,6 +131,7 @@ class Interface(Document): # Labels labels = ListField(StringField()) effective_labels = ListField(StringField()) + extra_labels = DictField() PROFILE_LINK = "profile" @@ -140,6 +142,14 @@ class Interface(Document): def get_by_id(cls, id) -> Optional["Interface"]: return Interface.objects.filter(id=id).first() + def clean(self): + if self.extra_labels: + self.labels += [ + ll + for ll in Label.merge_labels(self.extra_labels.values()) + if Interface.can_set_label(ll) + ] + @classmethod def get_component( cls, managed_object: "ManagedObject", interface=None, ifindex=None, **kwargs diff --git a/inv/models/sensor.py b/inv/models/sensor.py index 5fa30a5d1276251e1501135737a201d3b39ae09a..48fb1ba51496afdf1084e874cb563e6faf6a6535 100644 --- a/inv/models/sensor.py +++ b/inv/models/sensor.py @@ -14,7 +14,7 @@ from typing import Dict, Optional, Iterable, List # Third-party modules from mongoengine.document import Document -from mongoengine.fields import StringField, IntField, LongField, ListField, DateTimeField +from mongoengine.fields import StringField, IntField, LongField, ListField, DateTimeField, DictField import cachetools # NOC modules @@ -83,6 +83,7 @@ class Sensor(Document): # Labels labels = ListField(StringField()) effective_labels = ListField(StringField()) + extra_labels = DictField() _id_cache = cachetools.TTLCache(maxsize=100, ttl=60) _bi_id_cache = cachetools.TTLCache(maxsize=100, ttl=60) @@ -104,6 +105,14 @@ class Sensor(Document): def get_by_bi_id(cls, id): return Sensor.objects.filter(bi_id=id).first() + def clean(self): + if self.extra_labels: + self.labels += [ + ll + for ll in Label.merge_labels(self.extra_labels.values()) + if Sensor.can_set_label(ll) + ] + @property def munits(self) -> MeasurementUnits: """ diff --git a/main/models/label.py b/main/models/label.py index bce73aacdab46f0b70dea8831167d88651cb30e8..d4fbff46789670ef436a17e2ebc98694eec78159 100644 --- a/main/models/label.py +++ b/main/models/label.py @@ -414,6 +414,7 @@ class Label(Document): enable_managedobject=False, enable_slaprobe=False, enable_interface=False, + enable_sensor=False, bg_color1=0xFFFFFF, fg_color1=0x000000, bg_color2=0xFFFFFF, @@ -430,6 +431,7 @@ class Label(Document): :param enable_managedobject: :param enable_slaprobe: :param enable_interface: + :param enable_sensor: :param bg_color1: :param fg_color1: :param bg_color2: @@ -455,6 +457,8 @@ class Label(Document): settings["enable_slaprobe"] = enable_slaprobe if enable_interface: settings["enable_interface"] = enable_interface + if enable_sensor: + settings["enable_sensor"] = enable_sensor if bg_color1 and bg_color1 != 0xFFFFFF: settings["bg_color1"] = bg_color1 if fg_color1: diff --git a/sa/interfaces/igetinterfaces.py b/sa/interfaces/igetinterfaces.py index 17213947c20e58e82ae43e63b6853be599a84e94..00db94c52df4609d2c0e90a220833ee408ad6fc2 100644 --- a/sa/interfaces/igetinterfaces.py +++ b/sa/interfaces/igetinterfaces.py @@ -23,6 +23,7 @@ from .base import ( StringParameter, BooleanParameter, IntParameter, + LabelListParameter, ) @@ -368,7 +369,17 @@ class IGetInterfaces(BaseInterface): "description": StringParameter(required=False), "mac": MACAddressParameter(required=False), "snmp_ifindex": IntParameter(required=False), - "hints": StringListParameter(choices=["uplink", "uni", "nni"], required=False), + # noc::interface::role::uni/nni + # noc::topology::direction::uplink + "hints": LabelListParameter( + choices=["uplink", "uni", "nni"], + required=False, + allowed_scopes=[ + "noc::topology::direction", + "noc::interface::role", + "noc::interface::hints", + ], + ), "subinterfaces": ListOfParameter( element=DictParameter( attrs={ diff --git a/sa/interfaces/igetinventory.py b/sa/interfaces/igetinventory.py index a927e76433440f145d1e38659c8dbf645c6cb8dd..0fb1e27fa192e9d6eccde0c31ce54f32dc4cf2ee 100644 --- a/sa/interfaces/igetinventory.py +++ b/sa/interfaces/igetinventory.py @@ -15,6 +15,7 @@ from .base import ( StringListParameter, REStringParameter, OIDParameter, + LabelListParameter, ) @@ -69,6 +70,8 @@ class IGetInventory(BaseInterface): "status": BooleanParameter(default=True), # Optional description "description": StringParameter(required=False), + # + "labels": LabelListParameter(required=False), # MeasurementUnit Name "measurement": StringParameter(default="Scalar"), # Collected hints diff --git a/sa/interfaces/igetslaprobes.py b/sa/interfaces/igetslaprobes.py index d40e642c9106575832b6c3ee15aaab8187d7155d..bb1ab4551e986f49b6f4be72b48dacfe359eb5a2 100644 --- a/sa/interfaces/igetslaprobes.py +++ b/sa/interfaces/igetslaprobes.py @@ -10,9 +10,9 @@ from noc.core.interface.base import BaseInterface from .base import ( DictListParameter, StringParameter, - StringListParameter, BooleanParameter, IntParameter, + LabelListParameter, ) @@ -58,6 +58,6 @@ class IGetSLAProbes(BaseInterface): "target": StringParameter(), "hw_timestamp": BooleanParameter(default=False), # Custom field - "tags": StringListParameter(required=False), + "tags": LabelListParameter(required=False, default_scope="noc::sla::tag"), } ) diff --git a/services/discovery/jobs/base.py b/services/discovery/jobs/base.py index f841a80f100ba69cc8dc7c76bb885ee5c11977bf..5a1dfc4566599c255d1a0d1b2cb0615948eee956 100644 --- a/services/discovery/jobs/base.py +++ b/services/discovery/jobs/base.py @@ -586,6 +586,17 @@ class DiscoveryCheck(object): ignore_empty = ignore_empty or [] for k, v in values.items(): vv = getattr(obj, k) + if hasattr(obj, "extra_labels") and k == "extra_labels": + # Processed extra_labels + sa_labels = obj.extra_labels.get("sa", []) + if v != sa_labels: + remove_labels = set(sa_labels).difference(v) + if remove_labels: + obj.labels = [ll for ll in obj.labels if ll not in remove_labels] + changes += [("labels", obj.labels)] + obj.extra_labels["sa"] = v + changes += [("extra_labels", {"sa": sa_labels})] + continue if v != vv: if not isinstance(v, int) or not hasattr(vv, "id") or v != vv.id: if k in ignore_empty and (v is None or v == ""): diff --git a/services/discovery/jobs/box/asset.py b/services/discovery/jobs/box/asset.py index 4b632b5b90037c91a080c1a84d9e693fbb77772b..083b4fe4f3815a9a308fe8aaf8b90737caaf1190 100644 --- a/services/discovery/jobs/box/asset.py +++ b/services/discovery/jobs/box/asset.py @@ -19,6 +19,7 @@ import cachetools # NOC modules from noc.services.discovery.jobs.base import DiscoveryCheck +from noc.main.models.label import Label from noc.inv.models.objectmodel import ObjectModel, ConnectionRule from noc.inv.models.object import Object, ObjectAttr from noc.inv.models.vendor import Vendor @@ -466,6 +467,7 @@ class AssetCheck(DiscoveryCheck): label=sf.get("description"), snmp_oid=sf.get("snmp_oid"), ipmi_id=sf.get("ipmi_id"), + labels=sf.get("labels"), ) del self.sensors[(obj, sn)] else: @@ -482,6 +484,7 @@ class AssetCheck(DiscoveryCheck): label=si.get("description"), snmp_oid=si.get("snmp_oid"), ipmi_id=si.get("ipmi_id"), + labels=si.get("labels"), ) def submit_sensor( @@ -493,6 +496,7 @@ class AssetCheck(DiscoveryCheck): label: Optional[str] = None, snmp_oid: Optional[str] = None, ipmi_id: Optional[str] = None, + labels: List[str] = None, ): self.logger.info("[%s|%s] Creating new sensor '%s'", obj.name if obj else "-", "-", name) s = Sensor( @@ -517,6 +521,11 @@ class AssetCheck(DiscoveryCheck): "-", name, ) + if labels is not None: + for ll in labels: + Label.ensure_label(ll) + s.labels = [ll for ll in labels if Sensor.can_set_label(ll)] + s.extra_labels = {"sa": s.labels} s.save() s.seen(source="asset") @@ -528,6 +537,7 @@ class AssetCheck(DiscoveryCheck): label: Optional[str] = None, snmp_oid: Optional[str] = None, ipmi_id: Optional[str] = None, + labels: Optional[List[str]] = None, ): sensor.seen(source="asset") if not status: @@ -546,6 +556,18 @@ class AssetCheck(DiscoveryCheck): elif ipmi_id and sensor.ipmi_id != ipmi_id: sensor.protocol = "ipmi" sensor.ipmi_id = ipmi_id + sa_labels = sensor.extra_labels.get("sa", []) + labels = labels or [] + for ll in labels: + if ll in sa_labels: + continue + self.logger.info("[%s] Ensure Sensor label: %s", sensor.id, ll) + Label.ensure_label(ll, enable_sensor=True) + if labels != sa_labels: + remove_labels = set(sa_labels).difference(set(labels)) + if remove_labels: + sensor.labels = [ll for ll in sensor.labels if ll not in remove_labels] + sensor.extra_labels["sa"] = labels sensor.save() def normalize_sensor_units(self, units: str) -> MeasurementUnits: diff --git a/services/discovery/jobs/box/interface.py b/services/discovery/jobs/box/interface.py index 814d0b9dbaaba463c3e84440496eec425a82d773..bcbbcdddab95bf572bafdd97f613b369812fa081 100644 --- a/services/discovery/jobs/box/interface.py +++ b/services/discovery/jobs/box/interface.py @@ -127,7 +127,7 @@ class InterfaceCheck(PolicyDiscoveryCheck): aggregated_interface=agg, enabled_protocols=i.get("enabled_protocols", []), ifindex=i.get("snmp_ifindex"), - hints=i.get("hints", []), + labels=i.get("hints", []), ) icache[i["name"]] = iface # Submit subinterfaces @@ -252,10 +252,14 @@ class InterfaceCheck(PolicyDiscoveryCheck): aggregated_interface=None, enabled_protocols: List[str] = None, ifindex: Optional[int] = None, - hints: List[str] = None, + labels: List[str] = None, ): enabled_protocols = enabled_protocols or [] iface = self.get_interface_by_name(name) + labels = labels or [] + for ll in labels: + if Interface.can_set_label(ll): + Label.ensure_label(ll, enable_interface=True) if iface: ignore_empty = ["ifindex"] if self.is_confdb_source: @@ -271,7 +275,8 @@ class InterfaceCheck(PolicyDiscoveryCheck): "aggregated_interface": aggregated_interface, "enabled_protocols": enabled_protocols, "ifindex": ifindex, - "hints": hints or [], + "hints": labels or [], + "external_labels": [ll for ll in labels if Interface.can_set_label(ll)], }, ignore_empty=ignore_empty, ) @@ -289,6 +294,9 @@ class InterfaceCheck(PolicyDiscoveryCheck): enabled_protocols=enabled_protocols, ifindex=ifindex, ) + if labels: + iface.labels = [ll for ll in labels if Interface.can_set_label(ll)] + iface.extra_labels["sa"] = labels iface.save() self.set_interface(name, iface) if mac: diff --git a/services/discovery/jobs/box/sla.py b/services/discovery/jobs/box/sla.py index f34dd8e623f24b4e66b91a90899ed4aa57302160..580a2aa11aebb852df901aa8a3f20f1ea85f6d74 100644 --- a/services/discovery/jobs/box/sla.py +++ b/services/discovery/jobs/box/sla.py @@ -63,6 +63,12 @@ class SLACheck(DiscoveryCheck): self.logger.info("[%s|%s] Removing probe", group, p.name) p.fire_event("missed") continue + extra_labels = set(p.extra_labels.get("sa", [])) + for ll in new_data.get("tags", []): + if ll in extra_labels: + continue + self.logger.info("[%s] Ensure SLA label: %s", p.id, ll) + Label.ensure_label(ll, enable_slaprobe=True) self.update_if_changed( p, { @@ -71,10 +77,8 @@ class SLACheck(DiscoveryCheck): "tos": new_data.get("tos", 0), "target": new_data["target"], "hw_timestamp": new_data.get("hw_timestamp", False), - "labels": [ - ll - for ll in new_data.get("tags", []) - if Label.get_effective_setting(ll, "enable_slaprobe") + "extra_labels": [ + ll for ll in new_data.get("tags", []) if SLAProbe.can_set_label(ll) ], }, ) @@ -99,8 +103,10 @@ class SLACheck(DiscoveryCheck): tos=new_data.get("tos", 0), target=new_data["target"], hw_timestamp=new_data.get("hw_timestamp", False), - labels=new_data.get("tags", []), ) + if new_data.get("tags"): + probe.labels = [ll for ll in new_data["tags"] if SLAProbe.can_set_label(ll)] + probe.extra_labels["sa"] = new_data["tags"] probe.save() if not new_data["status"]: probe.fire_event("down") diff --git a/sla/models/slaprobe.py b/sla/models/slaprobe.py index e805cd374624eba98c40e02f0c4dd91b4045a3fa..dba92bfd5daeddd718231a11af3a8a75df282faa 100644 --- a/sla/models/slaprobe.py +++ b/sla/models/slaprobe.py @@ -21,6 +21,7 @@ from mongoengine.fields import ( LongField, IntField, ReferenceField, + DictField, ) import cachetools @@ -83,6 +84,7 @@ class SLAProbe(Document): # Labels labels = ListField(StringField()) effective_labels = ListField(StringField()) + extra_labels = DictField() # service = ReferenceField(Service) @@ -102,6 +104,14 @@ class SLAProbe(Document): def get_by_bi_id(cls, id): return SLAProbe.objects.filter(bi_id=id).first() + def clean(self): + if self.extra_labels: + self.labels += [ + ll + for ll in Label.merge_labels(self.extra_labels.values()) + if SLAProbe.can_set_label(ll) + ] + @cachetools.cached(_target_cache, key=lambda x: str(x.id), lock=id_lock) def get_target(self): address = self.target