Skip to content
GitLab
Commit 4101daf1 authored by Andrey Vertiprahov's avatar Andrey Vertiprahov
Browse files

Merge branch 'issue-1822-clear-InterfaceClassificationRule' into 'master' 

Remove InterfaceClassificationRule and InterfaceClassificationMatch models

See merge request noc/noc!6393
parents fb5afc66 c50c3337
Hide whitespace changes
Inline Side-by-side
commands/interface-profile.py
View file @ 4101daf1
......@@ -15,7 +15,6 @@ from noc.core.mongo.connection import connect
from noc.main.models.label import Label
from noc.inv.models.interface import Interface
from noc.inv.models.interfaceprofile import InterfaceProfile
from noc.inv.models.interfaceclassificationrule import InterfaceClassificationRule
from noc.inv.models.resourcegroup import ResourceGroup
from noc.core.text import alnum_key
......@@ -39,25 +38,7 @@ class Command(BaseCommand):
default=False,
help="Set not matched profile to default",
)
apply_parser.add_argument(
"--use-classification-rule",
action="store_true",
default=False,
help="Use InterfaceClassification Rule for works",
)
apply_parser.add_argument("mos", nargs=argparse.REMAINDER, help="List of object to showing")
apply_confdb_parser = subparsers.add_parser(
"apply-confdb", help="Apply ConfDB classification rules"
)
apply_confdb_parser.add_argument(
"--reset-default",
action="store_true",
default=False,
help="Set not matched profile to default",
)
apply_confdb_parser.add_argument(
"mos", nargs=argparse.REMAINDER, help="List of object to showing"
)
def handle(self, cmd, *args, **options):
connect()
......@@ -123,66 +104,11 @@ class Command(BaseCommand):
i.profile = InterfaceProfile.get_default_profile()
i.save()
def handle_apply_confdb(self, moo, *args, **kwargs):
default_profile = InterfaceProfile.get_default_profile()
for o in self.get_objects(moo):
self.stdout.write(
"%s (%s):\n" % (o.name, o.platform.name if o.platform else o.profile.name)
)
ifmap = {i.name: i for i in self.get_interfaces(o)}
if not ifmap:
self.stdout.write("No ifaces on object\n")
continue
tps = self.get_interface_template(ifmap.values())
proccessed = set()
selectors_skipping = set() # if selectors has not match
cdb = o.get_confdb()
ifprofilemap = {}
for icr in InterfaceClassificationRule.objects.filter(is_active=True).order_by("order"):
if icr.selector.id in selectors_skipping:
continue
r = next(cdb.query(icr.selector.get_confdb_query), None)
if r is None:
# Selectors already fail check
selectors_skipping.add(icr.selector.id)
continue
self.print("[%s] Check selector" % icr)
for match in cdb.query(icr.get_confdb_query):
if match["ifname"] in proccessed or match["ifname"] not in ifmap:
continue
self.print("[%s] Match %s" % (icr, match["ifname"]))
iface = ifmap[match["ifname"]]
proccessed.add(match["ifname"])
if iface.profile_locked:
continue
ifprofilemap[iface.name] = icr.profile
# Set profile
for ifname in ifmap:
i = ifmap[ifname]
if ifname in ifprofilemap and i.profile.id != ifprofilemap[ifname].id:
i.profile = ifprofilemap[ifname]
i.save()
v = "Set %s" % ifprofilemap[ifname].name
elif ifname in ifprofilemap and i.profile.id == ifprofilemap[ifname].id:
v = "Already set %s" % ifprofilemap[ifname].name
else:
v = "Not matched"
if kwargs.get("reset_default") and i.profile != default_profile:
i.profile = default_profile
i.save()
v = "Not matched. Reset to default"
self.show_interface(tps, i, v)
def handle_apply(self, moo, *args, **kwargs):
# sol = config.get("interface_discovery", "get_interface_profile")
# @todo Classification pyrule
default_profile = InterfaceProfile.get_default_profile()
if kwargs.get("use_classification_rule"):
get_profile = InterfaceClassificationRule
get_profile = get_profile.get_classificator()
# raise CommandError("No classification solution")
else:
get_profile = partial(Label.get_instance_profile, InterfaceProfile)
get_profile = partial(Label.get_instance_profile, InterfaceProfile)
pcache = {}
for o in self.get_objects(moo):
self.stdout.write(
......
inv/models/interfaceclassificationrule.py deleted 100644 → 0
View file @ fb5afc66
# ---------------------------------------------------------------------
# Interface Classification Rules models
# ---------------------------------------------------------------------
# Copyright (C) 2007-2020 The NOC Project
# See LICENSE for details
# ---------------------------------------------------------------------
# Python modules
import re
# Third-party modules
from mongoengine.document import Document, EmbeddedDocument
from mongoengine.fields import StringField, IntField, ListField, EmbeddedDocumentField, BooleanField
# NOC modules
from noc.core.mongo.fields import ForeignKeyField, PlainReferenceField
from noc.core.ip import IP
from noc.main.models.prefixtable import PrefixTable, PrefixTablePrefix
from noc.sa.models.managedobjectselector import ManagedObjectSelector
from noc.vc.models.vcfilter import VCFilter
from noc.core.comp import smart_text
from .interfaceprofile import InterfaceProfile
class InterfaceClassificationMatch(EmbeddedDocument):
# Field name
field = StringField(
choices=[
("name", "name"),
("description", "description"),
("ip", "ip"),
("tagged", "tagged vlan"),
("untagged", "untagged vlan"),
("hints", "hints"),
]
)
# Operation
op = StringField(choices=[("eq", "Equals"), ("regexp", "RegExp"), ("in", "in")])
#
value = StringField()
# "ip in"
prefix_table = ForeignKeyField(PrefixTable, required=False)
# *vlan in
vc_filter = ForeignKeyField(VCFilter, required=False)
description = StringField(required=False)
def __str__(self):
if self.prefix_table:
v = self.prefix_table.name
elif self.vc_filter:
v = self.vc_filter.name
else:
v = self.value
return "%s %s %s" % (self.field, self.op, v)
@property
def get_confdb_query(self):
query = ['Match("interfaces", ifname)']
if self.field == "name" and self.op == "eq":
query += ['Filter(ifname == "%s")' % self.value]
elif self.field == "name" and self.op == "regexp":
query += ['Re("%s", ifname, ignore_case=True)' % self.value]
if self.field == "description":
query += ['Match("interfaces", ifname, "description", ifdescr)']
if self.op == "eq":
query += ['Filter(ifdescr == "%s")' % self.value]
elif self.op == "regexp":
query += ['Re("%s", ifdescr, ignore_case=True)' % self.value]
if self.field == "hints" and self.op == "eq":
query += ['Match("interfaces", ifname, "hints", "%s")' % self.value]
if self.field == "ip" and self.op == "eq":
query += [
'Match("virtual-router", vr, "forwarding-instance", fi, "interfaces",'
' ifname, "unit", ifname, "inet", "address", "%s")' % self.value
]
elif self.field == "ip" and self.op == "in" and self.prefix_table:
prefix_match = "( %s )" % " or ".join(
" MatchPrefix('%s', address)" % ptp.prefix
for ptp in PrefixTablePrefix.objects.filter(table=self.prefix_table)
)
query += [
'Match("virtual-router", vr, "forwarding-instance", fi, "interfaces",'
' ifname, "unit", ifname, "inet", "address", address)'
" and %s and Del(vr, fi, address)" % prefix_match
]
if self.field == "untagged" and self.op == "eq" and self.value:
query += [
'Match("virtual-router", vr, "forwarding-instance", fi, "interfaces",'
' ifname, "unit", ifname, "bridge", "switchport", "untagged", %s)' % self.value
]
elif self.field == "untagged" and self.op == "in" and self.vc_filter:
query += [
'Match("virtual-router", vr, "forwarding-instance", fi, "interfaces",'
' ifname, "unit", ifname, "bridge", "switchport", "untagged", untagged)'
' and HasVLAN("%s", untagged) and Del(vr, fi, untagged)' % self.vc_filter.expression
]
if self.field == "tagged" and self.op == "eq" and (self.value or self.vc_filter):
query += [
'Match("virtual-router", vr, "forwarding-instance", fi, "interfaces",'
' ifname, "unit", ifname, "bridge", "switchport", "tagged", tagged)'
' and MatchExactVLAN("%s", tagged) and Del(vr, fi, tagged)'
% (self.value or self.vc_filter.expression)
]
elif self.field == "tagged" and self.op == "in" and self.vc_filter:
query += [
'Match("virtual-router", vr, "forwarding-instance", fi, "interfaces",'
' ifname, "unit", ifname, "bridge", "switchport", "tagged", tagged)'
' and MatchAnyVLAN("%s", tagged) and Del(vr, fi, tagged)'
% self.vc_filter.expression
]
return " and ".join(query)
def compile(self, f_name):
a = getattr(self, "compile_%s_%s" % (self.field, self.op), None)
if a:
return a(f_name)
else:
raise SyntaxError("%s %s is not implemented" % (self.field, self.op))
# name
def compile_name_eq(self, f_name):
return "\n".join(
[
"def %s(iface):" % f_name,
" return iface.name.lower() == %s" % repr(self.value.lower()),
]
)
def compile_name_regexp(self, f_name):
return "\n".join(
[
"rx_%s = re.compile(%s, re.IGNORECASE)" % (f_name, repr(self.value)),
"def %s(iface):" % f_name,
" return bool(rx_%s.search(iface.name))" % f_name,
]
)
# description
def compile_description_eq(self, f_name):
return "\n".join(
[
"def %s(iface):" % f_name,
" return iface.description.lower() == %s" % repr(self.value.lower()),
]
)
def compile_description_regexp(self, f_name):
return "\n".join(
[
"rx_%s = re.compile(%s, re.IGNORECASE)" % (f_name, repr(self.value)),
"def %s(iface):" % f_name,
" return iface.description and bool(rx_%s.search(iface.description))" % f_name,
]
)
# IP
def compile_ip_eq(self, f_name):
v = IP.prefix(self.value)
r = [
"def %s(iface):" % f_name,
" a = [si.ipv%(afi)s_addresses for si in iface.subinterface_set.filter(enabled_afi='IPv%(afi)s')]"
% {"afi": v.afi},
" a = sum(a, [])",
]
if "/" in self.value:
# Compare prefixes
r += [" return any(x for x in a if x == %r)" % v.prefix]
else:
# Compare addresses
v = v.prefix.split("/")[0]
r += [" return any(x for x in a if x.split('/')[0] == %r)" % v]
return "\n".join(r)
def compile_ip_in(self, f_name):
r = [
"pt_%s = PrefixTable.objects.get(id=%s)" % (f_name, self.prefix_table.id),
"def %s(iface):" % f_name,
" for si in iface.subinterface_set.filter(enabled_afi='IPv4'):",
" for a in si.ipv4_addresses:",
" if a in pt_%s:" % f_name,
" return True",
" for si in iface.subinterface_set.filter(enabled_afi='IPv6'):",
" for a in si.ipv6_addresses:",
" if a in pt_%s:" % f_name,
" return True",
" return False",
]
return "\n".join(r)
# Untagged
def compile_untagged_eq(self, f_name):
vlan = int(self.value)
if vlan < 1 or vlan > 4096:
raise SyntaxError("Invalid VLAN")
r = [
"def %s(iface):" % f_name,
" return bool(iface.parent.subinterface_set.filter(enabled_afi='BRIDGE', untagged_vlan=%d).count())"
% vlan,
]
return "\n".join(r)
def compile_untagged_in(self, f_name):
r = [
"vcf_%s = VCFilter.get_by_id(id=%s)" % (f_name, self.vc_filter.id),
"if not vcf_%s:" % f_name,
" raise ValueError('Invalid VC Filter: %s')" % self.vc_filter.name,
"def %s(iface):" % f_name,
" for si in iface.parent.subinterface_set.filter(enabled_afi='BRIDGE'):",
" if si.untagged_vlan and vcf_%s.check(si.untagged_vlan):" % f_name,
" return True",
" return False",
]
return "\n".join(r)
# Tagged
def compile_tagged_eq(self, f_name):
vlan = int(self.value)
if vlan < 1 or vlan > 4096:
raise SyntaxError("Invalid VLAN")
r = [
"def %s(iface):" % f_name,
" return bool(iface.parent.subinterface_set.filter(enabled_afi='BRIDGE', tagged_vlans=%d).count())"
% vlan,
]
return "\n".join(r)
def compile_tagged_in(self, f_name):
r = [
"vcf_%s = VCFilter.get_by_id(id=%s)" % (f_name, self.vc_filter.id),
"if not vcf_%s:" % f_name,
" raise ValueError('Invalid VC Filter: %s')" % self.vc_filter.name,
"def %s(iface):" % f_name,
" for si in iface.parent.subinterface_set.filter(enabled_afi='BRIDGE'):",
" if si.tagged_vlans:",
" if any(vlan for vlan in si.tagged_vlans if vcf_%s.check(vlan)):" % f_name,
" return True",
" return False",
]
return "\n".join(r)
def compile_hints_eq(self, f_name):
r = ["def %s(iface):" % f_name, " return iface.hints and %r in iface.hints" % self.value]
return "\n".join(r)
class InterfaceClassificationRule(Document):
meta = {
"collection": "noc.inv.interfaceclassificationrules",
"strict": False,
"auto_create_index": False,
}
name = StringField(required=False)
is_active = BooleanField(default=True)
description = StringField(required=False)
order = IntField()
selector = ForeignKeyField(ManagedObjectSelector, required=False)
match = ListField(EmbeddedDocumentField(InterfaceClassificationMatch), required=False)
profile = PlainReferenceField(InterfaceProfile, default=InterfaceProfile.get_default_profile)
def __str__(self):
r = [smart_text(x) for x in self.match]
return "%s -> %s" % (", ".join(r), self.profile.name)
@property
def match_expr(self):
"""
Stringified match expression
"""
if not len(self.match):
return "any"
elif len(self.match) == 1:
return smart_text(self.match[0])
else:
return " AND ".join("(%s)" % smart_text(m) for m in self.match)
@property
def get_confdb_query(self):
if not len(self.match):
return 'Match("interfaces", ifname, "type", "physical")'
elif len(self.match) == 1:
return self.match[0].get_confdb_query
else:
return " and ".join("(%s)" % m.get_confdb_query for m in self.match)
@classmethod
def get_classificator_code(cls):
r = ["import re", "import bson", "from noc.sa.models.selectorcache import SelectorCache"]
mf = [
"gsc = {}",
"def classify(interface):",
" def in_selector(o, s):",
" if s in s_cache:",
" return s_cache[s]",
" if s in gsc:",
" selector = gsc[s]",
" else:",
" selector = ManagedObjectSelector.get_by_id(s)",
" gsc[s] = selector",
" r = SelectorCache.is_in_selector(o, selector)",
" # r = o in selector",
" s_cache[s] = r",
" return r",
" s_cache = {}",
" mo = interface.managed_object",
]
for rule in cls.objects.filter(is_active=True).order_by("order"):
rid = str(rule.id)
lmn = []
for i, m in enumerate(rule.match):
mn = "match_%s_%d" % (rid, i)
r += [m.compile(mn)]
lmn += ["%s(interface)" % mn]
if lmn:
mf += [
" if in_selector(mo, %d) and %s:" % (rule.selector.id, " and ".join(lmn)),
" return bson.ObjectId('%s')" % rule.profile.id,
]
else:
mf += [
" if in_selector(mo, %d):" % rule.selector.id,
" return bson.ObjectId('%s')" % rule.profile.id,
]
r += mf
return "\n".join(r)
@classmethod
def get_classificator(cls):
code = cls.get_classificator_code() + "\nhandlers[0] = classify\n"
# Hack to retrieve reference
handlers = {}
# Compile code
exec(
code,
{
"re": re,
"PrefixTable": PrefixTable,
"VCFilter": VCFilter,
"ManagedObjectSelector": ManagedObjectSelector,
"handlers": handlers,
},
)
return handlers[0]
inv/models/interfaceprofile.py
View file @ 4101daf1
......@@ -87,7 +87,6 @@ class InterfaceProfileMetrics(EmbeddedDocument):
@on_delete_check(
check=[
("inv.Interface", "profile"),
("inv.InterfaceClassificationRule", "profile"),
("inv.SubInterface", "profile"),
("sa.ServiceProfile", "interface_profile"),
]
......
main/models/prefixtable.py
View file @ 4101daf1
......@@ -19,7 +19,6 @@ from noc.core.translation import ugettext as _
@on_delete_check(
check=[
# ("inv.InterfaceClassificationMatch", "prefix_table"),
("sa.ManagedObjectSelector", "filter_prefix"),
("main.Label", "match_prefixfilter.prefix_table"),
],
......
models.py
View file @ 4101daf1
......@@ -161,7 +161,6 @@ _MODELS = {
"inv.ForwardingInstance": "noc.inv.models.forwardinginstance.ForwardingInstance",
"inv.IfDescPatterns": "noc.inv.models.ifdescpatterns.IfDescPatterns",
"inv.Interface": "noc.inv.models.interface.Interface",
"inv.InterfaceClassificationRule": "noc.inv.models.interfaceclassificationrule.InterfaceClassificationRule",
"inv.InterfaceProfile": "noc.inv.models.interfaceprofile.InterfaceProfile",
"inv.Link": "noc.inv.models.link.Link",
"inv.MACBlacklist": "noc.inv.models.macblacklist.MACBlacklist",
......
sa/models/managedobjectselector.py
View file @ 4101daf1
......@@ -45,7 +45,6 @@ id_lock = Lock()
@on_delete
@on_delete_check(
check=[
("inv.InterfaceClassificationRule", "selector"),
("sa.ManagedObjectSelectorByAttribute", "selector"),
]
)
......
services/discovery/jobs/box/interface.py
View file @ 4101daf1
......@@ -69,7 +69,6 @@ class InterfaceCheck(PolicyDiscoveryCheck):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
# self.get_interface_profile = InterfaceClassificationRule.get_classificator()
self.get_interface_profile = partial(Label.get_instance_profile, InterfaceProfile)
# @todo bulk
self.confd_interface_profile_map = List[Tuple[str, InterfaceProfile]]
......@@ -185,8 +184,6 @@ class InterfaceCheck(PolicyDiscoveryCheck):
if_map[iface.name] = iface
# Delete hanging interfaces
self.seen_interfaces += [i["name"] for i in fi["interfaces"]]
# Classify interface
# self.interface_classification_confdb(if_map)
# Delete hanging interfaces
self.cleanup_interfaces(self.seen_interfaces)
# Delete hanging forwarding instances
......@@ -458,42 +455,6 @@ class InterfaceCheck(PolicyDiscoveryCheck):
if dsi:
dsi.delete()
def interface_classification_confdb(self, ifmap: Dict[str, Interface]):
"""
Perform interface classification by ConfDB
:param ifmap: Interface classification list
:return:
"""
from noc.inv.models.interfaceclassificationrule import InterfaceClassificationRule
self.logger.debug("Start interface classification")
proccessed = set()
selectors_skipping = set() # if selectors has not match
cdb = self.object.get_confdb()
# selector -> [(order, match, profile)]
for icr in InterfaceClassificationRule.objects.filter(is_active=True).order_by("order"):
if icr.selector.id in selectors_skipping:
continue
r = next(cdb.query(icr.selector.get_confdb_query), None)
if r is None:
# Selectors already fail check
selectors_skipping.add(icr.selector.id)
continue
for match in cdb.query(icr.get_confdb_query):
if match["ifname"] in proccessed or match["ifname"] not in ifmap:
continue
self.logger.debug("[%s] Match %s", icr, match["ifname"])
iface = ifmap[match["ifname"]]
proccessed.add(match["ifname"])
if iface.profile_locked:
continue
if icr.profile.id != iface.profile.id:
self.logger.info(
"Interface %s has been classified as '%s'", iface.name, icr.profile.name
)
iface.profile = icr.profile
iface.save()
def interface_classification(self, iface: "Interface"):
"""
Perform interface classification
......
services/web/apps/inv/interfaceclassificationrule/views.py deleted 100644 → 0
View file @ fb5afc66
# ---------------------------------------------------------------------
# inv.interfaceclassificationrule application
# ---------------------------------------------------------------------
# Copyright (C) 2007-2019 The NOC Project
# See LICENSE for details
# ---------------------------------------------------------------------
# NOC modules
from noc.lib.app.extdocapplication import ExtDocApplication
from noc.inv.models.interfaceclassificationrule import InterfaceClassificationRule
from noc.core.translation import ugettext as _
class InterfaceClassificationRuleApplication(ExtDocApplication):
"""
InterfaceClassificationRule application
"""
title = _("Interface Classification Rule")
menu = [_("Setup"), _("Interface Classification Rules")]
model = InterfaceClassificationRule
query_fields = ["name__icontains", "description__icontains"]
default_ordering = ["order"]
def field_row_class(self, o):
if o.profile and o.profile.style:
return o.profile.style.css_class_name
else:
return ""
def field_match_expr(self, o):
return o.match_expr
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment