Commit 21cd8d37 authored by Dmitry Volodin's avatar Dmitry Volodin
Browse files

ConfDB: Object Validation

parent 6027ce61
# -*- coding: utf-8 -*-
# ----------------------------------------------------------------------
# ConfDBQuery model
# ----------------------------------------------------------------------
# Copyright (C) 2007-2019 The NOC Project
# See LICENSE for details
# ----------------------------------------------------------------------
# Python modules
import threading
import operator
import os
# Third-party modules
from mongoengine.document import Document
from mongoengine.fields import StringField, UUIDField, BooleanField
import six
import cachetools
# NOC modules
from noc.lib.prettyjson import to_json
from noc.lib.text import quote_safe_path
from noc.core.model.decorator import on_delete_check
id_lock = threading.Lock()
@on_delete_check(check=[("cm.ObjectValidationPolicy", "rules.query")])
@six.python_2_unicode_compatible
class ConfDBQuery(Document):
meta = {
"collection": "confdbqueries",
"strict": True,
"auto_create_index": False,
"json_collection": "cm.confdbqueries",
"json_unique_fields": ["name"],
}
name = StringField(unique=True)
uuid = UUIDField(binary=True)
description = StringField()
source = StringField()
allow_object_validation = BooleanField(default=False)
allow_interface_validation = BooleanField(default=False)
allow_object_classification = BooleanField(default=False)
allow_interface_classification = BooleanField(default=False)
_id_cache = cachetools.TTLCache(maxsize=100, ttl=60)
def __str__(self):
return self.name
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_id_cache"), lock=lambda _: id_lock)
def get_by_id(cls, id):
return ConfDBQuery.objects.filter(id=id).first()
def get_json_path(self):
p = [quote_safe_path(n.strip()) for n in self.name.split("|")]
return os.path.join(*p) + ".json"
def query(self, engine, **kwargs):
"""
Run query against ConfDB engine
:param engine: ConfDB engine
:param kwargs: Optional arguments
:return:
"""
for ctx in engine.query(self.source, **kwargs):
yield ctx
def to_json(self):
r = {
"name": self.name,
"$collection": self._meta["json_collection"],
"uuid": self.uuid,
"source": self.source,
"allow_object_validation": self.allow_object_validation,
"allow_interface_validation": self.allow_interface_validation,
"allow_object_classification": self.allow_object_classification,
"allow_interface_classification": self.allow_interface_classification,
}
if self.description:
r["description"] = self.description
return to_json(
r,
order=[
"name",
"$collection",
"uuid",
"description",
"source",
"allow_object_validation",
"allow_interface_validation",
"allow_object_classification",
"allow_interface_classification",
],
)
# -*- coding: utf-8 -*-
# ----------------------------------------------------------------------
# ObjectValidationPolicy
# ----------------------------------------------------------------------
# Copyright (C) 2007-2019 The NOC Project
# See LICENSE for details
# ----------------------------------------------------------------------
# Python modules
from __future__ import absolute_import
import threading
import operator
# Third-party modules
from mongoengine.document import Document, EmbeddedDocument
from mongoengine.fields import StringField, BooleanField, ListField, EmbeddedDocumentField
from jinja2 import Template
import six
import cachetools
# NOC modules
from noc.core.mongo.fields import PlainReferenceField
from noc.core.model.decorator import on_delete_check
from noc.fm.models.alarmclass import AlarmClass
from .confdbquery import ConfDBQuery
id_lock = threading.Lock()
@six.python_2_unicode_compatible
class ObjectValidationRule(EmbeddedDocument):
query = PlainReferenceField(ConfDBQuery)
is_active = BooleanField(default=True)
error_code = StringField()
error_text_template = StringField(default="{{error}}")
alarm_class = PlainReferenceField(AlarmClass)
is_fatal = BooleanField(default=False)
def __str__(self):
return self.query.name
@on_delete_check(check=[("sa.ManageObjectProfile", "object_validation_policy")])
@six.python_2_unicode_compatible
class ObjectValidationPolicy(Document):
meta = {"collection": "objectvalidationpolicies", "strict": True, "auto_create_index": False}
name = StringField(unique=True)
description = StringField()
rules = ListField(EmbeddedDocumentField(ObjectValidationRule))
_id_cache = cachetools.TTLCache(maxsize=100, ttl=60)
def __str__(self):
return self.name
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_id_cache"), lock=lambda _: id_lock)
def get_by_id(cls, id):
return ObjectValidationPolicy.objects.filter(id=id).first()
def iter_problems(self, engine):
"""
Check rules agains ConfDB engine
:param engine: ConfDB Engine instance
:return: List of problems
"""
for rule in self.rules:
if not rule.is_active:
continue
for ctx in rule.query.query(engine):
if "error" in ctx:
tpl = Template(rule.error_text_template)
problem = {
"alarm_class": rule.alarm_class.name if rule.alarm_class else None,
"path": None,
"message": tpl.render(ctx),
"code": rule.error_code or None,
}
yield problem
if rule.is_fatal:
raise StopIteration
......@@ -37,6 +37,7 @@ class Command(BaseCommand):
"id",
"config",
"asset",
"configvalidation",
"vlan",
"nri",
"udld",
......
......@@ -239,9 +239,11 @@ _MODELS = {
"pm.MetricType": "noc.pm.models.metrictype.MetricType",
"pm.ThresholdProfile": "noc.pm.models.thresholdprofile.ThresholdProfile",
# cm models
"cm.ConfDBQuery": "noc.cm.models.confdbquery.ConfDBQuery",
"cm.ErrorType": "noc.cm.models.errortype.ErrorType",
"cm.ObjectFact": "noc.cm.models.objectfact.ObjectFact",
"cm.ObjectNotify": "noc.cm.models.objectnotify.ObjectNotify",
"cm.ObjectValidationPolicy": "noc.cm.models.objectvalidationpolicy.ObjectValidationPolicy",
"cm.ValidationPolicy": "noc.cm.models.validationpolicy.ValidationPolicy",
"cm.ValidationPolicySettings": "noc.cm.models.validationpolicysettings.ValidationPolicySettings",
"cm.ValidationRule": "noc.cm.models.validationrule.ValidationRule",
......@@ -356,4 +358,5 @@ COLLECTIONS = [
"fm.CloneClassificationRule",
"sa.ProfileCheckRule",
"bi.DashboardLayout",
"cm.ConfDBQuery",
]
# -*- coding: utf-8 -*-
# ----------------------------------------------------------------------
# ManagedObjectProfile.object_validation_policy
# ----------------------------------------------------------------------
# Copyright (C) 2007-2019 The NOC Project
# See LICENSE for details
# ----------------------------------------------------------------------
# NOC modules
from noc.core.migration.base import BaseMigration
from noc.core.model.fields import DocumentReferenceField
class Migration(BaseMigration):
def migrate(self):
self.db.add_column(
"sa_managedobjectprofile",
"object_validation_policy",
DocumentReferenceField("cm.ObjectValidationPolicy", null=True, blank=True),
)
......@@ -88,7 +88,7 @@ from .objectstatus import ObjectStatus
from .objectdata import ObjectData
# Increase whenever new field added or removed
MANAGEDOBJECT_CACHE_VERSION = 18
MANAGEDOBJECT_CACHE_VERSION = 19
Credentials = namedtuple(
"Credentials", ["user", "password", "super_password", "snmp_ro", "snmp_rw"]
......@@ -1132,23 +1132,35 @@ class ManagedObject(NOCModel):
except storage.Error as e:
logger.error("[%s] Failed to mirror config: %s", self.name, e)
def validate_config(self, changed):
def to_validate(self, changed):
"""
Apply config validation rules
:param changed:
:return:
Check if config is to be validated
:param changed: True if config has been changed
:return: Boolean
"""
logger.debug("[%s] Validating config", self.name)
policy = self.object_profile.config_validation_policy
# D - Disable
if policy == "D":
logger.debug("[%s] Validation is disabled by policy. Skipping", self.name)
return
# C - Mirror on Change
return False
# C - Validate on Change
if policy == "C" and not changed:
logger.debug("[%s] Configuration has not been changed. Skipping", self.name)
return False
return True
def validate_config(self, changed):
"""
Apply config validation rules (Legacy CLIPS path)
:param changed:
:return:
"""
logger.debug("[%s] Validating config (Legacy path)", self.name)
if not self.to_validate(changed):
return
# Validate
# Validate (Legacy Path)
from noc.cm.engine import Engine
engine = Engine(self)
......@@ -1158,6 +1170,23 @@ class ManagedObject(NOCModel):
logger.error("Failed to validate config for %s", self.name)
error_report()
def iter_validation_problems(self, changed):
"""
Yield validation problems
:param changed: True if config has been changed
:return:
"""
logger.debug("[%s] Validating config", self.name)
if not self.to_validate(changed):
return
if not self.object_profile.object_validation_policy:
logger.debug("[%s] Validation policy is not set. Skipping", self.name)
return
confdb = self.get_confdb()
for problem in self.object_profile.object_validation_policy.iter_problems(confdb):
yield problem
@property
def credentials(self):
"""
......
......@@ -38,6 +38,7 @@ from noc.vc.models.vpnprofile import VPNProfile
from noc.main.models.extstorage import ExtStorage
from noc.main.models.template import Template
from noc.core.datastream.decorator import datastream
from noc.cm.models.objectvalidationpolicy import ObjectValidationPolicy
from .authprofile import AuthProfile
from .capsprofile import CapsProfile
......@@ -470,6 +471,7 @@ class ManagedObjectProfile(NOCModel):
choices=[("D", "Disable"), ("A", "Always"), ("C", "Change")],
default="C",
)
object_validation_policy = DocumentReferenceField(ObjectValidationPolicy, null=True, blank=True)
# Interface discovery settings
interface_discovery_policy = models.CharField(
_("Interface Discovery Policy"),
......
......@@ -122,7 +122,9 @@ class MODiscoveryJob(PeriodicJob):
yield
self.check_timings += [(name, perf_counter() - t)]
def set_problem(self, check=None, alarm_class=None, path=None, message=None, fatal=False):
def set_problem(
self, check=None, alarm_class=None, path=None, message=None, fatal=False, **kwargs
):
"""
Set discovery problem
:param check: Check name
......@@ -131,8 +133,18 @@ class MODiscoveryJob(PeriodicJob):
:param message: Text message
:param fatal: True if problem is fatal and all following checks
must be disabled
:param kwargs: Optional variables
:return:
"""
self.logger.debug(
"[%s] Set problem: class=%s path=%s message=%s fatal=%s vars=%s",
check,
alarm_class,
path,
message,
fatal,
kwargs,
)
self.problems += [
{
"check": check,
......@@ -141,6 +153,7 @@ class MODiscoveryJob(PeriodicJob):
"path": str(path) if path else "",
"message": message,
"fatal": fatal,
"vars": kwargs,
}
]
if fatal:
......@@ -614,7 +627,7 @@ class DiscoveryCheck(object):
except ValueError as e:
self.logger.info("Failed to unlink: %s", e)
def set_problem(self, alarm_class=None, path=None, message=None, fatal=False):
def set_problem(self, alarm_class=None, path=None, message=None, fatal=False, **kwargs):
"""
Set discovery problem
:param alarm_class: Alarm class instance or name
......@@ -622,11 +635,17 @@ class DiscoveryCheck(object):
:param message: Text message
:param fatal: True if problem is fatal and all following checks
must be disabled
:param kwargs: Dict containing optional variables
:return:
"""
self.logger.info("Set path: %s" % path)
self.job.set_problem(
check=self.name, alarm_class=alarm_class, path=path, message=message, fatal=fatal
check=self.name,
alarm_class=alarm_class,
path=path,
message=message,
fatal=fatal,
**kwargs
)
def set_artefact(self, name, value=None):
......
......@@ -20,7 +20,18 @@ class ConfigValidationCheck(DiscoveryCheck):
def handler(self):
self.logger.info("Running config validation")
self.object.validate_config(self.get_artefact("config_changed") or False)
is_changed = self.get_artefact("config_changed") or False
# Legacy CLIPS path, problems are passed via Facts
self.object.validate_config(is_changed)
# New ConfDB path, problems are passed via alarms
n = 0
for problem in self.object.iter_validation_problems(is_changed):
self.set_problem(**problem)
n += 1
if n:
self.logger.info("%d problem(s) detected", n)
else:
self.logger.info("No problems detected")
def is_enabled(self):
checks = self.job.attrs.get("_checks", set())
......
# -*- coding: utf-8 -*-
# ----------------------------------------------------------------------
# cm.confdbquery application
# ----------------------------------------------------------------------
# Copyright (C) 2007-2019 The NOC Project
# See LICENSE for details
# ----------------------------------------------------------------------
# NOC modules
from noc.lib.app.extdocapplication import ExtDocApplication
from noc.cm.models.confdbquery import ConfDBQuery
from noc.core.translation import ugettext as _
class ConfDBQueryApplication(ExtDocApplication):
"""
ConfDBQuery application
"""
title = "ConfDBQuery"
menu = [_("Setup"), _("ConfDB Queries")]
model = ConfDBQuery
# -*- coding: utf-8 -*-
# ----------------------------------------------------------------------
# cm.objectvalidationpolicy application
# ----------------------------------------------------------------------
# Copyright (C) 2007-2019 The NOC Project
# See LICENSE for details
# ----------------------------------------------------------------------
# NOC modules
from noc.lib.app.extdocapplication import ExtDocApplication
from noc.cm.models.objectvalidationpolicy import ObjectValidationPolicy
from noc.core.translation import ugettext as _
class ObjectValidationPolicyApplication(ExtDocApplication):
"""
ObjectValidationPolicy application
"""
title = "ObjectValidationPolicy"
menu = [_("Setup"), _("Object Validation Policies")]
model = ObjectValidationPolicy
//---------------------------------------------------------------------
// cm.confdbquery application
//---------------------------------------------------------------------
// Copyright (C) 2007-2019 The NOC Project
// See LICENSE for details
//---------------------------------------------------------------------
console.debug("Defining NOC.cm.confdbquery.Application");
Ext.define("NOC.cm.confdbquery.Application", {
extend: "NOC.core.ModelApplication",
requires: [
"NOC.cm.confdbquery.Model"
],
model: "NOC.cm.confdbquery.Model",
search: true,
initComponent: function() {
var me = this;
me.jsonPanel = Ext.create("NOC.core.JSONPreview", {
app: me,
restUrl: new Ext.XTemplate('/cm/confdbquery/{id}/json/'),
previewName: new Ext.XTemplate('ConfDB Query: {name}')
});
me.ITEM_JSON = me.registerItem(me.jsonPanel);
Ext.apply(me, {
columns: [
{
text: __("Name"),
dataIndex: "name",
width: 150
}
],
fields: [
{
name: "name",
xtype: "textfield",
fieldLabel: __("Name"),
allowBlank: false,
uiStyle: "medium"
},
{
name: "uuid",
xtype: "displayfield",
fieldLabel: __("UUID"),
allowBlank: true
},
{
name: "description",
xtype: "textarea",
fieldLabel: __("Description"),
allowBlank: true
},
{
name: "source",
xtype: "cmtext",
fieldLabel: __("Source"),
allowBlank: false,
flex: 1,
mode: "python"
},
{
xtype: "fieldset",
title: __("Allow"),
layout: "hbox",
defaults: {
padding: 4
},
items: [
{
name: "allow_object_validation",
xtype: "checkbox",
boxLabel: __("Object Validation")
},
{
name: "allow_interface_validation",
xtype: "checkbox",
boxLabel: __("Interface Validation")
}
]
}
],
formToolbar: [
{
text: __("JSON"),
glyph: NOC.glyph.file,
tooltip: __("Show JSON"),
hasAccess: NOC.hasPermission("read"),
scope: me,
handler: me.onJSON
}
]
});
me.callParent();
},
//
onJSON: function() {
var me = this;
me.showItem(me.ITEM_JSON);
me.jsonPanel.preview(me.currentRecord);
}
});
\ No newline at end of file
//---------------------------------------------------------------------
// NOC.cm.confdbquery.Lookup
//---------------------------------------------------------------------
// Copyright (C) 2007-2019 The NOC Project
// See LICENSE for details
//---------------------------------------------------------------------
console.debug("Defining NOC.cm.confdbquery.LookupField");
Ext.define("NOC.cm.confdbquery.LookupField", {
extend: "NOC.core.LookupField",
alias: "widget.cm.confdbquery.LookupField",
uiStyle: "medium"
});
\ No newline at end of file
//---------------------------------------------------------------------
// cm.confdbquery Model
//---------------------------------------------------------------------
// Copyright (C) 2007-2019 The NOC Project
// See LICENSE for details
//---------------------------------------------------------------------
console.debug("Defining NOC.cm.confdbquery.Model");
Ext.define("NOC.cm.confdbquery.Model", {
extend: "Ext.data.Model",
rest_url: "/cm/confdbquery/",
fields: [
{
name: "id",
type: "string"
},
{
name: "allow_object_classification",
type: "boolean"
},