From 91c04f26e738d7199f6d6403c2616217342427d7 Mon Sep 17 00:00:00 2001 From: Andrey Vertiprahov Date: Fri, 12 Jul 2019 08:22:20 +0500 Subject: [PATCH 1/4] Fix refresh cfgping datastream when timepattern change and fix trace creating document without initial_data field. --- core/datastream/decorator.py | 2 +- main/models/timepattern.py | 10 ++++++++++ sa/models/managedobject.py | 5 ++++- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/core/datastream/decorator.py b/core/datastream/decorator.py index 089d91155b..10284cba67 100644 --- a/core/datastream/decorator.py +++ b/core/datastream/decorator.py @@ -54,7 +54,7 @@ def _on_model_change(sender, instance, *args, **kwargs): def _on_document_change(sender, document, *args, **kwargs): - _on_change(document, changed_fields=document._changed_fields) + _on_change(document, changed_fields=getattr(document, "_changed_fields", {})) def _on_change(obj, changed_fields=None): diff --git a/main/models/timepattern.py b/main/models/timepattern.py index 99fad6b376..58010990ee 100644 --- a/main/models/timepattern.py +++ b/main/models/timepattern.py @@ -20,10 +20,12 @@ from django.db import models from noc.core.model.base import NOCModel from noc.lib.timepattern import TimePattern as TP from noc.core.model.decorator import on_delete_check +from noc.core.datastream.decorator import datastream id_lock = Lock() +@datastream @on_delete_check( check=[ # ("fm.EscalationItem", "administrative_domain") @@ -88,6 +90,14 @@ class TimePattern(NOCModel): """ return self.time_pattern.match(d) + def iter_changed_datastream(self, changed_fields=None): + from noc.sa.models.managedobject import ManagedObject + + for mo in ManagedObject.objects.filter(time_pattern=self): + for c in mo.iter_changed_datastream( + changed_fields={"time_pattern_term"} + ): + yield c # Avoid circular references # No delete, fixed 'TimePattern' object has no attribute 'timepatternterm_set' diff --git a/sa/models/managedobject.py b/sa/models/managedobject.py index 758a2b6d4a..e0b7a508bb 100644 --- a/sa/models/managedobject.py +++ b/sa/models/managedobject.py @@ -531,7 +531,9 @@ class ManagedObject(NOCModel): def iter_changed_datastream(self, changed_fields=None): if ( config.datastream.enable_managedobject - and "managed_object_profile" not in changed_fields + and not changed_fields.intersection( + {"time_pattern_term", "managed_object_profile"} + ) ): yield "managedobject", self.id if config.datastream.enable_cfgping and changed_fields.intersection( @@ -544,6 +546,7 @@ class ManagedObject(NOCModel): "ping_count", "ping_timeout_ms", "report_ping_attempts", + "time_pattern_term", "event_processing_policy", } ): -- GitLab From d76de212caad816c1ddb6b216ffc4f28d239afba Mon Sep 17 00:00:00 2001 From: Andrey Vertiprahov Date: Tue, 16 Jul 2019 14:04:41 +0500 Subject: [PATCH 2/4] Fix subinterface changes on datastream. --- inv/models/subinterface.py | 4 +++ main/models/timepattern.py | 11 ------- main/models/timepatternterm.py | 13 ++++++++ sa/models/managedobject.py | 55 ++++++++++++++++++++----------- sa/models/managedobjectprofile.py | 2 +- 5 files changed, 54 insertions(+), 31 deletions(-) diff --git a/inv/models/subinterface.py b/inv/models/subinterface.py index 102cc288e1..c2f7f94c30 100644 --- a/inv/models/subinterface.py +++ b/inv/models/subinterface.py @@ -101,6 +101,10 @@ class SubInterface(Document): def iter_changed_datastream(self, changed_fields=None): if config.datastream.enable_managedobject: yield "managedobject", self.managed_object.id + if config.datastream.enable_cfgsyslog and "ipv4_addresses" in changed_fields: + yield "cfgsyslog", self.managed_object.id + if config.datastream.enable_cfgtrap and "ipv4_addresses" in changed_fields: + yield "cfgtrap", self.managed_object.id @property def effective_vc_domain(self): diff --git a/main/models/timepattern.py b/main/models/timepattern.py index 58010990ee..b6a3b3df9a 100644 --- a/main/models/timepattern.py +++ b/main/models/timepattern.py @@ -20,12 +20,10 @@ from django.db import models from noc.core.model.base import NOCModel from noc.lib.timepattern import TimePattern as TP from noc.core.model.decorator import on_delete_check -from noc.core.datastream.decorator import datastream id_lock = Lock() -@datastream @on_delete_check( check=[ # ("fm.EscalationItem", "administrative_domain") @@ -90,15 +88,6 @@ class TimePattern(NOCModel): """ return self.time_pattern.match(d) - def iter_changed_datastream(self, changed_fields=None): - from noc.sa.models.managedobject import ManagedObject - - for mo in ManagedObject.objects.filter(time_pattern=self): - for c in mo.iter_changed_datastream( - changed_fields={"time_pattern_term"} - ): - yield c - # Avoid circular references # No delete, fixed 'TimePattern' object has no attribute 'timepatternterm_set' from .timepatternterm import TimePatternTerm # noqa diff --git a/main/models/timepatternterm.py b/main/models/timepatternterm.py index 33a549a7ae..717383bb20 100644 --- a/main/models/timepatternterm.py +++ b/main/models/timepatternterm.py @@ -16,9 +16,13 @@ from django.db import models # NOC modules from noc.core.model.base import NOCModel from noc.lib.timepattern import TimePattern as TP +from noc.core.model.decorator import on_init, on_delete_check +from noc.core.datastream.decorator import datastream from .timepattern import TimePattern +@on_init +@datastream @six.python_2_unicode_compatible class TimePatternTerm(NOCModel): """ @@ -53,3 +57,12 @@ class TimePatternTerm(NOCModel): """ TimePatternTerm.check_syntax(self.term) super(TimePatternTerm, self).save(*args, **kwargs) + + def iter_changed_datastream(self, changed_fields=None): + from noc.sa.models.managedobject import ManagedObject + + for mo in ManagedObject.objects.filter(time_pattern=self.time_pattern): + for c in mo.iter_changed_datastream( + changed_fields=changed_fields + ): + yield c diff --git a/sa/models/managedobject.py b/sa/models/managedobject.py index e0b7a508bb..3c079f1f5a 100644 --- a/sa/models/managedobject.py +++ b/sa/models/managedobject.py @@ -529,39 +529,56 @@ class ManagedObject(NOCModel): return None def iter_changed_datastream(self, changed_fields=None): - if ( - config.datastream.enable_managedobject - and not changed_fields.intersection( - {"time_pattern_term", "managed_object_profile"} - ) - ): + if config.datastream.enable_managedobject: yield "managedobject", self.id if config.datastream.enable_cfgping and changed_fields.intersection( { - "report_ping_rtt", - "enable_ping", - "ping_interval", - "ping_policy", - "ping_size", - "ping_count", - "ping_timeout_ms", - "report_ping_attempts", - "time_pattern_term", + "name", + "bi_id", + "is_managed", + "pool", + "address", + "time_pattern", "event_processing_policy", + "object_profile__enable_ping", + "object_profile__ping_interval", + "object_profile__ping_policy", + "object_profile__ping_size", + "object_profile__ping_count", + "object_profile__ping_timeout_ms", + "object_profile__report_ping_rtt", + "object_profile__report_ping_attempts", + "object_profile__event_processing_policy", } ): yield "cfgping", self.id if config.datastream.enable_cfgsyslog and changed_fields.intersection( { + "name", + "bi_id", + "is_managed", + "pool", + "address", "event_processing_policy", - "syslog_archive_policy", - "syslog_source_type", - "syslog_source_ip", + "object_profile__event_processing_policy", + "object_profile__syslog_archive_policy", + "object_profile__syslog_source_type", + "object_profile__syslog_source_ip", } ): yield "cfgsyslog", self.id if config.datastream.enable_cfgtrap and changed_fields.intersection( - {"event_processing_policy", "trap_source_type", "trap_source_ip"} + { + "name", + "bi_id", + "is_managed", + "pool", + "address", + "event_processing_policy", + "object_profile__event_processing_policy", + "object_profile__trap_source_type", + "object_profile__trap_source_ip", + } ): yield "cfgtrap", self.id diff --git a/sa/models/managedobjectprofile.py b/sa/models/managedobjectprofile.py index 3279386e4f..5e2c7c7ab4 100644 --- a/sa/models/managedobjectprofile.py +++ b/sa/models/managedobjectprofile.py @@ -574,7 +574,7 @@ class ManagedObjectProfile(NOCModel): for mo in ManagedObject.objects.filter(object_profile=self): for c in mo.iter_changed_datastream( - changed_fields={"managed_object_profile"}.union(changed_fields) + changed_fields={"object_profile__%s" % f for f in changed_fields} ): yield c -- GitLab From 2e306267380c472607ecd8caca22817c7d696cb8 Mon Sep 17 00:00:00 2001 From: Andrey Vertiprahov Date: Tue, 16 Jul 2019 14:16:09 +0500 Subject: [PATCH 3/4] Fix black. --- main/models/timepattern.py | 1 + main/models/timepatternterm.py | 6 ++---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/main/models/timepattern.py b/main/models/timepattern.py index b6a3b3df9a..99fad6b376 100644 --- a/main/models/timepattern.py +++ b/main/models/timepattern.py @@ -88,6 +88,7 @@ class TimePattern(NOCModel): """ return self.time_pattern.match(d) + # Avoid circular references # No delete, fixed 'TimePattern' object has no attribute 'timepatternterm_set' from .timepatternterm import TimePatternTerm # noqa diff --git a/main/models/timepatternterm.py b/main/models/timepatternterm.py index 717383bb20..d4ab14f875 100644 --- a/main/models/timepatternterm.py +++ b/main/models/timepatternterm.py @@ -16,7 +16,7 @@ from django.db import models # NOC modules from noc.core.model.base import NOCModel from noc.lib.timepattern import TimePattern as TP -from noc.core.model.decorator import on_init, on_delete_check +from noc.core.model.decorator import on_init from noc.core.datastream.decorator import datastream from .timepattern import TimePattern @@ -62,7 +62,5 @@ class TimePatternTerm(NOCModel): from noc.sa.models.managedobject import ManagedObject for mo in ManagedObject.objects.filter(time_pattern=self.time_pattern): - for c in mo.iter_changed_datastream( - changed_fields=changed_fields - ): + for c in mo.iter_changed_datastream(changed_fields=changed_fields): yield c -- GitLab From c8520b096d18d71c9741954590755c765c413aa3 Mon Sep 17 00:00:00 2001 From: Andrey Vertiprahov Date: Tue, 16 Jul 2019 15:06:36 +0500 Subject: [PATCH 4/4] Fix Black. --- sa/models/managedobject.py | 21 ++++------------ sa/models/managedobjectprofile.py | 42 ++++++++++++++++++++++++++----- 2 files changed, 41 insertions(+), 22 deletions(-) diff --git a/sa/models/managedobject.py b/sa/models/managedobject.py index 3c079f1f5a..d311e23ca0 100644 --- a/sa/models/managedobject.py +++ b/sa/models/managedobject.py @@ -540,15 +540,6 @@ class ManagedObject(NOCModel): "address", "time_pattern", "event_processing_policy", - "object_profile__enable_ping", - "object_profile__ping_interval", - "object_profile__ping_policy", - "object_profile__ping_size", - "object_profile__ping_count", - "object_profile__ping_timeout_ms", - "object_profile__report_ping_rtt", - "object_profile__report_ping_attempts", - "object_profile__event_processing_policy", } ): yield "cfgping", self.id @@ -560,10 +551,9 @@ class ManagedObject(NOCModel): "pool", "address", "event_processing_policy", - "object_profile__event_processing_policy", - "object_profile__syslog_archive_policy", - "object_profile__syslog_source_type", - "object_profile__syslog_source_ip", + "syslog_archive_policy", + "syslog_source_type", + "syslog_source_ip", } ): yield "cfgsyslog", self.id @@ -575,9 +565,8 @@ class ManagedObject(NOCModel): "pool", "address", "event_processing_policy", - "object_profile__event_processing_policy", - "object_profile__trap_source_type", - "object_profile__trap_source_ip", + "trap_source_type", + "trap_source_ip", } ): yield "cfgtrap", self.id diff --git a/sa/models/managedobjectprofile.py b/sa/models/managedobjectprofile.py index 5e2c7c7ab4..6e3fadc332 100644 --- a/sa/models/managedobjectprofile.py +++ b/sa/models/managedobjectprofile.py @@ -19,6 +19,7 @@ import cachetools # NOC modules from noc.core.model.base import NOCModel +from noc.config import config from noc.main.models.style import Style from noc.core.stencil import stencil_registry from noc.core.model.fields import TagsField, PickledField, DocumentReferenceField @@ -572,11 +573,42 @@ class ManagedObjectProfile(NOCModel): def iter_changed_datastream(self, changed_fields=None): from noc.sa.models.managedobject import ManagedObject - for mo in ManagedObject.objects.filter(object_profile=self): - for c in mo.iter_changed_datastream( - changed_fields={"object_profile__%s" % f for f in changed_fields} + if config.datastream.enable_managedobject and changed_fields.intersection( + {"name", "remote_system", "remote_id"} + ): + for mo_id in ManagedObject.objects.filter(object_profile=self).values_list( + "id", flat=True + ): + yield "managedobject", mo_id + if config.datastream.enable_cfgping and changed_fields.intersection( + { + "enable_ping", + "ping_interval", + "ping_policy", + "ping_size", + "ping_count", + "ping_timeout_ms", + "report_ping_rtt", + "report_ping_attempts", + "event_processing_policy", + } + ): + for mo_id in ManagedObject.objects.filter(object_profile=self).values_list( + "id", flat=True + ): + yield "cfgping", mo_id + if config.datastream.enable_cfgsyslog and changed_fields.intersection( + {"event_processing_policy", "syslog_archive_policy"} + ): + for mo_id in ManagedObject.objects.filter(object_profile=self).values_list( + "id", flat=True + ): + yield "cfgsyslog", mo_id + if config.datastream.enable_cfgtrap and "event_processing_policy" in changed_fields: + for mo_id in ManagedObject.objects.filter(object_profile=self).values_list( + "id", flat=True ): - yield c + yield "cfgtrap", mo_id def iter_pools(self): """ @@ -601,7 +633,6 @@ class ManagedObjectProfile(NOCModel): box_changed=box_changed, periodic_changed=periodic_changed, ) - if access_changed: cache.delete_many( ["cred-%s" % x for x in self.managedobject_set.values_list("id", flat=True)] @@ -669,7 +700,6 @@ def apply_discovery_jobs(profile_id, box_changed, periodic_changed): profile = ManagedObjectProfile.objects.get(id=profile_id) except ManagedObjectProfile.DoesNotExist: return - for mo_id, is_managed, pool in iter_objects(): if box_changed: if profile.enable_box_discovery and is_managed: -- GitLab