From 47b02271b950c9572d9824fb5274d831471c2525 Mon Sep 17 00:00:00 2001 From: Dmitry Volodin Date: Sat, 24 Sep 2022 10:09:16 +0200 Subject: [PATCH 1/8] Cleanup CDAG subgraph implementation --- core/cdag/node/subgraph.py | 20 ++++++++++---------- tests/cdag/node/test_subgraph.py | 4 ++-- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/core/cdag/node/subgraph.py b/core/cdag/node/subgraph.py index 386e72b899..07b29e92f4 100644 --- a/core/cdag/node/subgraph.py +++ b/core/cdag/node/subgraph.py @@ -17,7 +17,7 @@ import yaml # NOC modules from ..graph import CDAG from ..factory.config import ConfigCDAGFactory, GraphConfig -from .base import BaseCDAGNode, ValueType, Category +from .base import BaseCDAGNode, ValueType, Category, IN_INVALID, IN_REQUIRED class InputMapping(BaseModel): @@ -121,7 +121,6 @@ class SubgraphNode(BaseCDAGNode): if not node: continue self.input_mappings[m.public_name] = InputItem(node=node, input=m.name) - self.static_inputs.add(m.public_name) # Inject measure node when necessary self.measure_node = None if cfg.output: @@ -140,11 +139,10 @@ class SubgraphNode(BaseCDAGNode): def iter_inputs(self) -> Iterable[str]: yield from self.input_mappings - def has_input(self, name: str) -> bool: - return name in self.input_mappings - - def is_required_input(self, name: str) -> bool: - return name in self.input_mappings + def get_input_type(self, name: str) -> int: + if name in self.input_mappings: + return IN_REQUIRED + return IN_INVALID def get_value(self, *args, **kwargs) -> Optional[ValueType]: # Start sub-transaction @@ -153,8 +151,10 @@ class SubgraphNode(BaseCDAGNode): im = self.input_mappings[p] im.node.activate(tx, im.input, v) changed_state = tx.get_changed_state() - if self.state.state is None: - self.state.state = {} - self.state.state.update(changed_state) + if changed_state: + if self.state.state: + self.state.state.update(changed_state) + else: + self.state.state = changed_state.copy() out = tx.get_inputs(self.measure_node) return out.get("x") # None node has `x` input diff --git a/tests/cdag/node/test_subgraph.py b/tests/cdag/node/test_subgraph.py index 01e24dc6a6..491a41573b 100644 --- a/tests/cdag/node/test_subgraph.py +++ b/tests/cdag/node/test_subgraph.py @@ -1,7 +1,7 @@ # ---------------------------------------------------------------------- # subgraph node test # ---------------------------------------------------------------------- -# Copyright (C) 2007-2021 The NOC Project +# Copyright (C) 2007-2022 The NOC Project # See LICENSE for details # ---------------------------------------------------------------------- @@ -69,7 +69,7 @@ def test_config(): "x,y,expected", [(0, 0, 0), (1, 0, 2), (0, 1, 1), (2, 1, 5)], ) -def test_subgrapg_node(x, y, expected): +def test_subgraph_node(x, y, expected): cdag = NodeCDAG("subgraph", config=CONFIG) assert cdag.is_activated() is False cdag.activate("x", x) -- GitLab From 75fe2aa558e106944e3e804e70458d2f7384a0e9 Mon Sep 17 00:00:00 2001 From: Andrey Vertiprahov Date: Sat, 24 Sep 2022 14:36:55 +0500 Subject: [PATCH 2/8] Fix collection config. --- collections/pm.measurementunits/DuplexStatus.json | 4 ++++ .../pm.measurementunits/InterfaceStatus.json | 4 ++++ .../pm.metricactions/Average_CPU_Usage.json | 14 ++++++++++---- .../pm.metricactions/Average_Memory_Usage.json | 4 ++-- .../Interface_Bandwidth_more_than_N.json | 4 ++-- .../pm.metricactions/Interface_Errors_In.json | 10 ++++++++-- .../0012_migrate_threshold_profile_metic_rules.py | 4 ++-- services/metrics/service.py | 8 +++----- 8 files changed, 35 insertions(+), 17 deletions(-) diff --git a/collections/pm.measurementunits/DuplexStatus.json b/collections/pm.measurementunits/DuplexStatus.json index e5da4c85a3..6e26577287 100644 --- a/collections/pm.measurementunits/DuplexStatus.json +++ b/collections/pm.measurementunits/DuplexStatus.json @@ -6,6 +6,10 @@ "description": "Interface Duplex state", "label": " ", "dashboard_label": " ", + "convert_from": [{ + "expr": "x", + "unit__code": "1" + }], "enum": [ { "key": "Unknown", diff --git a/collections/pm.measurementunits/InterfaceStatus.json b/collections/pm.measurementunits/InterfaceStatus.json index 715fe9513b..5a21c734c9 100644 --- a/collections/pm.measurementunits/InterfaceStatus.json +++ b/collections/pm.measurementunits/InterfaceStatus.json @@ -6,6 +6,10 @@ "description": "Operational state of the interface", "label": " ", "dashboard_label": null, + "convert_from": [{ + "expr": "x", + "unit__code": "1" + }], "enum": [ { "key": "up", diff --git a/collections/pm.metricactions/Average_CPU_Usage.json b/collections/pm.metricactions/Average_CPU_Usage.json index 1988887747..b50936cfdf 100644 --- a/collections/pm.metricactions/Average_CPU_Usage.json +++ b/collections/pm.metricactions/Average_CPU_Usage.json @@ -5,17 +5,17 @@ "description": "Средняя загрузка CPU больше N", "params": [ { - "name": "activation.min_window", + "name": "activation-window.min_window", "description": null, - "default": "5", + "default": "3", "max_value": 5.0, "min_value": 3.0, "type": "int" }, { - "name": "activation.max_window", + "name": "activation-window.max_window", "description": null, - "default": "3", + "default": "5", "max_value": 5.0, "min_value": 3.0, "type": "int" @@ -47,5 +47,11 @@ "window_config": {}, "window_function": "mean", "window_type": "tick" + }, + "alarm_config": { + "activation_level": 1.0, + "alarm_class": "Metric | Object | High CPU Usage", + "deactivation_level": 1.0, + "reference": null } } diff --git a/collections/pm.metricactions/Average_Memory_Usage.json b/collections/pm.metricactions/Average_Memory_Usage.json index c4cc1465c6..b395ca618b 100644 --- a/collections/pm.metricactions/Average_Memory_Usage.json +++ b/collections/pm.metricactions/Average_Memory_Usage.json @@ -21,7 +21,7 @@ "type": "int" }, { - "name": "activation.min_window", + "name": "activation-window.min_window", "description": null, "default": "3", "max_value": 10.0, @@ -29,7 +29,7 @@ "type": "int" }, { - "name": "activation.max_window", + "name": "activation-window.max_window", "description": null, "default": "5", "max_value": 10.0, diff --git a/collections/pm.metricactions/Interface_Bandwidth_more_than_N.json b/collections/pm.metricactions/Interface_Bandwidth_more_than_N.json index fc9737005e..e6c3dfaf8b 100644 --- a/collections/pm.metricactions/Interface_Bandwidth_more_than_N.json +++ b/collections/pm.metricactions/Interface_Bandwidth_more_than_N.json @@ -21,7 +21,7 @@ "type": "float" }, { - "name": "activation.min_window", + "name": "activation-window.min_window", "description": null, "default": "900", "max_value": 300.0, @@ -29,7 +29,7 @@ "type": "int" }, { - "name": "activation.max_window", + "name": "activation-window.max_window", "description": null, "default": "1200", "max_value": 300.0, diff --git a/collections/pm.metricactions/Interface_Errors_In.json b/collections/pm.metricactions/Interface_Errors_In.json index 247e21ae18..b8af194308 100644 --- a/collections/pm.metricactions/Interface_Errors_In.json +++ b/collections/pm.metricactions/Interface_Errors_In.json @@ -21,7 +21,7 @@ "type": "float" }, { - "name": "activation.min_window", + "name": "activation-window.min_window", "description": null, "default": "900", "max_value": 3600.0, @@ -29,7 +29,7 @@ "type": "int" }, { - "name": "activation.max_window", + "name": "activation-window.max_window", "description": null, "default": "1200", "max_value": 3600.0, @@ -51,5 +51,11 @@ }, "window_function": "percentile", "window_type": "seconds" + }, + "alarm_config": { + "activation_level": 1.0, + "alarm_class": "Metric | Interface | Error", + "deactivation_level": 1.0, + "reference": null } } diff --git a/pm/migrations/0012_migrate_threshold_profile_metic_rules.py b/pm/migrations/0012_migrate_threshold_profile_metic_rules.py index 102488c5a7..169ca99ea3 100644 --- a/pm/migrations/0012_migrate_threshold_profile_metic_rules.py +++ b/pm/migrations/0012_migrate_threshold_profile_metic_rules.py @@ -208,8 +208,8 @@ class Migration(BaseMigration): if "invert_condition" in ac: params["alarm.invert_condition"] = ac["invert_condition"] if wc: - params["activation.max_window"] = wc["max_window"] - params["activation.min_window"] = wc["min_window"] + params["activation-window.max_window"] = wc["max_window"] + params["activation-window.min_window"] = wc["min_window"] mr_bulk += [ InsertOne( { diff --git a/services/metrics/service.py b/services/metrics/service.py index 8b972c482e..7dc677437b 100755 --- a/services/metrics/service.py +++ b/services/metrics/service.py @@ -287,8 +287,8 @@ class MetricsService(FastAPIService): asyncio.get_running_loop().create_task(self.subscribe_metrics()) async def subscribe_metrics(self) -> None: - self.logger.info("Waiting for mappings") - await self.mappings_ready_event.wait() + self.logger.info("Waiting for rules") + await self.rules_ready_event.wait() self.logger.info("Mappings are ready") await self.subscribe_stream("metrics", self.slot_number, self.on_metrics, async_cursor=True) @@ -780,11 +780,9 @@ class MetricsService(FastAPIService): if not mu: continue # Missed field probe = card.probes.get(n) - if probe.name == ComposeProbeNode.name: # Skip composed probe - continue if self.lazy_init and not probe: probe = self.add_probe(n, k) - if not probe: + if not probe or probe.name == ComposeProbeNode.name: # Skip composed probe continue probe.activate(tx, "ts", ts) probe.activate(tx, "x", data[n]) -- GitLab From acac2788baf58d847f4bd91cadd65a62e555c513 Mon Sep 17 00:00:00 2001 From: Andrey Vertiprahov Date: Sat, 24 Sep 2022 14:54:24 +0500 Subject: [PATCH 3/8] Fix typo. --- collections/pm.metricactions/Interface_Errors_In.json | 2 +- pm/models/metricaction.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/collections/pm.metricactions/Interface_Errors_In.json b/collections/pm.metricactions/Interface_Errors_In.json index b8af194308..4c30280fd9 100644 --- a/collections/pm.metricactions/Interface_Errors_In.json +++ b/collections/pm.metricactions/Interface_Errors_In.json @@ -54,7 +54,7 @@ }, "alarm_config": { "activation_level": 1.0, - "alarm_class": "Metric | Interface | Error", + "alarm_class__name": "Metric | Interface | Error", "deactivation_level": 1.0, "reference": null } diff --git a/pm/models/metricaction.py b/pm/models/metricaction.py index 6a96bb9814..73deb8e4ce 100644 --- a/pm/models/metricaction.py +++ b/pm/models/metricaction.py @@ -106,7 +106,7 @@ class AlarmConfig(EmbeddedDocument): @property def json_data(self) -> Dict[str, Any]: return { - "alarm_class": self.alarm_class.name, + "alarm_class__name": self.alarm_class.name, "reference": self.reference, "activation_level": self.activation_level, "deactivation_level": self.deactivation_level, -- GitLab From c02375be94e46988a3de3205b84f802c0223849d Mon Sep 17 00:00:00 2001 From: Andrey Vertiprahov Date: Sat, 24 Sep 2022 19:23:06 +0500 Subject: [PATCH 4/8] Fix typo. --- collections/pm.metricactions/Average_CPU_Usage.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/collections/pm.metricactions/Average_CPU_Usage.json b/collections/pm.metricactions/Average_CPU_Usage.json index b50936cfdf..6f95df7ec2 100644 --- a/collections/pm.metricactions/Average_CPU_Usage.json +++ b/collections/pm.metricactions/Average_CPU_Usage.json @@ -50,7 +50,7 @@ }, "alarm_config": { "activation_level": 1.0, - "alarm_class": "Metric | Object | High CPU Usage", + "alarm_class__name": "Metric | Object | High CPU Usage", "deactivation_level": 1.0, "reference": null } -- GitLab From 4a3f4ee61ca64d34192b9cd5ae840a58361a9ea1 Mon Sep 17 00:00:00 2001 From: Andrey Vertiprahov Date: Sat, 24 Sep 2022 19:54:21 +0500 Subject: [PATCH 5/8] Fix division by zero on probe node. --- collections/pm.metricactions/Average_CPU_Usage.json | 3 --- collections/pm.metricactions/Interface_Errors_In.json | 2 -- core/cdag/node/probe.py | 4 ++++ 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/collections/pm.metricactions/Average_CPU_Usage.json b/collections/pm.metricactions/Average_CPU_Usage.json index 6f95df7ec2..7b9618f786 100644 --- a/collections/pm.metricactions/Average_CPU_Usage.json +++ b/collections/pm.metricactions/Average_CPU_Usage.json @@ -49,9 +49,6 @@ "window_type": "tick" }, "alarm_config": { - "activation_level": 1.0, - "alarm_class__name": "Metric | Object | High CPU Usage", - "deactivation_level": 1.0, "reference": null } } diff --git a/collections/pm.metricactions/Interface_Errors_In.json b/collections/pm.metricactions/Interface_Errors_In.json index 4c30280fd9..a2034316c1 100644 --- a/collections/pm.metricactions/Interface_Errors_In.json +++ b/collections/pm.metricactions/Interface_Errors_In.json @@ -53,9 +53,7 @@ "window_type": "seconds" }, "alarm_config": { - "activation_level": 1.0, "alarm_class__name": "Metric | Interface | Error", - "deactivation_level": 1.0, "reference": null } } diff --git a/core/cdag/node/probe.py b/core/cdag/node/probe.py index b98af9fd3a..a4efa1d393 100644 --- a/core/cdag/node/probe.py +++ b/core/cdag/node/probe.py @@ -114,6 +114,10 @@ class ProbeNode(BaseCDAGNode): # No previous measurement, store state and exit self.set_state(ts, x) return None + if (ts - self.state.lt) < NS: + # Too less timestamp different, Division by zero exception + logger.info("[%s] Skipping already processed value", self.node_id) + return None if ts <= self.state.lt: # Timer stepback, reset state and exit self.set_state(None, None) -- GitLab From 8cc26743bc77b55ddddda8c5d49dc82266255a11 Mon Sep 17 00:00:00 2001 From: Andrey Vertiprahov Date: Sat, 24 Sep 2022 20:31:19 +0500 Subject: [PATCH 6/8] Fix typo. --- core/cdag/node/probe.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/cdag/node/probe.py b/core/cdag/node/probe.py index a4efa1d393..c4ad13894e 100644 --- a/core/cdag/node/probe.py +++ b/core/cdag/node/probe.py @@ -114,14 +114,14 @@ class ProbeNode(BaseCDAGNode): # No previous measurement, store state and exit self.set_state(ts, x) return None - if (ts - self.state.lt) < NS: - # Too less timestamp different, Division by zero exception - logger.info("[%s] Skipping already processed value", self.node_id) - return None if ts <= self.state.lt: # Timer stepback, reset state and exit self.set_state(None, None) return None + elif (ts - self.state.lt) < NS: + # Too less timestamp different, Division by zero exception + logger.info("[%s] Skipping already processed value", self.node_id) + return None if fn.has_time_delta: kwargs["time_delta"] = (ts - self.state.lt) // NS # Always positive if fn.has_delta: -- GitLab From ac0fc1e0f98976c467a1ed6c68f91d3918bd7e57 Mon Sep 17 00:00:00 2001 From: Andrey Vertiprahov Date: Sat, 24 Sep 2022 21:06:11 +0500 Subject: [PATCH 7/8] Fix MetricAction json data. --- pm/models/metricaction.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/pm/models/metricaction.py b/pm/models/metricaction.py index 73deb8e4ce..ae11c203c7 100644 --- a/pm/models/metricaction.py +++ b/pm/models/metricaction.py @@ -105,12 +105,18 @@ class AlarmConfig(EmbeddedDocument): @property def json_data(self) -> Dict[str, Any]: - return { - "alarm_class__name": self.alarm_class.name, - "reference": self.reference, - "activation_level": self.activation_level, - "deactivation_level": self.deactivation_level, - } + r = {} + if self.alarm_class: + r["alarm_class"] = self.alarm_class.name + if self.reference: + r["reference"] = self.reference + if self.activation_level != 1.0: + r["activation_level"] = self.activation_level + if self.deactivation_level != 1.0: + r["deactivation_level"] = self.deactivation_level + if self.invert_condition: + r["invert_condition"] = self.invert_condition + return r class ActivationConfig(EmbeddedDocument): -- GitLab From 5931acf5fccf9d467d58561d1c17f4bc4c9ad034 Mon Sep 17 00:00:00 2001 From: Andrey Vertiprahov Date: Sat, 24 Sep 2022 21:39:54 +0500 Subject: [PATCH 8/8] Remove Alarm Class. --- collections/pm.metricactions/Interface_Errors_In.json | 1 - 1 file changed, 1 deletion(-) diff --git a/collections/pm.metricactions/Interface_Errors_In.json b/collections/pm.metricactions/Interface_Errors_In.json index a2034316c1..75cab0cb9e 100644 --- a/collections/pm.metricactions/Interface_Errors_In.json +++ b/collections/pm.metricactions/Interface_Errors_In.json @@ -53,7 +53,6 @@ "window_type": "seconds" }, "alarm_config": { - "alarm_class__name": "Metric | Interface | Error", "reference": null } } -- GitLab