Verified Commit 8ece6710 authored by Andrey Vertiprahov's avatar Andrey Vertiprahov
Browse files

Add capabilities to ManagedObject ETL extractor.

parent abb8722f
......@@ -83,9 +83,11 @@ class BaseLoader(object):
workflow_add_event = "seen"
workflow_delete_event = "missed"
# Label
label_enable_setting = None
label_enable_setting: Optional[str] = None
# Incremental
checkpoint_field = "checkpoint"
# Post save fields (example - capabilities)
post_save_fields: Optional[Set[str]] = None
REPORT_INTERVAL = 1000
......@@ -423,6 +425,11 @@ class BaseLoader(object):
del v[self.checkpoint_field]
for fn in set(v).intersection(self.workflow_fields):
del v[fn]
# Post save update fields (example capabilities)
psf: Dict[str, Any] = {}
if self.post_save_fields:
for fn in self.post_save_fields:
psf[fn] = v.pop(fn)
o = self.find_object(v)
if o:
self.c_change += 1
......@@ -444,6 +451,8 @@ class BaseLoader(object):
self.change_workflow(
o, getattr(item, "state", None), getattr(item, "state_changed", None)
)
if psf:
self.post_save(o, psf)
self.set_mappings(item.id, o.id)
def on_change(self, o: BaseModel, n: BaseModel):
......@@ -456,9 +465,14 @@ class BaseLoader(object):
changes = {"remote_system": nv["remote_system"], "remote_id": nv["remote_id"]}
incremental_changes = {}
ov = self.clean(o)
# Post save update fields (example capabilities)
psf: Dict[str, Any] = {}
for fn in self.data_model.__fields__:
if fn == "id" or fn in self.workflow_fields:
continue
if self.post_save_fields and fn in self.post_save_fields:
psf[fn] = nv.pop(fn)
continue
if ov[fn] != nv[fn]:
self.logger.debug(" %s: %s -> %s", fn, ov[fn], nv[fn])
changes[fn] = nv[fn]
......@@ -473,6 +487,8 @@ class BaseLoader(object):
self.change_workflow(
o, getattr(n, "state", None), getattr(n, "state_changed", None)
)
if psf:
self.post_save(o, psf)
else:
self.logger.error("Cannot map id '%s'. Skipping.", n.id)
......@@ -552,6 +568,15 @@ class BaseLoader(object):
mdata = "\n".join("%s,%s" % (k, self.mappings[k]) for k in sorted(self.mappings))
safe_rewrite(self.mappings_path, mdata)
def post_save(self, o, fields: Dict[str, Any]):
"""
Method fields updated separate method (example - capabilities)
:param o:
:param fields:
:return:
"""
return
def clean(self, item: BaseModel) -> Dict[str, Any]:
"""
Cleanup row and return a dict of field name -> value
......
......@@ -5,6 +5,9 @@
# See LICENSE for details
# ----------------------------------------------------------------------
# Python modules
from typing import Dict, Any
# NOC modules
from .base import BaseLoader
from ..models.managedobject import ManagedObject
......@@ -12,6 +15,7 @@ from noc.main.models.pool import Pool
from noc.sa.models.managedobject import ManagedObject as ManagedObjectModel
from noc.sa.models.profile import Profile
from noc.inv.models.resourcegroup import ResourceGroup
from noc.inv.models.capability import Capability
class ManagedObjectLoader(BaseLoader):
......@@ -22,6 +26,7 @@ class ManagedObjectLoader(BaseLoader):
name = "managedobject"
model = ManagedObjectModel
data_model = ManagedObject
post_save_fields = {"capabilities"}
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
......@@ -34,6 +39,7 @@ class ManagedObjectLoader(BaseLoader):
self.clean_map["static_client_groups"] = lambda x: [
str(x.id) for x in ResourceGroup.objects.filter(remote_id__in=x)
]
self.available_caps = {x.name for x in Capability.objects.filter()}
def purge(self):
"""
......@@ -50,3 +56,15 @@ class ManagedObjectLoader(BaseLoader):
except self.model.DoesNotExist:
pass # Already deleted
self.pending_deletes = []
def post_save(self, o: ManagedObjectModel, fields: Dict[str, Any]):
if not fields or "capabilities" not in fields:
return
caps = {}
for cc in fields["capabilities"]:
c_name = f"ETL | {self.system.name} | {cc.name}"
if c_name not in self.available_caps:
continue
caps[c_name] = cc.value
if caps:
o.update_caps(caps, source="etl")
......@@ -6,7 +6,7 @@
# ----------------------------------------------------------------------
# Python modules
from typing import Optional, List
from typing import Optional, List, Union
from enum import Enum
from pydantic import IPvAnyAddress, validator
......@@ -32,6 +32,11 @@ class SourceType(str, Enum):
a = "a" # All interface addresses
class CapsItem(BaseModel):
name: str
value: Union[str, bool, int]
class ManagedObject(BaseModel):
id: str
name: str
......@@ -63,6 +68,7 @@ class ManagedObject(BaseModel):
tt_queue: Optional[str]
tt_system_id: Optional[str]
project: Optional[Reference["Project"]]
capabilities: Optional[List[CapsItem]]
checkpoint: Optional[str]
@validator("address")
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment