From 8ae73ea898653ee20eb4245b349129adbc3783d3 Mon Sep 17 00:00:00 2001 From: Andrey Vertiprahov Date: Wed, 5 Jun 2019 21:44:47 +0500 Subject: [PATCH 1/5] Move report metric to Report Detail format. --- lib/app/reportdatasources/report_metrics.py | 142 ++++++ services/web/apps/inv/reportmetrics/views.py | 468 +++++++++---------- ui/web/inv/reportmetrics/Application.js | 305 ++++++++++++ 3 files changed, 661 insertions(+), 254 deletions(-) create mode 100644 lib/app/reportdatasources/report_metrics.py create mode 100644 ui/web/inv/reportmetrics/Application.js diff --git a/lib/app/reportdatasources/report_metrics.py b/lib/app/reportdatasources/report_metrics.py new file mode 100644 index 0000000000..d028f35721 --- /dev/null +++ b/lib/app/reportdatasources/report_metrics.py @@ -0,0 +1,142 @@ +# -*- coding: utf-8 -*- +# ---------------------------------------------------------------------- +# ReportMetrics datasource +# ---------------------------------------------------------------------- +# Copyright (C) 2007-2017 The NOC Project +# See LICENSE for details +# ---------------------------------------------------------------------- + +# Python modules +from __future__ import absolute_import +import time +from collections import namedtuple, OrderedDict +# Third-party modules +# NOC modules +from .base import BaseReportColumn +from noc.core.clickhouse.connect import connection + + +class ReportMetrics(BaseReportColumn): + CHUNK_SIZE = 4000 + TABLE_NAME = None + SELECT_QUERY_MAP = { + # (List#, Name, Alias): TypeNormalizer or (TypeNormalizer, DefaultValue) + } + # KEY_FIELDS = OrderedDict([("iface_name", "path")]) + KEY_FIELDS = None + + def __init__(self, mos_ids, f_date, to_date, columns=None): + super(ReportMetrics, self).__init__(mos_ids) + self.from_date = f_date + self.to_date = to_date + self.ch_client = connection() + if not (self.TABLE_NAME and self.SELECT_QUERY_MAP): + raise NotImplementedError + if columns and isinstance(columns, list): + for c in set(self.ATTRS) - set(columns): + self.ATTRS.pop(c) + elif columns and isinstance(columns, OrderedDict): + self.ATTRS = columns + self.unknown_value = ([[""] * (len(self.SELECT_QUERY_MAP) + len(self.KEY_FIELDS))],) + + @staticmethod + def get_mo_filter(ids, use_dictionary=False): + return "managed_object IN (%s)" % ", ".join([str(c) for c in ids]) + + def get_filter_cond(self): + return {"q_where": [], "q_having": []} + + def get_query_ch(self, ids, from_date, to_date): + def update_dict(d1, d2): + for k in d2: + if k in d1: + d1[k] += d2[k] + else: + d1[k] = d2[k] + ts_from_date = time.mktime(from_date.timetuple()) + ts_to_date = time.mktime(to_date.timetuple()) + def_map = {"q_select": [], + "q_where": ["%s", # mo_filter + "(date >= toDate(%d)) AND (ts >= toDateTime(%d) AND ts <= toDateTime(%d))" % + (ts_from_date, ts_from_date, ts_to_date)], + "q_group": self.KEY_FIELDS, + "q_order_by": self.KEY_FIELDS} + condition = self.get_filter_cond() + if condition["q_having"]: + update_dict(def_map, condition["q_having"]) + for num, field, alias in sorted(self.SELECT_QUERY_MAP, key=lambda x: x[0]): + func = self.SELECT_QUERY_MAP[(num, field, alias)] or "avg(%s)" % field + def_map["q_select"] += ["%s AS %s" % (func, alias or "a_" + field)] + query = " ".join(["select %s" % ",".join(def_map["q_select"]), + "from %s" % self.TABLE_NAME, + "where %s" % " and ".join(def_map["q_where"]), + "group by %s" % ",".join(def_map["q_group"]), + "order by %s" % ",".join(def_map["q_order_by"])]) + return query + + def do_query(self): + mo_ids = self.sync_ids[:] + f_date, to_date = self.from_date, self.to_date + query = self.get_query_ch("", f_date, to_date) + while mo_ids: + chunk, mo_ids = mo_ids[:self.CHUNK_SIZE], mo_ids[self.CHUNK_SIZE:] + for row in self.ch_client.execute(query % self.get_mo_filter(chunk)): + yield row + + def extract(self): + # do_query_ch(self, moss, query_map, f_date, to_date) + Metrics = namedtuple("Metrics", [x[1] for x in self.KEY_FIELDS] + self.ATTRS.keys()) + Metrics.__new__.__defaults__ = ("",) * len(Metrics._fields) + current_mo, block = None, [] + for row in self.do_query(): + # print (row) + if current_mo and row[0] != current_mo: + yield int(row[0]), block + block = [] + block += [Metrics(*row[1:])] + block = row[1:] + current_mo = row[0] + if current_mo and block: + yield int(current_mo), block + + +class ReportInterfaceMetrics(ReportMetrics): + TABLE_NAME = "noc.interface" + SELECT_QUERY_MAP = { + # (List#, Name, Alias): TypeNormalizer or (TypeNormalizer, DefaultValue) + # Column#, db_name, column_alias, query + (0, 'managed_object', None): "", + (1, 'path', 'iface'): "arrayStringConcat(path)", + (2, 'load_in', 'l_in'): "round(quantile(0.90)(load_in), 0)", + (3, 'load_in', 'load_in_p'): "", + (4, 'load_out', None): "", + (5, 'load_out', 'load_out_p'): "", + (6, 'packets_in', None): "", + (7, 'packets_out', None): "", + (8, 'errors_in', None): "", + (9, 'errors_out', None): "", + (10, 'speed', None): "", + } + # KEY_FIELDS = OrderedDict([("iface_name", "path")]) + KEY_FIELDS = ("managed_object", "path") + + +class ReportCPUMetrics(ReportMetrics): + TABLE_NAME = "noc.cpu" + SELECT_QUERY_MAP = {(0, 'managed_object', None): "", + (1, "usage", "cpu_usage"): ""} + KEY_FIELDS = ["managed_object", "path"] + + +class ReportMemoryMetrics(ReportMetrics): + TABLE_NAME = "noc.memory" + SELECT_QUERY_MAP = {(0, 'managed_object', None): "", + (1, "usage", "memory_usage"): ""} + KEY_FIELDS = ["managed_object", "path"] + + +class ReportPingMetrics(ReportMetrics): + TABLE_NAME = "noc.ping" + SELECT_QUERY_MAP = {(0, 'managed_object', None): "", + (1, "avg(rtt)", "ping_rtt"): ""} + KEY_FIELDS = ["managed_object"] diff --git a/services/web/apps/inv/reportmetrics/views.py b/services/web/apps/inv/reportmetrics/views.py index d36c108868..f15d583855 100644 --- a/services/web/apps/inv/reportmetrics/views.py +++ b/services/web/apps/inv/reportmetrics/views.py @@ -10,186 +10,142 @@ # Python modules import datetime import time -from collections import defaultdict -# Django modules -from django import forms -from django.contrib.admin.widgets import AdminDateWidget +from collections import namedtuple +import csv +# Third-party modules +import xlsxwriter +from six import StringIO +from django.http import HttpResponse # NOC modules from noc.sa.models.managedobject import ManagedObject from noc.inv.models.interface import Interface -from noc.inv.models.interfaceprofile import InterfaceProfile -from noc.core.clickhouse.connect import connection +from noc.inv.models.platform import Platform +# from noc.inv.models.networksegment import NetworkSegment +from noc.lib.app.reportdatasources.report_metrics import ReportInterfaceMetrics, ReportCPUMetrics,\ + ReportMemoryMetrics, ReportPingMetrics from noc.sa.models.useraccess import UserAccess -from noc.lib.app.simplereport import SimpleReport, TableColumn +from noc.lib.app.extapplication import ExtApplication, view +from noc.sa.interfaces.base import StringParameter, BooleanParameter from noc.core.translation import ugettext as _ from noc.sa.models.managedobjectselector import ManagedObjectSelector from noc.sa.models.administrativedomain import AdministrativeDomain -class ReportForm(forms.Form): - reporttype = forms.ChoiceField(choices=[ - ("load_interfaces", _("Load Interfaces")), - ("load_cpu", _("Load CPU/Memory")), - ("errors", _("Errors on the Interface")), - ("ping", _("Ping RTT and Ping Attempts")) - ], label=_("Report Type")) - from_date = forms.CharField( - widget=AdminDateWidget, - label=_("From Date"), - required=True - ) - to_date = forms.CharField( - widget=AdminDateWidget, - label=_("To Date"), - required=True - ) - # skip_avail = forms.BooleanField( - # label=_("Skip full available"), - # required=False - # ) +def get_column_width(name): + excel_column_format = {"ID": 6, "OBJECT_NAME": 38, "OBJECT_STATUS": 10, "OBJECT_PROFILE": 17, + "OBJECT_PLATFORM": 25, "AVAIL": 6, "ADMIN_DOMAIN": 25, "PHYS_INTERFACE_COUNT": 5} + if name.startswith("Up") or name.startswith("Down") or name.startswith("-"): + return 8 + elif name.startswith("ADM_PATH"): + return excel_column_format["ADMIN_DOMAIN"] + elif name in excel_column_format: + return excel_column_format[name] + return 15 - # managed_object = forms.ModelChoiceField( - # label=_("Managed Object"), - # required=False, - # queryset=ManagedObject.objects.filter() - # ) - selectors = forms.ModelChoiceField( - label=_("Selectors"), - required=True, - queryset=ManagedObjectSelector.objects.order_by("name") - ) - administrative_domain = forms.ModelChoiceField( - label=_("Administrative Domain (or)"), - required=False, - queryset=AdministrativeDomain.objects.order_by("name") - ) - # object_profile = forms.ModelChoiceField( - # label=_("Object Profile"), - # required=False, - # queryset=ManagedObjectProfile.objects.exclude(name__startswith="tg.").order_by("name") - # ) - interface_profile = forms.ModelChoiceField( - label=_("Interface Profile (For Load Interfaces Report Type)"), - required=False, - queryset=InterfaceProfile.objects.order_by("name") - ) - allow_archive = forms.BooleanField( - label=_("Use archive (InfluxDB) for report"), - required=False - ) - zero = forms.BooleanField( - label=_("Exclude 0"), - required=False - ) - percent = forms.BooleanField( - label=_("Load interface in % (For Load Interfaces Report Type)"), - required=False - ) - filter_default = forms.BooleanField( - label=_("Enable Default interface profile (For Load Interfaces Report Type)"), - required=False - ) +class ReportMetricsDetailApplication(ExtApplication): + menu = _("Reports") + "|" + _("Load Metrics") + title = _("Load Metrics") + metric_source = { + "load_interfaces": ReportInterfaceMetrics, + "load_cpu": ReportCPUMetrics, + "load_memory": ReportMemoryMetrics, + "ping": ReportPingMetrics + } -class ReportMetric(object): - CHUNK_SIZE = 10 + @view("^download/$", method=["GET"], access="launch", api=True, + validate={ + "from_date": StringParameter(required=True), + "to_date": StringParameter(required=True), + "reporttype": StringParameter(required=True, choices=["load_interfaces", "load_cpu", "ping"]), + "administrative_domain": StringParameter(required=False), + # "pool": StringParameter(required=False), + # "segment": StringParameter(required=False), + "selector": StringParameter(required=False), + "interface_profile": StringParameter(required=False), + "exclude_zero": BooleanParameter(required=False), + "filter_default": BooleanParameter(required=False), + "columns": StringParameter(required=False), + "o_format": StringParameter(choices=["csv", "xlsx"])}) + def api_report(self, request, reporttype=None, from_date=None, to_date=None, object_profile=None, + filter_default=None, exclude_zero=None, interface_profile=None, allow_archive=False, + selector=None, administrative_domain=None, columns=None, o_format=None, enable_autowidth=False, + **kwargs): - def __init__(self, db="ch"): - self.rows = [] - self.get_query = self.get_query_inf - self.query_map = "" - self.reporttype = "load_interfaces" + def translate_row(row, cmap): + return [row[i] for i in cmap] - def get_query_inf(self, query_map, from_date, to_date): - fd = from_date.strftime("%Y-%m-%dT%H:%M:%SZ") - td = to_date.strftime("%Y-%m-%dT%H:%M:%SZ") map_table = {"load_interfaces": "/Interface\s\|\sLoad\s\|\s[In|Out]/", "load_cpu": "/[CPU|Memory]\s\|\sUsage/", "errors": "/Interface\s\|\s[Errors|Discards]\s\|\s[In|Out]/", "ping": "/Ping\s\|\sRTT/"} - def_map = {"q_select": ['percentile(\"value\", 98)'], - "q_from": [], # q_from = "from %s" % map_table[reporttype] - "q_where": ["%s", "time >= '%s'" % fd, "time <= '%s'" % td], - "q_group": ["object"]} - # m_r = "(%s)" % " OR ".join(["object = '%s'" % name for name in moss]) - for e in query_map: - if e.startswith("q_"): - def_map[e] += query_map[e] - query = " ".join(["SELECT %s" % ",".join(def_map["q_select"]), - "FROM %s" % map_table[self.reporttype], - "WHERE %s" % " AND ".join(def_map["q_where"]), - "GROUP BY %s" % ",".join(def_map["q_group"])]) - return query - - def get_query_ch(self, query_map, from_date, to_date): - ts_from_date = time.mktime(from_date.timetuple()) - ts_to_date = time.mktime(to_date.timetuple()) - map_table = {"load_interfaces": "interface", - "load_cpu": "cpu", - "errors": "interface", - "ping": "ping"} - def_map = {"q_select": ["managed_object"], - "q_from": [], # q_from = "from %s" % map_table[reporttype] - "q_where": ["(ts >= toDateTime(%d) AND ts <= toDateTime(%d))" % - (ts_from_date, ts_to_date)], - "q_group": ["managed_object"]} - for e in query_map: - if e.startswith("q_"): - def_map[e] += query_map[e] - query = " ".join(["select %s" % ",".join(def_map["q_select"]), - "from %s" % map_table[self.reporttype], - "prewhere %s" % "date >= toDate(%d) AND date <= toDate(%d) AND %s" % ( - ts_from_date, ts_to_date, "managed_object IN (%s)"), - "where %s" % " AND ".join(def_map["q_where"]), - "group by %s" % ",".join(def_map["q_group"])]) - return query - - def do_query_ch(self, moss, query_map, f_date, to_date): - n = 0 - client = connection() - - mos_name = sorted(moss) - query = self.get_query_ch(query_map, f_date, to_date) - self.CHUNK_SIZE = 4000 - while mos_name: - mos_name, m_r = mos_name[self.CHUNK_SIZE:], mos_name[:self.CHUNK_SIZE] - for row in client.execute(query % ", ".join(m_r)): - yield row[0:2], row[2:] - n += 1 - - def do_query(self, moss, query_map, f_date, to_date): - n = 0 - ex_res = defaultdict(list) - - client = None - mos_name = moss.keys() - query = self.get_query(query_map, f_date, to_date) - while mos_name[n:n + self.CHUNK_SIZE]: - m_r = "(%s)" % " OR ".join(["object = '%s'" % name for name in mos_name[n:n + self.CHUNK_SIZE]]) - # print query % m_r + cols = [ + "id", + "object_name", + "object_address", + "object_platform", + "object_adm_domain", + "object_segment", + # "object_hostname", + # "object_status", + # "profile_name", + # "object_profile", + # "object_vendor", + "iface_name", + "iface_description", + "iface_speed", + "load_in", + "load_in_p", + "load_out", + "load_out_p", + "errors_in", + "errors_out", + "slot", + "cpu_usage", + "memory_usage", + "ping_rtt", + "ping_attempts" + "interface_flap", + "interface_load_url" + ] - for row in client.query(query % m_r): - if self.reporttype in ["ping", "load_cpu"]: - ex_res[(row["object"],)] += [row["percentile"]] - else: - ex_res[(row["object"], row["interface"])] += [row["percentile"]] - n += self.CHUNK_SIZE - for ll in ex_res: - yield ll, ex_res[ll] + header_row = [ + "ID", + "OBJECT_NAME", + "OBJECT_ADDRESS", + "OBJECT_PLATFORM", + "OBJECT_ADM_DOMAIN", + "OBJECT_SEGMENT", + "IFACE_NAME", + "IFACE_DESCRIPTION", + "IFACE_SPEED", + "LOAD_IN", + "LOAD_IN_P", + "LOAD_OUT", + "LOAD_OUT_P", + "ERRORS_IN", + "ERRORS_OUT", + "CPU_USAGE", + "MEMORY_USAGE", + "PING_RTT", + "PING_ATTEMPTS", + "INTERFACE_FLAP", + "INTERFACE_LOAD_URL" + ] - -class ReportTraffic(SimpleReport): - title = _("Load Metrics") - form = ReportForm - - def get_data(self, request, reporttype=None, from_date=None, to_date=None, object_profile=None, percent=None, - filter_default=None, zero=None, interface_profile=None, managed_object=None, - selectors=None, administrative_domain=None, allow_archive=True, **kwargs): - - map_table = {"load_interfaces": "/Interface\s\|\sLoad\s\|\s[In|Out]/", - "load_cpu": "/[CPU|Memory]\s\|\sUsage/", - "errors": "/Interface\s\|\s[Errors|Discards]\s\|\s[In|Out]/", - "ping": "/Ping\s\|\sRTT/"} + if columns: + cmap = [] + for c in columns.split(","): + try: + cmap += [cols.index(c)] + except ValueError: + continue + else: + cmap = list(range(len(cols))) + columns_order = columns.split(",") + columns_filter = set(columns_order) + r = [translate_row(header_row, cmap)] + object_columns = [c for c in columns_order if c.startswith("object")] # Date Time Block if not from_date: @@ -211,18 +167,12 @@ class ReportTraffic(SimpleReport): if not request.user.is_superuser: mos = mos.filter( administrative_domain__in=UserAccess.get_domains(request.user)) - if managed_object: - mos = [managed_object] - else: - if selectors: - mos = mos.filter(selectors.Q) - if administrative_domain: - # @todo fix - mos = mos.filter(administrative_domain=administrative_domain) - if object_profile: - mos = mos.filter(object_profile=object_profile) - - columns = [_("Managed Object"), _("Address")] + if selector: + mos = mos.filter(ManagedObjectSelector.objects.get(id=int(selector)).Q) + if administrative_domain: + mos = mos.filter(administrative_domain__in=AdministrativeDomain.get_nested_ids(int(administrative_domain))) + if object_profile: + mos = mos.filter(object_profile=object_profile) iface_dict = {} d_url = { @@ -240,53 +190,49 @@ class ReportTraffic(SimpleReport): "url": '%(path)s?title=interface&biid=%(biid)s' '&obj=%(oname)s&iface=%(iname)s&from=%(from)s&to=%(to)s', "q_group": ["interface"], - "columns": [_("Int Name"), _("Int Descr"), - TableColumn(_("IN bps"), align="right"), - TableColumn(_("OUT bps"), align="right"), - ] + "q_select": {(0, 'managed_object', None): "managed_object", + (1, 'path', 'iface'): "arrayStringConcat(path)"} }, "errors": { "url": """%(path)s?title=errors&biid=%(biid)s&obj=%(oname)s&iface=%(iname)s&from=%(from)s&to=%(to)s""", - "columns": [_("Int Name"), - TableColumn(_("Errors IN"), align="right"), - TableColumn(_("Errors OUT"), align="right"), - TableColumn(_("Discards IN"), align="right"), - TableColumn(_("Discards OUT"), align="right")], "q_group": ["interface"] }, "load_cpu": { "url": """%(path)s?title=cpu&biid=%(biid)s&obj=%(oname)s&from=%(from)s&to=%(to)s""", - "columns": [TableColumn(_("Memory | Usage %"), align="right"), - TableColumn(_("CPU | Usage %"), align="right")] + "q_select": {(0, 'managed_object', None): "managed_object", + (1, 'path', 'slot'): "arrayStringConcat(path)"} }, "ping": { "url": """%(path)s?title=ping&biid=%(biid)s&obj=%(oname)s&from=%(from)s&to=%(to)s""", - "columns": [TableColumn(_("Ping | RTT (ms)"), align="right"), - TableColumn(_("Ping | Attempts"), align="right")] + "q_select": {(0, 'managed_object', None): "managed_object"} } } - if not allow_archive: - report_map["load_interfaces"].update({ - "q_select": ["arrayStringConcat(path)", - "round(quantile(0.98)(load_in), 0) as l_in", - "round(quantile(0.98)(load_out), 0) as l_out"], - "q_group": ["path"], - "q_where": ["(load_in != 0 and load_out != 0)"] if zero else []}) - report_map["errors"].update({ - "q_select": ["arrayStringConcat(path)", - "quantile(0.98)(errors_in) as err_in", "quantile(0.98)(errors_out) as err_out", - "quantile(0.98)(discards_in) as dis_in", "quantile(0.98)(discards_out) as dis_out"], - "q_group": ["path"], - "q_where": ["(errors_in != 0 or errors_out != 0 or discards_in != 0 or discards_out != 0)"] if zero - else []}) - report_map["load_cpu"].update({"q_select": ["arrayStringConcat(path)", "quantile(0.98)(usage) as usage"], - "q_group": ["path"]}) - report_map["ping"].update({ - "q_select": ["managed_object", "round(quantile(0.98)(rtt) / 1000, 2)", "avg(attempts)"]}) - moss = {str(mo[0]): mo for mo in mos.values_list("bi_id", "id", "name", "address")} - else: - moss = {mo[0]: mo for mo in mos.values_list("name", "id", "name", "address")} - # mos_s = set(mos.values_list("id", flat=True)) + + query_map = { + "iface_description": ('', 'iface_description', "''"), + "iface_speed": ('speed', 'iface_speed', "max(speed)"), + "load_in": ('load_in', 'l_in', "round(quantile(0.90)(load_in), 0)"), + "load_in_p": ('load_in', 'l_in_p', "round(quantile(0.90)(load_in), 0) / max(speed)"), + "load_out": ('load_out', 'l_out', "round(quantile(0.90)(load_out), 0)"), + "load_out_p": ('load_out', 'l_out_p', "round(quantile(0.90)(load_out), 0) / max(speed)"), + "errors_in": ('errors_in', 'err_in', "quantile(0.90)(errors_in)"), + "errors_out": ('errors_out', 'err_out', "quantile(0.90)(errors_out)"), + "cpu_usage": ('usage', 'cpu_usage', "quantile(0.90)(usage)"), + "ping_rtt": ('rtt', 'ping_rtt', "round(quantile(0.90)(rtt) / 1000, 2)"), + "ping_attempts": ('attempts', 'ping_attempts', "avg(attempts)") + } + for c in columns.split(","): + if c not in query_map: + continue + field, alias, func = query_map[c] + report_map[reporttype]["q_select"][ + (columns_order.index(c), field, alias)] = func + + mo_attrs = namedtuple("MOATTRs", object_columns) + moss = {} + for row in mos.values_list("bi_id", "name", "address", "platform", "administrative_domain__name", "segment"): + x = mo_attrs(*[row[1], row[2], str(Platform.get_by_id(row[3]) if row[3] else ""), row[4], row[5]]) + moss[row[0]] = [getattr(x, y) for y in object_columns] if reporttype in ["load_interfaces", "errors"]: ifaces = Interface._get_collection() xx = set(mos.values_list("id", flat=True)) @@ -299,56 +245,70 @@ class ReportTraffic(SimpleReport): iface["name"])] = (iface.get("description", ""), iface.get("in_speed", 0), iface.get("out_speed", 0)) - columns += report_map[reporttype]["columns"] - if percent: - columns.insert(-1, TableColumn(_("IN %"), align="right")) - columns += [TableColumn(_("OUT %"), align="right")] - columns += [TableColumn(_("Graphic"), format="url"), - TableColumn(_("Interval days"), align="right") - ] url = report_map[reporttype].get("url", "") - rm = ReportMetric() - rm.reporttype = reporttype - - r = [] - if allow_archive: - rm_data = rm.do_query(moss, report_map[reporttype], from_date, to_date) - else: - rm_data = rm.do_query_ch(moss, report_map[reporttype], from_date, to_date) - - for l, data in rm_data: - if not l: - continue - mo = moss[l[0]] - d_url["biid"] = l[0] - d_url["oname"] = mo[2].replace("#", "%23") - if zero and allow_archive and not sum(data): - # For InfluxDB query - continue - if reporttype in ["load_interfaces", "errors"]: - d_url["iname"] = l[1] - res = [mo[2], mo[3], l[1]] + report_metric = self.metric_source[reporttype](tuple(sorted(moss)), from_date, to_date, columns=None) + report_metric.SELECT_QUERY_MAP = report_map[reporttype]["q_select"] + # OBJECT_PLATFORM, ADMIN_DOMAIN, SEGMENT, OBJECT_HOSTNAM + for row in report_metric.do_query(): + bi_id, data = row[0], row[1:] + mo = moss[int(bi_id)] + res = [bi_id] if "id" in columns_filter else [] + res += mo + if reporttype == "load_interfaces": + iface_name, data = data[0], data[1:] + d_url["iname"] = row[1] + res += [iface_name] res += data - if "load_interfaces" in reporttype: - i_d = iface_dict.get((mo[1], l[1]), ["", "", ""]) - res.insert(3, i_d[0]) - if percent: - in_p = float(data[0]) - in_p = round((in_p / 1000.0) / (i_d[1] / 100.0), 2) if i_d[1] and in_p > 0 else 0 - # in - res.insert(-1, in_p) - # out - out_p = float(data[1]) - res += [round((out_p / 1000.0) / (i_d[2] / 100.0), 2) if i_d[2] and out_p > 0 else 0] + # if "load_interfaces" in reporttype: + # i_d = iface_dict.get((mo[1], l[1]), ["", "", ""]) + # res.insert(3, i_d[0]) + # if percent_in: + # in_p = float(data[0]) + # in_p = round((in_p / 1000.0) / (i_d[1] / 100.0), 2) if i_d[1] and in_p > 0 else 0 + # # in + # res.insert(-1, in_p) + # if percent_out: + # # out + # out_p = float(data[1]) + # res += [round((out_p / 1000.0) / (i_d[2] / 100.0), 2) if i_d[2] and out_p > 0 else 0] else: - res = [mo[2], mo[3]] res += data - res += [url % d_url, interval] + if "interface_load_url" in columns_filter: + d_url["biid"] = row[1] + d_url["oname"] = mo[2].replace("#", "%23") + res += [url % d_url, interval] r += [res] - return self.from_dataset( - title=self.title, - columns=columns, - data=r, - enumerate=True - ) + if o_format == "csv": + response = HttpResponse(content_type="text/csv") + response[ + "Content-Disposition"] = "attachment; filename=\"metrics.csv\"" + writer = csv.writer(response) + writer.writerows(r) + return response + elif o_format == "xlsx": + response = StringIO() + wb = xlsxwriter.Workbook(response) + cf1 = wb.add_format({"bottom": 1, "left": 1, "right": 1, "top": 1}) + ws = wb.add_worksheet("Alarms") + # max_column_data_length = {} + for rn, x in enumerate(r): + for cn, c in enumerate(x): + # if rn and (r[0][cn] not in max_column_data_length + # or len(str(c)) > max_column_data_length[r[0][cn]]): + # max_column_data_length[r[0][cn]] = len(str(c)) + ws.write(rn, cn, c, cf1) + ws.autofilter(0, 0, rn, cn) + ws.freeze_panes(1, 0) + # for cn, c in enumerate(r[0]): + # # Set column width + # width = get_column_width(c) + # if enable_autowidth and width < max_column_data_length[c]: + # width = max_column_data_length[c] + # ws.set_column(cn, cn, width=width) + wb.close() + response.seek(0) + response = HttpResponse(response.getvalue(), content_type="application/vnd.ms-excel") + response["Content-Disposition"] = "attachment; filename=\"metrics.xlsx\"" + response.close() + return response diff --git a/ui/web/inv/reportmetrics/Application.js b/ui/web/inv/reportmetrics/Application.js new file mode 100644 index 0000000000..0d77b9c978 --- /dev/null +++ b/ui/web/inv/reportmetrics/Application.js @@ -0,0 +1,305 @@ +//--------------------------------------------------------------------- +// inv.reportmetricdetail application +//--------------------------------------------------------------------- +// Copyright (C) 2007-2018 The NOC Project +// See LICENSE for details +//--------------------------------------------------------------------- +console.debug("Defining NOC.inv.reportmetrics.Application"); + +Ext.define("NOC.inv.reportmetrics.Application", { + extend: "NOC.core.Application", + requires: [ + "NOC.inv.networksegment.TreeCombo", + "NOC.sa.administrativedomain.TreeCombo", + "NOC.sa.managedobjectselector.LookupField" + ], + + initComponent: function() { + var me = this; + me.reportType = "load_interfaces"; + + me.interfaceData = [ + ["id", __("ID"), false], + ["object_name", __("Object Name"), true], + ["object_address", __("IP"), true], + ["object_platform", __("Object Platform"), true], + ["object_adm_domain", __("Object Administrative domain"), true], + ["object_segment", __("Object Segemnt"), false], + ["iface_name", __("Interface Name"), true], + ["iface_description", __("Interface Description"), true], + ["iface_speed", __("Interface Speed"), false], + ["load_in", __("Load In"), true], + ["load_in_p", __("Load In (% Bandwith)"), false], + ["load_out", __("Load Out"), true], + ["load_out_p", __("Load Out (% Bandwith)"), false], + ["errors_in", __("Errors In"), true], + ["errors_out", __("Errors Out"), true], + ["interface_load_url", __("Interface Load URL"), false] + ]; + me.objectData = [ + ["id", __("ID"), false], + ["object_name", __("Object Name"), true], + ["object_address", __("IP"), true], + ["object_platform", __("Object Platform"), true], + ["object_adm_domain", __("Object Administrative domain"), true], + ["object_segment", __("Object Segemnt"), false], + ["slot", __("Slot"), false], + ["cpu_usage", __("CPU Usage"), true], + ["memory_usage", __("Memory Usage"), true] + ]; + me.availabilityData = [ + ["id", __("ID"), false], + ["object_name", __("Object Name"), true], + ["object_address", __("IP"), true], + ["object_platform", __("Object Platform"), true], + ["object_adm_domain", __("Object Administrative domain"), true], + ["object_segment", __("Object Segemnt"), false], + ["ping_rtt", __("Ping RTT"), true], + ["ping_attempts", __("Ping Attempts"), true] + ]; + me.otherData = [ + ["id", __("ID"), false], + ["object_name", __("Other Data"), true] + ]; + me.columnsStore = Ext.create("Ext.data.Store", { + fields: ["id", "label", { + name: "is_active", + type: "boolean" + }], + data: me.interfaceData + }); + + me.columnsGrid = Ext.create("Ext.grid.Panel", { + store: me.columnsStore, + autoScroll: true, + columns: [ + { + text: __("Active"), + dataIndex: "is_active", + width: 25, + renderer: NOC.render.Bool, + editor: "checkbox" + }, + { + text: __("Field"), + dataIndex: "label", + flex: 1 + } + ], + selModel: "cellmodel", + plugins: [ + { + ptype: "cellediting", + clicksToEdit: 1 + } + ], + viewConfig: { + plugins: { + ptype: 'gridviewdragdrop', + dragText: 'Drag and drop to reorganize' + } + } + }); + + me.formatButton = Ext.create("Ext.button.Segmented", { + width: 150, + items: [ + { + text: __("CSV"), + width: 70 + }, + { + text: __("Excel"), + pressed: true, + width: 70 + } + ] + }); + + me.adm_domain = null; + me.segment = null; + me.selector = null; + + me.formPanel = Ext.create("Ext.form.Panel", { + autoScroll: true, + defaults: { + padding: "0 10 10 10", + labelWidth: 120 + }, + items: [ + { + name: "metric_source", + xtype: "radiogroup", + columns: 3, + vertical: false, + fieldLabel: __("Metric source"), + allowBlank: false, + margin: 0, + width: 300, + defaults: { + padding: "0 5" + }, + items: [ + {boxLabel: 'Interfaces', name: 'rb', inputValue: 'interface', checked: true}, + {boxLabel: 'Objects', name: 'rb', inputValue: 'object'}, + {boxLabel: 'Ping', name: 'rb', inputValue: 'ping'}, + {boxLabel: 'Other', name: 'rb', inputValue: 'other'} + ], + listeners: { + scope: me, + change: me.onChangeSource + } + }, + { + name: "from_date", + xtype: "datefield", + startDay: 1, + fieldLabel: __("From"), + allowBlank: false, + format: "d.m.Y", + margin: 0, + width: 210 + }, + { + name: "to_date", + xtype: "datefield", + startDay: 1, + fieldLabel: __("To"), + allowBlank: false, + format: "d.m.Y", + margin: 0, + width: 210 + }, + { + name: "segment", + xtype: "inv.networksegment.TreeCombo", + fieldLabel: __("Segment"), + listWidth: 1, + listAlign: 'left', + labelAlign: "left", + width: 500, + listeners: { + scope: me, + select: function(combo, record) { + me.segment = record.get("id") + } + } + }, + { + name: "administrative_domain", + xtype: "sa.administrativedomain.TreeCombo", + fieldLabel: __("By Adm. domain"), + listWidth: 1, + listAlign: 'left', + labelAlign: "left", + width: 500, + allowBlank: true, + listeners: { + scope: me, + select: function(combo, record) { + me.adm_domain = record.get("id") + } + } + }, + { + name: "Selector", + xtype: "sa.managedobjectselector.LookupField", + fieldLabel: __("By Selector"), + listWidth: 1, + listAlign: 'left', + labelAlign: "left", + width: 500, + allowBlank: true, + listeners: { + scope: me, + select: function(combo, record) { + me.selector = record.get("id") + } + } + }, + me.formatButton, + me.columnsGrid + ], + dockedItems: [ + { + xtype: "toolbar", + dock: "top", + items: [ + { + text: __("Download"), + glyph: NOC.glyph.download, + scope: me, + handler: me.onDownload, + formBind: true + } + ] + } + ] + }); + Ext.apply(me, { + items: [me.formPanel] + }); + me.callParent(); + }, + + onDownload: function() { + var me = this, + v = me.formPanel.getValues(), + format = "csv", + url, + columns = []; + + if(me.formatButton.items.items[1].pressed) { + format = "xlsx" + } + + url = [ + "/inv/reportmetrics/download/?o_format=" + format + + "&from_date=" + v.from_date + "&to_date=" + v.to_date + ]; + + if (me.interfaceData) { + url.push("&reporttype=" + me.reportType); + } + if(me.adm_domain) { + url.push("&administrative_domain=" + me.adm_domain); + } + + if(me.segment) { + url.push("&segment=" + me.segment); + } + + if(me.selector) { + url.push("&selector=" + me.selector); + } + + me.columnsStore.each(function(record) { + if(record.get("is_active")) { + columns.push(record.get("id")); + } + }); + + url.push("&columns=" + columns.join(",")); + + window.open(url.join("")); + }, + // + onChangeSource: function(self, newVal) { + var me = this, data = me.otherData; + switch(newVal.rb) { + case "interface": + data = me.interfaceData; + me.reportType = "load_interfaces"; + break; + case "object": + data = me.objectData; + me.reportType = "load_cpu"; + break; + case "ping": + data = me.availabilityData; + me.reportType = "ping"; + break; + } + me.columnsStore.loadData(data); + } +}); -- GitLab From f257a89bd691b5bde490eb81e9049cfcf4190940 Mon Sep 17 00:00:00 2001 From: Andrey Vertiprahov Date: Thu, 6 Jun 2019 10:39:42 +0500 Subject: [PATCH 2/5] Fix url. --- services/web/apps/inv/reportmetrics/views.py | 42 +++++++++++--------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/services/web/apps/inv/reportmetrics/views.py b/services/web/apps/inv/reportmetrics/views.py index f15d583855..b15c591005 100644 --- a/services/web/apps/inv/reportmetrics/views.py +++ b/services/web/apps/inv/reportmetrics/views.py @@ -20,7 +20,7 @@ from django.http import HttpResponse from noc.sa.models.managedobject import ManagedObject from noc.inv.models.interface import Interface from noc.inv.models.platform import Platform -# from noc.inv.models.networksegment import NetworkSegment +from noc.inv.models.networksegment import NetworkSegment from noc.lib.app.reportdatasources.report_metrics import ReportInterfaceMetrics, ReportCPUMetrics,\ ReportMemoryMetrics, ReportPingMetrics from noc.sa.models.useraccess import UserAccess @@ -33,7 +33,7 @@ from noc.sa.models.administrativedomain import AdministrativeDomain def get_column_width(name): excel_column_format = {"ID": 6, "OBJECT_NAME": 38, "OBJECT_STATUS": 10, "OBJECT_PROFILE": 17, - "OBJECT_PLATFORM": 25, "AVAIL": 6, "ADMIN_DOMAIN": 25, "PHYS_INTERFACE_COUNT": 5} + "OBJECT_PLATFORM": 25, "AVAIL": 6, "OBJECT_ADM_DOMAIN": 25, "PHYS_INTERFACE_COUNT": 5} if name.startswith("Up") or name.startswith("Down") or name.startswith("-"): return 8 elif name.startswith("ADM_PATH"): @@ -158,7 +158,7 @@ class ReportMetricsDetailApplication(ExtApplication): else: to_date = datetime.datetime.strptime(to_date, "%d.%m.%Y") + datetime.timedelta(days=1) - interval = (to_date - from_date).days + # interval = (to_date - from_date).days ts_from_date = time.mktime(from_date.timetuple()) ts_to_date = time.mktime(to_date.timetuple()) @@ -212,9 +212,11 @@ class ReportMetricsDetailApplication(ExtApplication): "iface_description": ('', 'iface_description', "''"), "iface_speed": ('speed', 'iface_speed', "max(speed)"), "load_in": ('load_in', 'l_in', "round(quantile(0.90)(load_in), 0)"), - "load_in_p": ('load_in', 'l_in_p', "round(quantile(0.90)(load_in), 0) / max(speed)"), + "load_in_p": ('load_in', 'l_in_p', + "replaceOne(toString(round(quantile(0.90)(load_in) / max(speed), 4) * 100), '.', ',')"), "load_out": ('load_out', 'l_out', "round(quantile(0.90)(load_out), 0)"), - "load_out_p": ('load_out', 'l_out_p', "round(quantile(0.90)(load_out), 0) / max(speed)"), + "load_out_p": ('load_out', 'l_out_p', + "replaceOne(toString(round(quantile(0.90)(load_out) / max(speed), 4) * 100), '.', ',')"), "errors_in": ('errors_in', 'err_in', "quantile(0.90)(errors_in)"), "errors_out": ('errors_out', 'err_out', "quantile(0.90)(errors_out)"), "cpu_usage": ('usage', 'cpu_usage', "quantile(0.90)(usage)"), @@ -228,10 +230,11 @@ class ReportMetricsDetailApplication(ExtApplication): report_map[reporttype]["q_select"][ (columns_order.index(c), field, alias)] = func - mo_attrs = namedtuple("MOATTRs", object_columns) + mo_attrs = namedtuple("MOATTRs", [c for c in cols if c.startswith("object")]) moss = {} for row in mos.values_list("bi_id", "name", "address", "platform", "administrative_domain__name", "segment"): - x = mo_attrs(*[row[1], row[2], str(Platform.get_by_id(row[3]) if row[3] else ""), row[4], row[5]]) + x = mo_attrs(*[row[1], row[2], str(Platform.get_by_id(row[3]) if row[3] else ""), + row[4], str(NetworkSegment.get_by_id(row[5])) if row[5] else ""]) moss[row[0]] = [getattr(x, y) for y in object_columns] if reporttype in ["load_interfaces", "errors"]: ifaces = Interface._get_collection() @@ -274,9 +277,10 @@ class ReportMetricsDetailApplication(ExtApplication): else: res += data if "interface_load_url" in columns_filter: - d_url["biid"] = row[1] + d_url["biid"] = bi_id d_url["oname"] = mo[2].replace("#", "%23") - res += [url % d_url, interval] + # res += [url % d_url, interval] + res += [url % d_url] r += [res] if o_format == "csv": @@ -291,21 +295,21 @@ class ReportMetricsDetailApplication(ExtApplication): wb = xlsxwriter.Workbook(response) cf1 = wb.add_format({"bottom": 1, "left": 1, "right": 1, "top": 1}) ws = wb.add_worksheet("Alarms") - # max_column_data_length = {} + max_column_data_length = {} for rn, x in enumerate(r): for cn, c in enumerate(x): - # if rn and (r[0][cn] not in max_column_data_length - # or len(str(c)) > max_column_data_length[r[0][cn]]): - # max_column_data_length[r[0][cn]] = len(str(c)) + if rn and (r[0][cn] not in max_column_data_length + or len(str(c)) > max_column_data_length[r[0][cn]]): + max_column_data_length[r[0][cn]] = len(str(c)) ws.write(rn, cn, c, cf1) ws.autofilter(0, 0, rn, cn) ws.freeze_panes(1, 0) - # for cn, c in enumerate(r[0]): - # # Set column width - # width = get_column_width(c) - # if enable_autowidth and width < max_column_data_length[c]: - # width = max_column_data_length[c] - # ws.set_column(cn, cn, width=width) + for cn, c in enumerate(r[0]): + # Set column width + width = get_column_width(c) + if enable_autowidth and width < max_column_data_length[c]: + width = max_column_data_length[c] + ws.set_column(cn, cn, width=width) wb.close() response.seek(0) response = HttpResponse(response.getvalue(), content_type="application/vnd.ms-excel") -- GitLab From 4d0de37acc46c5aacacdabd5873b4c1846a94fdf Mon Sep 17 00:00:00 2001 From: Andrey Vertiprahov Date: Sat, 8 Jun 2019 12:03:54 +0500 Subject: [PATCH 3/5] Add exclude_zero and interface_profile filter. --- lib/app/reportdatasources/report_metrics.py | 36 +++++++++--------- services/web/apps/inv/reportmetrics/views.py | 25 +++++++++---- ui/web/inv/reportmetrics/Application.js | 39 ++++++++++++++++++++ 3 files changed, 74 insertions(+), 26 deletions(-) diff --git a/lib/app/reportdatasources/report_metrics.py b/lib/app/reportdatasources/report_metrics.py index d028f35721..f9b3c83600 100644 --- a/lib/app/reportdatasources/report_metrics.py +++ b/lib/app/reportdatasources/report_metrics.py @@ -23,6 +23,10 @@ class ReportMetrics(BaseReportColumn): # (List#, Name, Alias): TypeNormalizer or (TypeNormalizer, DefaultValue) } # KEY_FIELDS = OrderedDict([("iface_name", "path")]) + CUSTOM_FILTER = { + "having": [], + "where": [], + } KEY_FIELDS = None def __init__(self, mos_ids, f_date, to_date, columns=None): @@ -43,41 +47,35 @@ class ReportMetrics(BaseReportColumn): def get_mo_filter(ids, use_dictionary=False): return "managed_object IN (%s)" % ", ".join([str(c) for c in ids]) - def get_filter_cond(self): - return {"q_where": [], "q_having": []} + def get_custom_conditions(self): + return self.CUSTOM_FILTER - def get_query_ch(self, ids, from_date, to_date): - def update_dict(d1, d2): - for k in d2: - if k in d1: - d1[k] += d2[k] - else: - d1[k] = d2[k] + def get_query_ch(self, from_date, to_date): ts_from_date = time.mktime(from_date.timetuple()) ts_to_date = time.mktime(to_date.timetuple()) + custom_conditions = self.get_custom_conditions() def_map = {"q_select": [], "q_where": ["%s", # mo_filter "(date >= toDate(%d)) AND (ts >= toDateTime(%d) AND ts <= toDateTime(%d))" % - (ts_from_date, ts_from_date, ts_to_date)], + (ts_from_date, ts_from_date, ts_to_date)] + custom_conditions["where"][:], "q_group": self.KEY_FIELDS, + "q_having": custom_conditions["having"][:], "q_order_by": self.KEY_FIELDS} - condition = self.get_filter_cond() - if condition["q_having"]: - update_dict(def_map, condition["q_having"]) for num, field, alias in sorted(self.SELECT_QUERY_MAP, key=lambda x: x[0]): func = self.SELECT_QUERY_MAP[(num, field, alias)] or "avg(%s)" % field def_map["q_select"] += ["%s AS %s" % (func, alias or "a_" + field)] - query = " ".join(["select %s" % ",".join(def_map["q_select"]), - "from %s" % self.TABLE_NAME, - "where %s" % " and ".join(def_map["q_where"]), - "group by %s" % ",".join(def_map["q_group"]), - "order by %s" % ",".join(def_map["q_order_by"])]) + query = " ".join(["SELECT %s" % ",".join(def_map["q_select"]), + "FROM %s" % self.TABLE_NAME, + "WHERE %s" % " AND ".join(def_map["q_where"]), + "GROUP BY %s" % ",".join(def_map["q_group"]), + "HAVING %s" % " AND ".join(def_map["q_having"]) if def_map["q_having"] else "", + "ORDER BY %s" % ",".join(def_map["q_order_by"])]) return query def do_query(self): mo_ids = self.sync_ids[:] f_date, to_date = self.from_date, self.to_date - query = self.get_query_ch("", f_date, to_date) + query = self.get_query_ch(f_date, to_date) while mo_ids: chunk, mo_ids = mo_ids[:self.CHUNK_SIZE], mo_ids[self.CHUNK_SIZE:] for row in self.ch_client.execute(query % self.get_mo_filter(chunk)): diff --git a/services/web/apps/inv/reportmetrics/views.py b/services/web/apps/inv/reportmetrics/views.py index b15c591005..96568d2190 100644 --- a/services/web/apps/inv/reportmetrics/views.py +++ b/services/web/apps/inv/reportmetrics/views.py @@ -19,6 +19,7 @@ from django.http import HttpResponse # NOC modules from noc.sa.models.managedobject import ManagedObject from noc.inv.models.interface import Interface +from noc.inv.models.interfaceprofile import InterfaceProfile from noc.inv.models.platform import Platform from noc.inv.models.networksegment import NetworkSegment from noc.lib.app.reportdatasources.report_metrics import ReportInterfaceMetrics, ReportCPUMetrics,\ @@ -26,9 +27,9 @@ from noc.lib.app.reportdatasources.report_metrics import ReportInterfaceMetrics, from noc.sa.models.useraccess import UserAccess from noc.lib.app.extapplication import ExtApplication, view from noc.sa.interfaces.base import StringParameter, BooleanParameter -from noc.core.translation import ugettext as _ from noc.sa.models.managedobjectselector import ManagedObjectSelector from noc.sa.models.administrativedomain import AdministrativeDomain +from noc.core.translation import ugettext as _ def get_column_width(name): @@ -60,7 +61,7 @@ class ReportMetricsDetailApplication(ExtApplication): "reporttype": StringParameter(required=True, choices=["load_interfaces", "load_cpu", "ping"]), "administrative_domain": StringParameter(required=False), # "pool": StringParameter(required=False), - # "segment": StringParameter(required=False), + "segment": StringParameter(required=False), "selector": StringParameter(required=False), "interface_profile": StringParameter(required=False), "exclude_zero": BooleanParameter(required=False), @@ -68,8 +69,8 @@ class ReportMetricsDetailApplication(ExtApplication): "columns": StringParameter(required=False), "o_format": StringParameter(choices=["csv", "xlsx"])}) def api_report(self, request, reporttype=None, from_date=None, to_date=None, object_profile=None, - filter_default=None, exclude_zero=None, interface_profile=None, allow_archive=False, - selector=None, administrative_domain=None, columns=None, o_format=None, enable_autowidth=False, + filter_default=None, exclude_zero=None, interface_profile=None, selector=None, + administrative_domain=None, columns=None, o_format=None, enable_autowidth=False, **kwargs): def translate_row(row, cmap): @@ -209,7 +210,10 @@ class ReportMetricsDetailApplication(ExtApplication): } query_map = { - "iface_description": ('', 'iface_description', "''"), + # "iface_description": ('', 'iface_description', "''"), + "iface_description": ( + '', 'iface_description', + "dictGetString('interfaceattributes','description' , (managed_object, arrayStringConcat(path)))"), "iface_speed": ('speed', 'iface_speed', "max(speed)"), "load_in": ('load_in', 'l_in', "round(quantile(0.90)(load_in), 0)"), "load_in_p": ('load_in', 'l_in_p', @@ -251,6 +255,13 @@ class ReportMetricsDetailApplication(ExtApplication): url = report_map[reporttype].get("url", "") report_metric = self.metric_source[reporttype](tuple(sorted(moss)), from_date, to_date, columns=None) report_metric.SELECT_QUERY_MAP = report_map[reporttype]["q_select"] + if exclude_zero and reporttype == "load_interfaces": + report_metric.CUSTOM_FILTER["having"] += ["max(load_in) != 0 AND max(load_out) != 0"] + interface_profile = InterfaceProfile.objects.filter(id=interface_profile).first() + if interface_profile: + report_metric.CUSTOM_FILTER["having"] += [ + "dictGetString('interfaceattributes', 'profile', " + "(managed_object, arrayStringConcat(path))) = '%s'" % interface_profile.name] # OBJECT_PLATFORM, ADMIN_DOMAIN, SEGMENT, OBJECT_HOSTNAM for row in report_metric.do_query(): bi_id, data = row[0], row[1:] @@ -298,8 +309,8 @@ class ReportMetricsDetailApplication(ExtApplication): max_column_data_length = {} for rn, x in enumerate(r): for cn, c in enumerate(x): - if rn and (r[0][cn] not in max_column_data_length - or len(str(c)) > max_column_data_length[r[0][cn]]): + if rn and (r[0][cn] not in max_column_data_length or + len(str(c)) > max_column_data_length[r[0][cn]]): max_column_data_length[r[0][cn]] = len(str(c)) ws.write(rn, cn, c, cf1) ws.autofilter(0, 0, rn, cn) diff --git a/ui/web/inv/reportmetrics/Application.js b/ui/web/inv/reportmetrics/Application.js index 0d77b9c978..14e9b5cfdb 100644 --- a/ui/web/inv/reportmetrics/Application.js +++ b/ui/web/inv/reportmetrics/Application.js @@ -10,6 +10,7 @@ Ext.define("NOC.inv.reportmetrics.Application", { extend: "NOC.core.Application", requires: [ "NOC.inv.networksegment.TreeCombo", + "NOC.inv.interfaceprofile.LookupField", "NOC.sa.administrativedomain.TreeCombo", "NOC.sa.managedobjectselector.LookupField" ], @@ -119,6 +120,8 @@ Ext.define("NOC.inv.reportmetrics.Application", { me.adm_domain = null; me.segment = null; me.selector = null; + me.interface_profile = null; + me.exclude_zero = null; me.formPanel = Ext.create("Ext.form.Panel", { autoScroll: true, @@ -217,6 +220,35 @@ Ext.define("NOC.inv.reportmetrics.Application", { } } }, + { + name: "Interface Profile Filter", + xtype: "inv.interfaceprofile.LookupField", + fieldLabel: __("By Interface Profile"), + listWidth: 1, + listAlign: 'left', + labelAlign: "left", + width: 500, + allowBlank: true, + listeners: { + scope: me, + select: function(combo, record) { + me.interface_profile = record.get("id") + } + } + }, + { + name: "exclude_zero", + xtype: "checkboxfield", + boxLabel: __("Filter interface has zero load"), + allowBlank: false, + defaultValue: false, + listeners: { + scope: me, + change: function(value) { + me.exclude_zero = value.getValue() + } + } + }, me.formatButton, me.columnsGrid ], @@ -273,6 +305,13 @@ Ext.define("NOC.inv.reportmetrics.Application", { url.push("&selector=" + me.selector); } + if(me.interface_profile) { + url.push("&interface_profile=" + me.interface_profile); + } + if(me.exclude_zero != null) { + url.push("&exclude_zero=" + me.exclude_zero); + } + me.columnsStore.each(function(record) { if(record.get("is_active")) { columns.push(record.get("id")); -- GitLab From ae692eb6ebc0d56eff0cefcc3d7938b4211f2e7b Mon Sep 17 00:00:00 2001 From: Andrey Vertiprahov Date: Sat, 8 Jun 2019 16:14:28 +0500 Subject: [PATCH 4/5] Use dictionary value for calculate interface bandwith. --- services/web/apps/inv/reportmetrics/views.py | 32 +++++++++++--------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/services/web/apps/inv/reportmetrics/views.py b/services/web/apps/inv/reportmetrics/views.py index 96568d2190..57654d94f7 100644 --- a/services/web/apps/inv/reportmetrics/views.py +++ b/services/web/apps/inv/reportmetrics/views.py @@ -217,10 +217,14 @@ class ReportMetricsDetailApplication(ExtApplication): "iface_speed": ('speed', 'iface_speed', "max(speed)"), "load_in": ('load_in', 'l_in', "round(quantile(0.90)(load_in), 0)"), "load_in_p": ('load_in', 'l_in_p', - "replaceOne(toString(round(quantile(0.90)(load_in) / max(speed), 4) * 100), '.', ',')"), + "replaceOne(toString(round(quantile(0.90)(load_in) / " + "if(max(speed) = 0, dictGetUInt32('interfaceattributes', 'in_speed', " + "(managed_object, arrayStringConcat(path))), max(speed)), 4) * 100), '.', ',')"), "load_out": ('load_out', 'l_out', "round(quantile(0.90)(load_out), 0)"), "load_out_p": ('load_out', 'l_out_p', - "replaceOne(toString(round(quantile(0.90)(load_out) / max(speed), 4) * 100), '.', ',')"), + "replaceOne(toString(round(quantile(0.90)(load_out) / " + "if(max(speed) = 0, dictGetUInt32('interfaceattributes', 'in_speed', " + "(managed_object, arrayStringConcat(path))), max(speed)), 4) * 100), '.', ',')"), "errors_in": ('errors_in', 'err_in', "quantile(0.90)(errors_in)"), "errors_out": ('errors_out', 'err_out', "quantile(0.90)(errors_out)"), "cpu_usage": ('usage', 'cpu_usage', "quantile(0.90)(usage)"), @@ -240,25 +244,25 @@ class ReportMetricsDetailApplication(ExtApplication): x = mo_attrs(*[row[1], row[2], str(Platform.get_by_id(row[3]) if row[3] else ""), row[4], str(NetworkSegment.get_by_id(row[5])) if row[5] else ""]) moss[row[0]] = [getattr(x, y) for y in object_columns] - if reporttype in ["load_interfaces", "errors"]: - ifaces = Interface._get_collection() - xx = set(mos.values_list("id", flat=True)) - for iface in ifaces.find( - {"type": "physical"}, - {"managed_object": 1, "name": 1, "description": 1, "in_speed": 1, "out_speed": 1}): - if iface["managed_object"] not in xx: - continue - iface_dict[(iface["managed_object"], - iface["name"])] = (iface.get("description", ""), - iface.get("in_speed", 0), iface.get("out_speed", 0)) + # if reporttype in ["load_interfaces", "errors"]: + # ifaces = Interface._get_collection() + # xx = set(mos.values_list("id", flat=True)) + # for iface in ifaces.find( + # {"type": "physical"}, + # {"managed_object": 1, "name": 1, "description": 1, "in_speed": 1, "out_speed": 1}): + # if iface["managed_object"] not in xx: + # continue + # iface_dict[(iface["managed_object"], + # iface["name"])] = (iface.get("description", ""), + # iface.get("in_speed", 0), iface.get("out_speed", 0)) url = report_map[reporttype].get("url", "") report_metric = self.metric_source[reporttype](tuple(sorted(moss)), from_date, to_date, columns=None) report_metric.SELECT_QUERY_MAP = report_map[reporttype]["q_select"] if exclude_zero and reporttype == "load_interfaces": report_metric.CUSTOM_FILTER["having"] += ["max(load_in) != 0 AND max(load_out) != 0"] - interface_profile = InterfaceProfile.objects.filter(id=interface_profile).first() if interface_profile: + interface_profile = InterfaceProfile.objects.filter(id=interface_profile).first() report_metric.CUSTOM_FILTER["having"] += [ "dictGetString('interfaceattributes', 'profile', " "(managed_object, arrayStringConcat(path))) = '%s'" % interface_profile.name] -- GitLab From e97ee18ba7a4b611d1affc7790e47c206952de1b Mon Sep 17 00:00:00 2001 From: Andrey Vertiprahov Date: Sat, 8 Jun 2019 16:23:04 +0500 Subject: [PATCH 5/5] Fix interfaceattributes in_speed column type. --- services/web/apps/inv/reportmetrics/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/web/apps/inv/reportmetrics/views.py b/services/web/apps/inv/reportmetrics/views.py index 57654d94f7..485d90e9d3 100644 --- a/services/web/apps/inv/reportmetrics/views.py +++ b/services/web/apps/inv/reportmetrics/views.py @@ -218,12 +218,12 @@ class ReportMetricsDetailApplication(ExtApplication): "load_in": ('load_in', 'l_in', "round(quantile(0.90)(load_in), 0)"), "load_in_p": ('load_in', 'l_in_p', "replaceOne(toString(round(quantile(0.90)(load_in) / " - "if(max(speed) = 0, dictGetUInt32('interfaceattributes', 'in_speed', " + "if(max(speed) = 0, dictGetUInt64('interfaceattributes', 'in_speed', " "(managed_object, arrayStringConcat(path))), max(speed)), 4) * 100), '.', ',')"), "load_out": ('load_out', 'l_out', "round(quantile(0.90)(load_out), 0)"), "load_out_p": ('load_out', 'l_out_p', "replaceOne(toString(round(quantile(0.90)(load_out) / " - "if(max(speed) = 0, dictGetUInt32('interfaceattributes', 'in_speed', " + "if(max(speed) = 0, dictGetUInt64('interfaceattributes', 'in_speed', " "(managed_object, arrayStringConcat(path))), max(speed)), 4) * 100), '.', ',')"), "errors_in": ('errors_in', 'err_in', "quantile(0.90)(errors_in)"), "errors_out": ('errors_out', 'err_out', "quantile(0.90)(errors_out)"), -- GitLab