Merge branch 'add_report_load_metric_max' into 'master'

add new report - load metric max

See merge request noc/noc!3322

services/web/apps/inv/reportmaxmetrics/views.py

+# -*- coding: utf-8 -*-
+# ----------------------------------------------------------------------
+# inv.reportmaxmetrics
+# ----------------------------------------------------------------------
+# Copyright (C) 2007-2020 The NOC Project
+# See LICENSE for details
+# ----------------------------------------------------------------------
+
+# Python modules
+import datetime
+from collections import defaultdict, Iterable
+from collections import namedtuple
+import csv
+
+# Third-party modules
+import xlsxwriter
+from six import StringIO
+from django.http import HttpResponse
+from pymongo import ReadPreference
+from bson import ObjectId
+
+# NOC modules
+from noc.inv.models.platform import Platform
+from noc.inv.models.networksegment import NetworkSegment
+from noc.core.clickhouse.connect import connection as ch_connection
+from noc.core.clickhouse.error import ClickhouseError
+from noc.core.text import alnum_key
+from noc.inv.models.interface import Interface
+from noc.inv.models.link import Link
+from noc.sa.models.managedobject import ManagedObject
+from noc.lib.app.reportdatasources.report_container import ReportContainerData
+from noc.sa.models.useraccess import UserAccess
+from noc.lib.app.extapplication import ExtApplication, view
+from noc.sa.interfaces.base import StringParameter, BooleanParameter
+from noc.sa.models.managedobjectselector import ManagedObjectSelector
+from noc.sa.models.administrativedomain import AdministrativeDomain
+from noc.core.translation import ugettext as _
+
+
+def get_column_width(name):
+    excel_column_format = {
+        "ID": 6,
+        "OBJECT_NAME": 38,
+        "OBJECT_STATUS": 10,
+        "OBJECT_PROFILE": 17,
+        "OBJECT_PLATFORM": 25,
+        "AVAIL": 6,
+        "OBJECT_ADM_DOMAIN": 25,
+        "PHYS_INTERFACE_COUNT": 5,
+    }
+    if name.startswith("Up") or name.startswith("Down") or name.startswith("-"):
+        return 8
+    elif name.startswith("ADM_PATH"):
+        return excel_column_format["ADMIN_DOMAIN"]
+    elif name in excel_column_format:
+        return excel_column_format[name]
+    return 15
+
+
+class ReportMaxMetricsmaxDetailApplication(ExtApplication):
+    menu = _("Reports") + "|" + _("Load Metrics max")
+    title = _("Load Metrics max")
+
+    @view(
+        r"^download/$",
+        method=["GET"],
+        access="launch",
+        api=True,
+        validate={
+            "from_date": StringParameter(required=True),
+            "to_date": StringParameter(required=True),
+            "administrative_domain": StringParameter(required=False),
+            # "pool": StringParameter(required=False),
+            "segment": StringParameter(required=False),
+            "selector": StringParameter(required=False),
+            "object_profile": StringParameter(required=False),
+            "interface_profile": StringParameter(required=False),
+            "exclude_zero": BooleanParameter(required=False),
+            "filter_default": BooleanParameter(required=False),
+            "columns": StringParameter(required=False),
+            "description": StringParameter(required=False),
+            "o_format": StringParameter(choices=["csv", "xlsx"]),
+        },
+    )
+    def api_report(
+        self,
+        request,
+        reporttype=None,
+        from_date=None,
+        to_date=None,
+        object_profile=None,
+        filter_default=None,
+        exclude_zero=True,
+        interface_profile=None,
+        selector=None,
+        administrative_domain=None,
+        columns=None,
+        description=None,
+        o_format=None,
+        enable_autowidth=False,
+        **kwargs
+    ):
+        # get maximum metrics for the period
+        def get_interface_metrics(managed_objects, from_date, to_date):
+            if not isinstance(managed_objects, Iterable):
+                managed_objects = [managed_objects]
+            bi_map = {str(getattr(mo, "bi_id", mo)): mo for mo in managed_objects}
+
+            from_date = from_date.replace(microsecond=0)
+            to_date = to_date.replace(microsecond=0)
+            SQL = """SELECT managed_object, path[4] as iface, argMax(ts, ts), divide(max(load_in),1048576) as load_in_max,
+                             divide(max(load_out),1048576) as load_out_max, argMax(ts,load_in) as max_load_in_time,
+                             argMax(ts,load_out) as max_load_out_time, divide(avg(load_in),1048576) as avg_load_in,
+                             divide(avg(load_out),1048576) as avg_load_out
+                    FROM interface
+                    WHERE
+                      ts >= toDateTime('%s')
+                      AND ts <= toDateTime('%s')
+                      AND managed_object IN (%s)
+                    GROUP BY managed_object, iface
+                    """ % (
+                # from_date.date().isoformat(),
+                # to_date.date().isoformat(),
+                from_date.isoformat(sep=" "),
+                to_date.isoformat(sep=" "),
+                ", ".join(bi_map),
+            )
+            ch = ch_connection()
+            metric_map = defaultdict(dict)
+            try:
+                for (
+                    mo_bi_id,
+                    iface,
+                    ts,
+                    load_in_max,
+                    load_out_max,
+                    max_load_in_time,
+                    max_load_out_time,
+                    avg_load_in,
+                    avg_load_out,
+                ) in ch.execute(post=SQL):
+                    mo = bi_map.get(mo_bi_id)
+                    if mo:
+                        metric_map[mo][iface] = {
+                            "max_load_in": float(load_in_max),
+                            "max_load_out": float(load_out_max),
+                            "max_load_in_time": max_load_in_time,
+                            "max_load_out_time": max_load_out_time,
+                            "avg_load_in": float(avg_load_in),
+                            "avg_load_out": float(avg_load_out),
+                        }
+            except ClickhouseError:
+                pass
+            return metric_map
+
+        # get interfaces
+        def load(mo_ids, description, interface_profile):
+            # mo_ids list(mo.id)
+            iface = defaultdict(dict)
+            match = {
+                "managed_object": {"$in": mo_ids},
+                "type": {"$in": ["physical"]},
+                "admin_status": True,
+            }
+            if description:
+                match["description"] = {
+                    "$regex": description
+                }  # ({ "description": {$regex:/401/ }})
+
+            if interface_profile:
+                match["profile"] = {"$in": [ObjectId(str(interface_profile))]}
+
+            result = (
+                Interface._get_collection()
+                .with_options(read_preference=ReadPreference.SECONDARY_PREFERRED)
+                .aggregate([{"$match": match}])  # , {"$lookup": lookup}])
+            )
+
+            for i in result:
+                iface[i["managed_object"]][i["name"]] = {
+                    "bandwidth": i["in_speed"] if i.get("in_speed") else "",
+                    "description": i["description"] if i.get("description") else "",
+                }
+            return iface
+
+        def translate_row(row, cmap):
+            return [row[i] for i in cmap]
+
+        cols = [
+            "id",
+            "object_name",
+            "object_address",
+            "object_platform",
+            "object_adm_domain",
+            "object_segment",
+            "object_container",
+            # "object_hostname",
+            # "object_status",
+            # "profile_name",
+            # "object_profile",
+            # "object_vendor",
+            "iface_name",
+            "iface_description",
+            "iface_speed",
+            "max_load_in",
+            "max_load_in_time",
+            "max_load_out",
+            "max_load_out_time",
+            "avg_load_in",
+            "avg_load_out",
+            "uplink_iface_name",
+            "uplink_iface_description",
+            "uplink_max_load_in",
+            "uplink_max_load_in_time",
+            "uplink_max_load_out",
+            "uplink_max_load_out_time",
+            "uplink_avg_load_in",
+            "uplink_avg_load_out",
+            "uplink_iface_speed",
+        ]
+
+        header_row = [
+            "ID",
+            _("OBJECT_NAME"),
+            _("OBJECT_ADDRESS"),
+            _("OBJECT_PLATFORM"),
+            _("OBJECT_ADMDOMAIN"),
+            _("OBJECT_SEGMENT"),
+            _("CONTAINER_ADDRESS"),
+            _("IFACE_NAME"),
+            _("IFACE_DESCRIPTION"),
+            _("IFACE_SPEED"),
+            _("MAX_LOAD_IN, Mbps"),
+            _("MAX_LOAD_IN_TIME"),
+            _("MAX_LOAD_OUT, Mbps"),
+            _("MAX_LOAD_OUT_TIME"),
+            _("AVG_LOAD_IN, Mbps"),
+            _("AVG_LOAD_OUT, Mbps"),
+            _("UPLINK_IFACE_NAME"),
+            _("UPLINK_IFACE_DESCRIPTION"),
+            _("UPLINK_MAX_LOAD_IN, Mbps"),
+            _("UPLINK_MAX_TIME_IN"),
+            _("UPLINK_MAX_LOAD_OUT, Mbps"),
+            _("UPLINK_MAX_TIME_OUT"),
+            _("UPLINK_AVG_LOAD_IN, Mbps"),
+            _("UPLINK_AVG_LOAD_OUT, Mbps"),
+            _("UPLINK_IFACE_SPEED"),
+        ]
+
+        if columns:
+            cmap = []
+            for c in columns.split(","):
+                try:
+                    cmap += [cols.index(c)]
+                except ValueError:
+                    continue
+        else:
+            cmap = list(range(len(cols)))
+        columns_order = columns.split(",")
+        columns_filter = set(columns_order)
+        r = [translate_row(header_row, cmap)]
+
+        # Date Time Block
+        if not from_date:
+            from_date = datetime.datetime.now() - datetime.timedelta(days=1)
+        else:
+            from_date = datetime.datetime.strptime(from_date, "%d.%m.%Y")
+        if not to_date or from_date == to_date:
+            to_date = from_date + datetime.timedelta(days=1)
+        else:
+            to_date = datetime.datetime.strptime(to_date, "%d.%m.%Y") + datetime.timedelta(days=1)
+
+        # Load managed objects
+        mos = ManagedObject.objects.filter(is_managed=True)
+        if not request.user.is_superuser:
+            mos = mos.filter(administrative_domain__in=UserAccess.get_domains(request.user))
+        if selector:
+            mos = mos.filter(ManagedObjectSelector.objects.get(id=int(selector)).Q)
+        if administrative_domain:
+            mos = mos.filter(
+                administrative_domain__in=AdministrativeDomain.get_nested_ids(
+                    int(administrative_domain)
+                )
+            )
+        if object_profile:
+            mos = mos.filter(object_profile=object_profile)
+
+        mo_attrs = namedtuple("MOATTRs", [c for c in cols if c.startswith("object")])
+        containers_address = {}
+        if "object_container" in columns_filter:
+            containers_address = ReportContainerData(set(mos.values_list("id", flat=True)))
+            containers_address = dict(list(containers_address.extract()))
+
+        moss = {}
+        for row in mos.values_list(
+            "id", "name", "address", "platform", "administrative_domain__name", "segment", "id"
+        ):
+            moss[row[0]] = mo_attrs(
+                *[
+                    row[1],
+                    row[2],
+                    str(Platform.get_by_id(row[3]) if row[3] else ""),
+                    row[4],
+                    str(NetworkSegment.get_by_id(row[5])) if row[5] else "",
+                    containers_address.get(row[6], "") if containers_address and row[6] else "",
+                ]
+            )
+        # get maximum metrics
+        ifaces_metrics = get_interface_metrics(mos, from_date, to_date)
+        # get interfaces
+        mos_id = list(mos.values_list("id", flat=True))
+        rld = load(mos_id, description, interface_profile)
+
+        for mm in ifaces_metrics:
+            mo_id = moss[int(mm.id)]
+            uplinks = set(mm.data.uplinks)
+            links = []
+
+            # find uplinks
+            for l in Link.object_links(mm):
+                local_interfaces = []
+                remote_interfaces = []
+                remote_objects = set()
+                for i in l.interfaces:
+                    if i.managed_object.id == mm.id:
+                        local_interfaces += [i]
+                    else:
+                        remote_interfaces += [i]
+                        remote_objects.add(i.managed_object)
+                if len(remote_objects) == 1:
+                    ro = remote_objects.pop()
+                    if ro.id in uplinks:
+                        role = "uplink"
+                    else:
+                        role = "downlink"
+                    links += [
+                        {
+                            "id": l.id,
+                            "role": role,
+                            "local_interface": sorted(
+                                local_interfaces, key=lambda x: alnum_key(x.name)
+                            ),
+                            "remote_object": ro,
+                            "remote_interface": sorted(
+                                remote_interfaces, key=lambda x: alnum_key(x.name)
+                            ),
+                            "remote_status": "up" if ro.get_status() else "down",
+                        }
+                    ]
+
+            for i in rld[mm.id]:
+                iface_metrics = ifaces_metrics[mm].get(i)
+                if iface_metrics:
+                    if not exclude_zero:
+                        if iface_metrics["max_load_in"] == 0 and iface_metrics["max_load_out"] == 0:
+                            continue
+                    row2 = [
+                        str(mm.id),
+                        mm.name,
+                        mm.address,
+                        getattr(mo_id, "object_platform"),
+                        getattr(mo_id, "object_adm_domain"),
+                        getattr(mo_id, "object_segment"),
+                        getattr(mo_id, "object_container"),
+                        i,
+                        rld[mm.id][i]["description"],
+                        rld[mm.id][i]["bandwidth"],
+                        iface_metrics["max_load_in"],
+                        iface_metrics["max_load_in_time"],
+                        iface_metrics["max_load_out"],
+                        iface_metrics["max_load_out_time"],
+                        iface_metrics["avg_load_in"],
+                        iface_metrics["avg_load_out"],
+                        "",
+                        "",
+                        "",
+                        "",
+                        "",
+                        "",
+                        "",
+                        "",
+                        "",
+                    ]
+
+                    if links:
+                        for link in links:
+                            if link["role"] == "uplink":
+                                ifname_uplink = link["local_interface"][0].name
+                                # uplink_data = get_uplink_metrics([mm], [ifname_uplink], from_date, to_date)
+                                row2[16] = ifname_uplink
+                                row2[17] = link["local_interface"][0].description
+                                row2[22] = ifaces_metrics[mm][ifname_uplink]["avg_load_in"]
+                                row2[23] = ifaces_metrics[mm][ifname_uplink]["avg_load_out"]
+                                row2[18] = ifaces_metrics[mm][ifname_uplink]["max_load_in"]
+                                row2[20] = ifaces_metrics[mm][ifname_uplink]["max_load_out"]
+                                row2[19] = ifaces_metrics[mm][ifname_uplink]["max_load_in_time"]
+                                row2[21] = ifaces_metrics[mm][ifname_uplink]["max_load_out_time"]
+                                r += [translate_row(row2, cmap)]
+                    else:
+                        r += [translate_row(row2, cmap)]
+
+        filename = "metrics_detail_report_%s" % datetime.datetime.now().strftime("%Y%m%d")
+        if o_format == "csv":
+            response = HttpResponse(content_type="text/csv")
+            response["Content-Disposition"] = 'attachment; filename="%s.csv"' % filename
+            writer = csv.writer(response, dialect="excel", delimiter=",", quoting=csv.QUOTE_MINIMAL)
+            writer.writerows(r)
+            return response
+        elif o_format == "xlsx":
+            response = StringIO()
+            wb = xlsxwriter.Workbook(response)
+            cf1 = wb.add_format({"bottom": 1, "left": 1, "right": 1, "top": 1})
+            ws = wb.add_worksheet("Metrics")
+            max_column_data_length = {}
+            for rn, x in enumerate(r):
+                for cn, c in enumerate(x):
+                    if rn and (
+                        r[0][cn] not in max_column_data_length
+                        or len(str(c)) > max_column_data_length[r[0][cn]]
+                    ):
+                        max_column_data_length[r[0][cn]] = len(str(c))
+                    ws.write(rn, cn, c, cf1)
+            ws.autofilter(0, 0, rn, cn)
+            ws.freeze_panes(1, 0)
+            for cn, c in enumerate(r[0]):
+                # Set column width
+                width = get_column_width(c)
+                if enable_autowidth and width < max_column_data_length[c]:
+                    width = max_column_data_length[c]
+                ws.set_column(cn, cn, width=width)
+            wb.close()
+            response.seek(0)
+            response = HttpResponse(response.getvalue(), content_type="application/vnd.ms-excel")
+            response["Content-Disposition"] = 'attachment; filename="%s.xlsx"' % filename
+            response.close()
+            return response

services/web/translations/ru/LC_MESSAGES/messages.po

@@ -24,6 +24,7 @@ msgstr ""
 #: services/web/apps/fm/reportalarmdetail/views.py:108
 #: services/web/apps/inv/reportifacestatus/views.py:98
 #: services/web/apps/inv/reportlinkdetail/views.py:109
+#: services/web/apps/inv/reportmaxmetrics/views.py:75
 #: services/web/apps/inv/reportmetrics/views.py:61
 #: services/web/apps/sa/reportobjectdetail/views.py:95
 msgid "Reports"
@@ -403,10 +404,12 @@ msgid "DURATION_SEC"
 msgstr "Длительность (сек)"
 
 #: services/web/apps/fm/reportalarmdetail/views.py:225
+#: services/web/apps/inv/reportmaxmetrics/views.py:284
 msgid "OBJECT_NAME"
 msgstr "Имя объекта"
 
 #: services/web/apps/fm/reportalarmdetail/views.py:226
+#: services/web/apps/inv/reportmaxmetrics/views.py:285
 msgid "OBJECT_ADDRESS"
 msgstr "IP Адрес"
 
@@ -419,10 +422,12 @@ msgid "OBJECT_PROFILE"
 msgstr "Профиль"
 
 #: services/web/apps/fm/reportalarmdetail/views.py:229
+#: services/web/apps/inv/reportmaxmetrics/views.py:287
 msgid "OBJECT_ADMDOMAIN"
 msgstr "З.О."
 
 #: services/web/apps/fm/reportalarmdetail/views.py:230
+#: services/web/apps/inv/reportmaxmetrics/views.py:286
 msgid "OBJECT_PLATFORM"
 msgstr "Платформа"
 
@@ -1087,6 +1092,87 @@ msgstr "Статусы интерфейсов"
 msgid "Link Detail"
 msgstr "Детализация по линкам"
 
+#: services/web/apps/inv/reportmaxmetrics/views.py:75
+#: services/web/apps/inv/reportmaxmetrics/views.py:76
+msgid "Load Metrics max"
+msgstr "Отчет по пиковой загрузке"
+
+#: services/web/apps/inv/reportmaxmetrics/views.py:288
+msgid "OBJECT_SEGMENT"
+msgstr "Сегмент объекта"
+
+#: services/web/apps/inv/reportmaxmetrics/views.py:290
+msgid "IFACE_NAME"
+msgstr "Имя интерфейса"
+
+#: services/web/apps/inv/reportmaxmetrics/views.py:291
+msgid "IFACE_DESCRIPTION"
+msgstr "Описание интерфейса"
+
+#: services/web/apps/inv/reportmaxmetrics/views.py:292
+msgid "IFACE_SPEED"
+msgstr "Скорость интерфейса"
+
+#: services/web/apps/inv/reportmaxmetrics/views.py:293
+msgid "MAX_LOAD_IN, Mbps"
+msgstr "Максимальная загрузка входящий, Мбит/с"
+
+#: services/web/apps/inv/reportmaxmetrics/views.py:294
+msgid "MAX_LOAD_IN_TIME"
+msgstr "Время максимальной нагрузки, входящий"
+
+#: services/web/apps/inv/reportmaxmetrics/views.py:295
+msgid "MAX_LOAD_OUT, Mbps"
+msgstr "Максимальная загрузка исходящий, Мбит/с"
+
+#: services/web/apps/inv/reportmaxmetrics/views.py:296
+msgid "MAX_LOAD_OUT_TIME"
+msgstr "Время максимальной нагрузки, исходящий"
+
+#: services/web/apps/inv/reportmaxmetrics/views.py:297
+msgid "AVG_LOAD_IN, Mbps"
+msgstr "Средняя загрузка входящий, Мбит/с"
+
+#: services/web/apps/inv/reportmaxmetrics/views.py:298
+msgid "AVG_LOAD_OUT, Mbps"
+msgstr "Средняя загрузка исходящий, Мбит/с"
+
+#: services/web/apps/inv/reportmaxmetrics/views.py:299
+msgid "UPLINK_IFACE_NAME"
+msgstr "Имя интерфейса Uplink"
+
+#: services/web/apps/inv/reportmaxmetrics/views.py:300
+msgid "UPLINK_IFACE_DESCRIPTION"
+msgstr "Описание интерфейса Uplink"
+
+#: services/web/apps/inv/reportmaxmetrics/views.py:301
+msgid "UPLINK_MAX_LOAD_IN, Mbps"
+msgstr "Максимальная загрузка Uplink входящий, Мбит/с"
+
+#: services/web/apps/inv/reportmaxmetrics/views.py:302
+msgid "UPLINK_MAX_TIME_IN"
+msgstr "Время максимальной нагрузки Uplink, входящий"
+
+#: services/web/apps/inv/reportmaxmetrics/views.py:303
+msgid "UPLINK_MAX_LOAD_OUT, Mbps"
+msgstr "Максимальная загрузка Uplink исходящий, Мбит/с"
+
+#: services/web/apps/inv/reportmaxmetrics/views.py:304
+msgid "UPLINK_MAX_TIME_OUT"
+msgstr "Время максимальной нагрузки Uplink, исходящий"
+
+#: services/web/apps/inv/reportmaxmetrics/views.py:305
+msgid "UPLINK_AVG_LOAD_IN, Mbps"
+msgstr "Средняя загрузка Uplink входящий, Мбит/с"
+
+#: services/web/apps/inv/reportmaxmetrics/views.py:306
+msgid "UPLINK_AVG_LOAD_OUT, Mbps"
+msgstr "Средняя загрузка Uplink исходящий, Мбит/с"
+
+#: services/web/apps/inv/reportmaxmetrics/views.py:307
+msgid "UPLINK_IFACE_SPEED"
+msgstr "Скорость интерфейса Uplink"
+
 #: services/web/apps/inv/reportmetrics/views.py:61
 #: services/web/apps/inv/reportmetrics/views.py:62
 msgid "Load Metrics"

services/web/translations/ru/LC_MESSAGES/messages_js.po

@@ -1846,6 +1846,7 @@ msgstr "Изменен"
 #: ui/web/fm/event/Application.js:162 ui/web/fm/event/EventPanel.js:24
 #: ui/web/fm/event/EventPanel.js:138 ui/web/fm/mib/Application.js:37
 #: ui/web/fm/reportalarmdetail/Application.js:152
+#: ui/web/inv/reportmaxmetrics/Application.js:110
 #: ui/web/inv/reportmetrics/Application.js:19
 #: ui/web/inv/reportmetrics/Application.js:37
 #: ui/web/inv/reportmetrics/Application.js:48
@@ -1955,6 +1956,7 @@ msgstr "Оба"
 #: ui/web/fm/event/EventPanel.js:95
 #: ui/web/fm/reportalarmdetail/Application.js:50
 #: ui/web/fm/reportalarmdetail/Application.js:154
+#: ui/web/inv/reportmaxmetrics/Application.js:28
 #: ui/web/inv/reportmetrics/Application.js:92
 msgid "From"
 msgstr "С"
@@ -1963,6 +1965,7 @@ msgstr "С"
 #: ui/web/fm/event/EventPanel.js:101
 #: ui/web/fm/reportalarmdetail/Application.js:59
 #: ui/web/fm/reportalarmdetail/Application.js:155
+#: ui/web/inv/reportmaxmetrics/Application.js:38
 #: ui/web/inv/reportmetrics/Application.js:102
 msgid "To"
 msgstr "По"
@@ -1972,6 +1975,7 @@ msgstr "По"
 #: ui/web/fm/reportalarmdetail/Application.js:67
 #: ui/web/inv/map/Application.js:34
 #: ui/web/inv/reportlinkdetail/Application.js:25
+#: ui/web/inv/reportmaxmetrics/Application.js:47
 #: ui/web/inv/reportmetrics/Application.js:111
 #: ui/web/maintenance/maintenance/Application.js:203
 #: ui/web/sa/managedobject/Application.js:260
@@ -1985,6 +1989,7 @@ msgstr "Сегмент"
 #: ui/web/fm/reportalarmdetail/Application.js:76
 #: ui/web/inv/reportifacestatus/Application.js:38
 #: ui/web/inv/reportlinkdetail/Application.js:34
+#: ui/web/inv/reportmaxmetrics/Application.js:56
 #: ui/web/inv/reportmetrics/Application.js:120
 #: ui/web/sa/reportobjectdetail/Application.js:34