diff --git a/lib/app/reportdatasources/report_discoveryresult.py b/lib/app/reportdatasources/report_discoveryresult.py index 5a63d52d361e4259e63f5c870c39fdb88858e9fa..46b775aefe4fbab800382ed22410a8eac29b7f56 100644 --- a/lib/app/reportdatasources/report_discoveryresult.py +++ b/lib/app/reportdatasources/report_discoveryresult.py @@ -11,7 +11,6 @@ from __future__ import absolute_import from collections import namedtuple # Third-party modules -import six from pymongo import ReadPreference # NOC modules @@ -25,7 +24,6 @@ class ReportDiscoveryResult(BaseReportColumn): """Report for MO links detail""" builtin_sorted = True - multiple_series = True safe_output = False # Convert outpur object to string COLL_NAME = "noc.schedules.discovery.%s" # @todo from managedobjectprofile @@ -111,17 +109,14 @@ class ReportDiscoveryResult(BaseReportColumn): def extract(self): r = {} ids = set(self.sync_ids[:]) - + pid = {} for p in Pool.objects.filter(): pool_ids = ids.intersection( - set( - ManagedObject.objects.filter(pool=p, is_managed=True).values_list( - "id", flat=True - ) - ) + set(ManagedObject.objects.filter(pool=p).values_list("id", flat=True)) ) if not pool_ids: continue + pid.update({mo_id: p.name for mo_id in pool_ids}) r[p.name] = self.convert( get_db()[self.COLL_NAME % p.name] .with_options(read_preference=ReadPreference.SECONDARY_PREFERRED) @@ -130,7 +125,9 @@ class ReportDiscoveryResult(BaseReportColumn): ids.difference_update(pool_ids) if not ids: break - return list(six.itervalues(r)) + for i in self.sync_ids: + yield next(r[pid[i]], (i, ("",) * len(self.ATTRS))) + # return list(six.itervalues(r)) def convert(self, val): dresult = namedtuple("DResult", self.ATTRS) diff --git a/lib/app/reportdatasources/report_objectconfig.py b/lib/app/reportdatasources/report_objectconfig.py index 718d9433ab488896d8cb9c795e48dc63b0e7752b..9406756cfc3aaad1c06ad999f32a38a7e610fed4 100644 --- a/lib/app/reportdatasources/report_objectconfig.py +++ b/lib/app/reportdatasources/report_objectconfig.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # ---------------------------------------------------------------------- -# ReportObjectLinkCount datasource +# ReportObjectConfig datasource # ---------------------------------------------------------------------- # Copyright (C) 2007-2019 The NOC Project # See LICENSE for details @@ -25,16 +25,17 @@ class ReportObjectConfig(BaseReportColumn): builtin_sorted = True def extract(self): + pipeline = [ + {"$group": {"_id": "$object", "last_ts": {"$max": "$ts"}}}, + {"$sort": {"_id": 1}}, + ] + if len(self.sync_ids) < 20000: + # @todo Very large list slowest encode, need research + pipeline.insert(0, {"$match": {"object": {"$in": self.sync_ids}}}) value = ( get_db()["noc.gridvcs.config.files"] .with_options(read_preference=ReadPreference.SECONDARY_PREFERRED) - .aggregate( - [ - {"$match": {"object": {"$in": self.sync_ids}}}, - {"$group": {"_id": "$object", "last_ts": {"$max": "$ts"}}}, - {"$sort": {"_id": 1}}, - ] - ) + .aggregate(pipeline) ) for v in value: if not v["_id"]: diff --git a/lib/app/reportdatasources/report_objectifacestypestat.py b/lib/app/reportdatasources/report_objectifacestypestat.py index f95d2db6a01c19b4467bc67aea8d550927b93887..424731c7bc1132d5af2b8f6c65749e0c07c6954a 100644 --- a/lib/app/reportdatasources/report_objectifacestypestat.py +++ b/lib/app/reportdatasources/report_objectifacestypestat.py @@ -27,7 +27,8 @@ class ReportObjectIfacesTypeStat(BaseReportColumn): def extract(self): i_type = "physical" match = {"type": i_type} - if self.sync_ids: + if len(self.sync_ids) < 20000: + # @todo Very large list slowest encode, need research match = {"type": i_type, "managed_object": {"$in": self.sync_ids}} value = ( get_db()["noc.interfaces"] diff --git a/services/web/apps/sa/reportobjectdetail/views.py b/services/web/apps/sa/reportobjectdetail/views.py index 6759c01eaafe67381439f41a57f3c3b3c2986e2c..8f22e1e06f99e82fd6d64759d54bfa4fc8ae935a 100644 --- a/services/web/apps/sa/reportobjectdetail/views.py +++ b/services/web/apps/sa/reportobjectdetail/views.py @@ -328,22 +328,24 @@ class ReportObjectDetailApplication(ExtApplication): platform, version, tags, - ) in mos.values_list( - "id", - "name", - "address", - "is_managed", - "profile", - "object_profile__name", - "auth_profile__name", - "administrative_domain__name", - "segment", - "vendor", - "platform", - "version", - "tags", - ).order_by( - "id" + ) in ( + mos.values_list( + "id", + "name", + "address", + "is_managed", + "profile", + "object_profile__name", + "auth_profile__name", + "administrative_domain__name", + "segment", + "vendor", + "platform", + "version", + "tags", + ) + .order_by("id") + .iterator() ): if (mos_filter and mo_id not in mos_filter) or not mos_id: continue @@ -359,38 +361,40 @@ class ReportObjectDetailApplication(ExtApplication): serial, hw_ver, boot_prom, patch = next(roa)[0] # noqa else: serial, hw_ver, boot_prom, patch = "", "", "", "" # noqa - r += [ - translate_row( - row( - [ - mo_id, - name, - address, - next(hn)[0], - "managed" if is_managed else "unmanaged", - Profile.get_by_id(sa_profile), - o_profile, - Vendor.get_by_id(vendor) if vendor else "", - Platform.get_by_id(platform) if platform else "", - hw_ver, - Firmware.get_by_id(version) if version else "", - boot_prom, - # Serial - mo_serials[0].get("serial", "") or serial, - patch or "", - auth_profile, - _("Yes") if avail.get(mo_id, None) else _("No"), - ad, - mo_continer[0], - NetworkSegment.get_by_id(m_segment) if m_segment else "", - next(iface_count)[0], - next(link_count)[0], - next(rc)[0], - ] - ), - cmap, - ) - ] + r.append( + [ + translate_row( + row( + [ + mo_id, + name, + address, + next(hn)[0], + "managed" if is_managed else "unmanaged", + Profile.get_by_id(sa_profile), + o_profile, + Vendor.get_by_id(vendor) if vendor else "", + Platform.get_by_id(platform) if platform else "", + hw_ver, + Firmware.get_by_id(version) if version else "", + boot_prom, + # Serial + mo_serials[0].get("serial", "") or serial, + patch or "", + auth_profile, + _("Yes") if avail.get(mo_id, None) else _("No"), + ad, + mo_continer[0], + NetworkSegment.get_by_id(m_segment) if m_segment else "", + next(iface_count)[0], + next(link_count)[0], + next(rc)[0], + ] + ), + cmap, + ) + ] + ) if "adm_path" in columns_filter: r[-1].extend([ad] + list(ad_path[ad])) if "interface_type_count" in columns_filter: @@ -414,7 +418,7 @@ class ReportObjectDetailApplication(ExtApplication): if o_format == "csv": response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = 'attachment; filename="%s.csv"' % filename - writer = csv.writer(response, dialect="excel", delimiter=";") + writer = csv.writer(response, dialect="excel", delimiter=";", quotechar='"') writer.writerows(r) return response elif o_format == "xlsx":