From a07b1086298e95bbbd6939a57ac6ae61dad1fff2 Mon Sep 17 00:00:00 2001 From: Andrey Vertiprahov Date: Sat, 5 Feb 2022 12:14:28 +0500 Subject: [PATCH 1/2] Optimize query on Report Moved MAC for reduce memory consumption. --- services/web/apps/inv/reportmovedmac/views.py | 29 ++++++++++++++----- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/services/web/apps/inv/reportmovedmac/views.py b/services/web/apps/inv/reportmovedmac/views.py index bac69bb676..ab0baf65ac 100644 --- a/services/web/apps/inv/reportmovedmac/views.py +++ b/services/web/apps/inv/reportmovedmac/views.py @@ -65,17 +65,32 @@ def get_column_width(name): return 15 -MAC_MOVED_QUERY = f"""SELECT +""" +Find mac, that more 1 unique interface the requested time range +groupUniqArray has high memory consumption (that groupUniqArray), so used subquery for filter data. +""" +MAC_MOVED_QUERY = f""" +SELECT managed_object, - MACNumToString(mac) as smac, + smac, dictGetString('{config.clickhouse.db_dictionaries}.managedobject', 'name', managed_object), dictGetString('{config.clickhouse.db_dictionaries}.managedobject', 'address', managed_object), dictGetString('{config.clickhouse.db_dictionaries}.managedobject', 'administrative_domain', managed_object), - groupUniqArray((interface, toUnixTimestamp(ts))) as ifaces, - groupUniqArray(interface) as migrate_ifaces, - uniqExact(interface) - FROM mac - WHERE %%s and %s and date >= '%%s' and date < '%%s' group by mac, managed_object, vlan having uniqExact(interface) > 1 + arrayReduce('groupUniqArray', ifaces), + arrayReduce('groupUniqArray', migrate_ifaces), + iface_count +FROM ( + SELECT + managed_object, + MACNumToString(mac) as smac, + groupArray((interface, toUnixTimestamp(ts))) as ifaces, + groupArray(interface) as migrate_ifaces, + uniq(interface) as iface_count + FROM mac + WHERE %%s and %s and date >= '%%s' and date < '%%s' + GROUP BY mac, managed_object, vlan + HAVING iface_count > 1 +) """ % " AND ".join( "(mac < %s or mac > %s)" % (int(MAC(x[0])), int(MAC(x[1]))) for x in MULTICAST_MACS ) -- GitLab From 77479220d3f72f6cf849982aac22cfc03dafa773 Mon Sep 17 00:00:00 2001 From: Andrey Vertiprahov Date: Sat, 5 Feb 2022 12:17:58 +0500 Subject: [PATCH 2/2] Fix flake. --- services/web/apps/inv/reportmovedmac/views.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/web/apps/inv/reportmovedmac/views.py b/services/web/apps/inv/reportmovedmac/views.py index ab0baf65ac..ddefb92166 100644 --- a/services/web/apps/inv/reportmovedmac/views.py +++ b/services/web/apps/inv/reportmovedmac/views.py @@ -66,11 +66,11 @@ def get_column_width(name): """ -Find mac, that more 1 unique interface the requested time range -groupUniqArray has high memory consumption (that groupUniqArray), so used subquery for filter data. +Find mac, that more 1 unique interface the requested time range groupUniqArray has high + memory consumption (that groupUniqArray), so used subquery for filter data. """ MAC_MOVED_QUERY = f""" -SELECT +SELECT managed_object, smac, dictGetString('{config.clickhouse.db_dictionaries}.managedobject', 'name', managed_object), -- GitLab