Initial zone-files generation fails
Steps to reproduce
-
Install all pre-requisites
-
Setup AAA,Zone profile, DNSServer, DNSZone
-
Run in console of Noc-node
./noc datastream rebuild --datastream=dnszone
What is the current bug behavior?
Generation fails with exception
What is the expected correct behavior?
Zone-files should be generated
Relevant logs and/or screenshots
root@noctest:/opt/noc# ./noc datastream rebuild --datastream=dnszone
2023-03-27 12:22:57,351 [noc.core.ioloop.util] Setting up asyncio event loop policy
2023-03-27 12:22:57,351 [noc.core.ioloop.util] Setting up default event loop
2023-03-27 12:22:57,374 [noc.core.dcs.base] Resolve near service postgres
2023-03-27 12:22:57,383 [noc.core.dcs.base] Resolved near service postgres to ['10.200.204.152:5432']
2023-03-27 12:22:57,539 [noc.core.dcs.base] [mongo] Starting resolver (near=False)
2023-03-27 12:22:57,541 [noc.core.dcs.base] [mongo] Set active services to: mongo: 10.200.204.152:27017
2023-03-27 12:22:57,541 [noc.core.dcs.base] [mongo] Stopping resolver
2023-03-27 12:22:57,542 [noc.core.mongo.connection] Connecting to MongoDB {'db': 'noc', 'username': 'noc', 'password': '********', 'authentication_source': 'noc', 'replicaSet': 'noc', 'readPreference': 'secondaryPreferred', 'maxIdleTimeMS': 60000, 'host': 'mongodb://noc:********@10.200.204.152:27017/noc'}
2023-03-27 12:22:57,546 [noc.core.loader.base] [datastream] Loading dnszone
2023-03-27 12:22:57,631 [noc.core.cache.base] Using cache backend: noc.core.cache.mongo.MongoCache
2023-03-27 12:22:58,032 [noc.core.debug] UNHANDLED EXCEPTION (2023-03-27 12:22:58.008030)
PROCESS: ./commands/datastream.py
VERSION: 22.2.3
BRANCH: HEAD CHANGESET: cf068c10
ERROR FINGERPRINT: c494ad7f-a265-53d6-86d9-ee107ab6273d
WORKING DIRECTORY: /opt/noc
EXCEPTION: <class 'django.core.exceptions.FieldError'> Cannot resolve keyword 'bi_id' into field. Choices are: description, dnszonerecord, effective_labels, id, is_auto_generated, labels, name, notification_group, notification_group_id, paid_till, profile, profile_id, project, project_id, serial, type
START OF TRACEBACK
------------------------------------------------------------------------
File: lib/python3.8/site-packages/django/db/models/sql/query.py (Line: 1887)
Function: add_fields
1880 # from the model on which the lookup failed.
1881 raise
1882 else:
1883 names = sorted([
1884 *get_field_names_from_opts(opts), *self.extra,
1885 *self.annotation_select, *self._filtered_relations
1886 ])
1887 ==> raise FieldError("Cannot resolve keyword %r into field. "
1888 "Choices are: %s" % (name, ", ".join(names)))
1889
1890 def add_ordering(self, *ordering):
1891 """
1892 Add items from the 'ordering' sequence to the query's "order by"
1893 clause. These items are either field names (not column names) --
Variables:
self = <django.db.models.sql.query.Query object at 0x7f93d9dcd610>
field_names = ['id', 'bi_id']
allow_m2m = True
alias = 'dns_dnszone'
opts = <Options for DNSZone>
cols = [Col(dns_dnszone, dns.DNSZone.id)]
name = 'bi_id'
join_info =
JoinInfo(final_field=<django.db.models.fields.AutoField: id>, targets=(<django.db.models.fields.AutoField: id>,), opts=<Options for DNSZone>, joins=['dns_dnszone'], path=[], transform_function=<function Query.setup_joins.<locals>.final_transformer at 0x7f93d9dd3b80>)
targets = (<django.db.models.fields.AutoField: id>,)
final_alias = 'dns_dnszone'
joins = ['dns_dnszone']
target = <django.db.models.fields.AutoField: id>
names =
['description',
'dnszonerecord',
'effective_labels',
'id',
'is_auto_generated',
'labels',
'name',
'notification_group',
'notification_group_id',
'paid_till',
'profile',
'profile_id',
'project',
'project_id',
'serial',
'type']
------------------------------------------------------------------------
File: lib/python3.8/site-packages/django/db/models/sql/query.py (Line: 2176)
Function: set_values
2169 for expr in self.group_by:
2170 if isinstance(expr, Ref) and expr.refs not in selected:
2171 expr = self.annotations[expr.refs]
2172 group_by.append(expr)
2173 self.group_by = tuple(group_by)
2174
2175 self.values_select = tuple(field_names)
2176 ==> self.add_fields(field_names, True)
2177
2178 @property
2179 def annotation_select(self):
2180 """
2181 Return the dictionary of aggregate columns that are not masked and
2182 should be used in the SELECT clause. Cache this result for performance.
Variables:
self = <django.db.models.sql.query.Query object at 0x7f93d9dcd610>
fields = ('id', 'bi_id')
field_names = ['id', 'bi_id']
extra_names = []
annotation_names = []
selected = frozenset({'id', 'bi_id'})
------------------------------------------------------------------------
File: lib/python3.8/site-packages/django/db/models/query.py (Line: 836)
Function: _values
829 return qs
830
831 def _values(self, *fields, **expressions):
832 clone = self._chain()
833 if expressions:
834 clone = clone.annotate(**expressions)
835 clone._fields = fields
836 ==> clone.query.set_values(fields)
837 return clone
838
839 def values(self, *fields, **expressions):
840 fields += tuple(expressions)
841 clone = self._values(*fields, **expressions)
842 clone._iterable_class = ValuesIterable
Variables:
self = <QuerySet [<DNSZone: powernet.com.ru>, <DNSZone: some.local>]>
fields = ('id', 'bi_id')
expressions = {}
clone = <QuerySet [<DNSZone: powernet.com.ru>, <DNSZone: some.local>]>
------------------------------------------------------------------------
File: lib/python3.8/site-packages/django/db/models/query.py (Line: 868)
Function: values_list
861 if field_id not in field_names:
862 break
863 expressions[field_id] = field
864 _fields.append(field_id)
865 else:
866 _fields.append(field)
867
868 ==> clone = self._values(*_fields, **expressions)
869 clone._iterable_class = (
870 NamedValuesListIterable if named
871 else FlatValuesListIterable if flat
872 else ValuesListIterable
873 )
874 return clone
Variables:
self = <QuerySet [<DNSZone: powernet.com.ru>, <DNSZone: some.local>]>
flat = False
named = False
fields = ('id', 'bi_id')
field_names = {'id', 'bi_id'}
_fields = ['id', 'bi_id']
expressions = {}
counter = 1
field = 'bi_id'
------------------------------------------------------------------------
File: lib/python3.8/site-packages/django/db/models/manager.py (Line: 85)
Function: manager_method
78 def check(self, **kwargs):
79 return []
80
81 @classmethod
82 def _get_queryset_methods(cls, queryset_class):
83 def create_method(name, method):
84 def manager_method(self, *args, **kwargs):
85 ==> return getattr(self.get_queryset(), name)(*args, **kwargs)
86 manager_method.__name__ = method.__name__
87 manager_method.__doc__ = method.__doc__
88 return manager_method
89
90 new_methods = {}
91 for name, method in inspect.getmembers(queryset_class, predicate=inspect.isfunction):
Variables:
self = <django.db.models.manager.Manager object at 0x7f93dbe39eb0>
args = ('id', 'bi_id')
kwargs = {}
name = 'values_list'
------------------------------------------------------------------------
File: commands/datastream.py (Line: 98)
Function: iter_id
91 yield f'{model_id}::{d["bi_id"]}'
92 else:
93 yield d["_id"]
94 if not d or (match and match["_id"]["$gt"] == d["_id"]):
95 break
96 match = {"_id": {"$gt": d["_id"]}}
97 else:
98 ==> for id, bi_id in m.objects.values_list("id", "bi_id").order_by("id"):
99 if as_bi_id:
100 yield f"{model_id}::{bi_id}"
101 else:
102 yield id
103
104 def get_total(self, model):
Variables:
self = <__main__.Command object at 0x7f93e5891df0>
model = (<class 'noc.dns.models.dnszone.DNSZone'>,)
as_bi_id = False
m = <class 'noc.dns.models.dnszone.DNSZone'>
model_id = 'dns.DNSZone'
------------------------------------------------------------------------
File: commands/datastream.py (Line: 160)
Function: <genexpr>
153 from multiprocessing.pool import ThreadPool
154
155 pool = ThreadPool(jobs)
156 iterable = pool.imap_unordered(
157 update_object, self.iter_id(model, datastream in self.BI_ID_DATASTREAM)
158 )
159 else:
160 ==> iterable = (
161 ds.bulk_update([b for b in bulk if b is not None])
162 for bulk in grouper(self.iter_id(model, datastream in self.BI_ID_DATASTREAM), BATCH)
163 )
164
165 if not self.no_progressbar:
166 # Disable logging
Variables:
.0 = <itertools.zip_longest object at 0x7f93d9dd8090>
ds = <class 'noc.services.datastream.streams.dnszone.DNSZoneDataStream'>
------------------------------------------------------------------------
File: lib/python3.8/site-packages/progressbar/bar.py (Line: 546)
Function: __next__
539 return self
540
541 def __iter__(self):
542 return self
543
544 def __next__(self):
545 try:
546 ==> value = next(self._iterable)
547 if self.start_time is None:
548 self.start()
549 else:
550 self.update(self.value + 1)
551 return value
552 except StopIteration:
Variables:
self = <progressbar.bar.ProgressBar object at 0x7f93dbe2c7c0>
------------------------------------------------------------------------
File: lib/python3.8/site-packages/progressbar/shortcuts.py (Line: 10)
Function: progressbar
3
4 def progressbar(iterator, min_value=0, max_value=None,
5 widgets=None, prefix=None, suffix=None, **kwargs):
6 progressbar = bar.ProgressBar(
7 min_value=min_value, max_value=max_value,
8 widgets=widgets, prefix=prefix, suffix=suffix, **kwargs)
9
10 ==> for result in progressbar(iterator):
11 yield result
Variables:
iterator =
<generator object Command.handle_rebuild.<locals>.<genexpr> at 0x7f93d9ddc040>
min_value = 0
max_value = 0.02
widgets = None
prefix = None
suffix = None
kwargs = {}
progressbar = <progressbar.bar.ProgressBar object at 0x7f93dbe2c7c0>
------------------------------------------------------------------------
File: core/management/base.py (Line: 212)
Function: progress
205 :return:
206 """
207 if self.no_progressbar:
208 yield from iter
209 else:
210 import progressbar
211
212 ==> yield from progressbar.progressbar(iter, max_value=max_value)
213
214 def show_usage(self, start, stop):
215 """
216 Show resource usage
217 :param start:
218 :param stop:
Variables:
self = <__main__.Command object at 0x7f93e5891df0>
iter =
<generator object Command.handle_rebuild.<locals>.<genexpr> at 0x7f93d9ddc040>
max_value = 0.02
progressbar =
<module 'progressbar' from '/opt/noc/lib/python3.8/site-packages/progressbar/__init__.py'>
------------------------------------------------------------------------
File: commands/datastream.py (Line: 171)
Function: handle_rebuild
164
165 if not self.no_progressbar:
166 # Disable logging
167 from noc.core.datastream.base import logger
168
169 logger.setLevel(logging.CRITICAL)
170
171 ==> for _ in self.progress(iterable, max_value=total / BATCH):
172 if self.no_progressbar and n == next_report:
173 self.print("[%02d%%]" % ((n * 100) // total))
174 next_report += report_interval
175 n += 1
176 self.print("Done")
177
Variables:
self = <__main__.Command object at 0x7f93e5891df0>
datastream = 'dnszone'
jobs = 0
args = ()
kwargs = {}
update_object =
<function Command.handle_rebuild.<locals>.update_object at 0x7f93e29cbc10>
grouper = <function Command.handle_rebuild.<locals>.grouper at 0x7f93e29cbd30>
do_update = <function Command.handle_rebuild.<locals>.do_update at 0x7f93e29cbca0>
model = <class 'noc.dns.models.dnszone.DNSZone'>
total = 2
STEP = 100
BATCH = 100
n = 1
report_interval = 1
next_report = 1
iterable =
<generator object Command.handle_rebuild.<locals>.<genexpr> at 0x7f93d9ddc040>
logger = <Logger noc.core.datastream.base (CRITICAL)>
ds = <class 'noc.services.datastream.streams.dnszone.DNSZoneDataStream'>
------------------------------------------------------------------------
File: commands/datastream.py (Line: 69)
Function: handle
62 get_parser.add_argument("--filter", action="append", help="Datastream filter")
63 get_parser.add_argument("objects", nargs=argparse.REMAINDER, help="Object ids")
64 # clean
65 clean_parser = subparsers.add_parser("clean")
66 clean_parser.add_argument("--datastream", help="Datastream name")
67
68 def handle(self, cmd, *args, **options):
69 ==> getattr(self, "handle_%s" % cmd)(*args, **options)
70
71 def handle_list(self):
72 for ds_name in sorted(loader.iter_classes()):
73 self.print(ds_name)
74
75 def iter_id(self, model, as_bi_id: bool = False):
Variables:
self = <__main__.Command object at 0x7f93e5891df0>
cmd = 'rebuild'
args = ()
options = {'datastream': 'dnszone', 'jobs': 0}
------------------------------------------------------------------------
File: core/management/base.py (Line: 84)
Function: run_from_argv
77
78 yappi.start()
79 try:
80 if show_usage:
81 import resource
82
83 start_usage = resource.getrusage(resource.RUSAGE_SELF)
84 ==> return self.handle(*args, **cmd_options) or 0
85 except CommandError as e:
86 self.print(str(e))
87 return 1
88 except KeyboardInterrupt:
89 self.print("Ctrl+C")
90 return 3
Variables:
self = <__main__.Command object at 0x7f93e5891df0>
argv = ['rebuild', '--datastream=dnszone']
parser =
ArgumentParser(prog='noc datastream', usage=None, description=None, formatter_class=<class 'argparse.HelpFormatter'>, conflict_handler='error', add_help=True)
options = Namespace(cmd='rebuild', datastream='dnszone', jobs=0)
cmd_options = {'cmd': 'rebuild', 'datastream': 'dnszone', 'jobs': 0}
args = ()
loglevel = 'info'
enable_profiling = False
show_metrics = False
show_usage = False
error_report = <function error_report at 0x7f93e43d0b80>
------------------------------------------------------------------------
END OF TRACEBACK
100% (0.02 of 0.02) |###########################################################################| Elapsed Time: 0:00:00 ETA: 00:00:00Possible fixes
Paste NOC version (./noc about)
NOC version is: 22.2.3
OS: Ubuntu Linux 20.04