config.py 35.5 KB
Newer Older
1
2
3
# ----------------------------------------------------------------------
# NOC config
# ----------------------------------------------------------------------
4
# Copyright (C) 2007-2020 The NOC Project
5
6
7
8
9
# See LICENSE for details
# ----------------------------------------------------------------------

# Python modules
import logging
10
import os
Aleksey Shirokih's avatar
Aleksey Shirokih committed
11
import socket
12
import sys
13
from collections import namedtuple
Dmitry Volodin's avatar
Dmitry Volodin committed
14
from urllib.parse import quote as urllib_quote
15

16
17
# NOC modules
from noc.core.config.base import BaseConfig, ConfigSection
18
from noc.core.config.params import (
19
20
21
22
23
24
25
26
27
28
29
    StringParameter,
    MapParameter,
    IntParameter,
    BooleanParameter,
    HandlerParameter,
    SecondsParameter,
    FloatParameter,
    ServiceParameter,
    SecretParameter,
    ListParameter,
)
30
31
32


class Config(BaseConfig):
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
    loglevel = MapParameter(
        default="info",
        mappings={
            # pylint: disable=used-before-assignment
            "critical": logging.CRITICAL,
            # pylint: disable=used-before-assignment
            "error": logging.ERROR,
            # pylint: disable=used-before-assignment
            "warning": logging.WARNING,
            # pylint: disable=used-before-assignment
            "info": logging.INFO,
            # pylint: disable=used-before-assignment
            "debug": logging.DEBUG,
        },
    )
48

49
    class activator(ConfigSection):
50
        tos = IntParameter(min=0, max=255, default=0)
51
52
53
54
55
56
57
        script_threads = IntParameter(default=10)
        buffer_size = IntParameter(default=1048576)
        connect_retries = IntParameter(default=3, help="retries on immediate disconnect")
        connect_timeout = IntParameter(default=3, help="timeout after immediate disconnect")
        http_connect_timeout = IntParameter(default=20)
        http_request_timeout = IntParameter(default=30)
        http_validate_cert = BooleanParameter(default=False)
Aleksey Shirokih's avatar
Aleksey Shirokih committed
58

59
60
61
62
63
64
65
    class audit(ConfigSection):
        command_ttl = SecondsParameter(default="1m")
        login_ttl = SecondsParameter(default="1m")
        reboot_ttl = SecondsParameter(default="0")
        config_ttl = SecondsParameter(default="1y")
        db_ttl = SecondsParameter(default="5y")
        config_changed_ttl = SecondsParameter(default="1y")
Aleksey Shirokih's avatar
Aleksey Shirokih committed
66

67
68
69
70
71
72
73
74
    class backup(ConfigSection):
        keep_days = SecondsParameter(default="14d")
        keep_weeks = SecondsParameter(default="12w")
        keep_day_of_week = IntParameter(default="6")
        keep_months = IntParameter(default="12")
        keep_day_of_month = IntParameter(default="1")

    class bi(ConfigSection):
Andrey Vertiprahov's avatar
Andrey Vertiprahov committed
75
        language = StringParameter(default="en", help="Language BI interface")
76
77
78
79
80
81
        query_threads = IntParameter(default=10)
        extract_delay_alarms = SecondsParameter(default="1h")
        clean_delay_alarms = SecondsParameter(default="1d")
        reboot_interval = SecondsParameter(default="1M")
        extract_delay_reboots = SecondsParameter(default="1h")
        clean_delay_reboots = SecondsParameter(default="1d")
82
        chunk_size = IntParameter(default=3000)
83
        extract_window = SecondsParameter(default="1d")
84
85
86
        enable_alarms = BooleanParameter(default=False)
        enable_reboots = BooleanParameter(default=False)
        enable_managedobjects = BooleanParameter(default=False)
87
        enable_alarms_archive = BooleanParameter(default=False)
88
89
90
91
92
93
94
95
96
97
        alarms_archive_policy = MapParameter(
            default="weekly",
            mappings={
                "weekly": '{{doc["clear_timestamp"].strftime("y%Yw%W")}}',
                "monthly": '{{doc["clear_timestamp"].strftime("y%Ym%m")}}',
                "quarterly": '{{doc["clear_timestamp"].strftime("y%Y")}}'
                '_quarter{{(doc["clear_timestamp"].month-1)//3 + 1}}',
                "yearly": '{{doc["clear_timestamp"].strftime("y%Y")}}',
            },
        )
98
        alarms_archive_batch_limit = IntParameter(default=10000)
99

Dmitry Volodin's avatar
Dmitry Volodin committed
100
101
102
    class biosegmentation(ConfigSection):
        processed_trials_ttl = SecondsParameter(default="1w")

Aleksey Shirokih's avatar
Aleksey Shirokih committed
103
104
    brand = StringParameter(default="NOC")

105
106
107
108
109
110
    class cache(ConfigSection):
        vcinterfacescount = SecondsParameter(default="1h")
        vcprefixes = SecondsParameter(default="1h")
        cache_class = StringParameter(default="noc.core.cache.mongo.MongoCache")
        default_ttl = SecondsParameter(default="1d")
        pool_size = IntParameter(default=8)
Aleksey Shirokih's avatar
Aleksey Shirokih committed
111

112
113
114
    class card(ConfigSection):
        language = StringParameter(default="en")
        alarmheat_tooltip_limit = IntParameter(default=5)
115

116
    class chwriter(ConfigSection):
EKbfh's avatar
EKbfh committed
117
118
119
120
121
122
123
124
125
        batch_size = IntParameter(default=50000, help="Size of one portion from queue")
        records_buffer = IntParameter(default=1000000, help="Own buffer of messages from queue")
        batch_delay_ms = IntParameter(default=10000, help="Send every period time")
        channel_expire_interval = SecondsParameter(
            default="5M", help="Close channel when no messages in this time"
        )
        suspend_timeout_ms = IntParameter(
            default=3000, help="How much time to sleep before continue"
        )
126
        # Topic to listen
EKbfh's avatar
EKbfh committed
127
        topic = StringParameter(default="chwriter", help="Topic in queue to listen to")
Dmitry Volodin's avatar
Dmitry Volodin committed
128
129
        # <address:port> of ClickHouse server to write
        write_to = StringParameter()
EKbfh's avatar
EKbfh committed
130
131
132
        max_in_flight = IntParameter(
            default=10, help="How many parts read simultaneously from queue"
        )
133

134
    class classifier(ConfigSection):
135
        lookup_handler = HandlerParameter(default="noc.services.classifier.rulelookup.RuleLookup")
136
137
        default_interface_profile = StringParameter(default="default")
        default_rule = StringParameter(default="Unknown | Default")
138
139

    class clickhouse(ConfigSection):
140
        rw_addresses = ServiceParameter(service="clickhouse", wait=True)
141
        db = StringParameter(default="noc")
142
143
144
        rw_user = StringParameter(default="default")
        rw_password = SecretParameter()
        ro_addresses = ServiceParameter(service="clickhouse", wait=True)
145
        ro_user = StringParameter(default="readonly")
Aleksey Shirokih's avatar
Aleksey Shirokih committed
146
        ro_password = SecretParameter()
Aleksey Shirokih's avatar
Aleksey Shirokih committed
147
148
        request_timeout = SecondsParameter(default="1h")
        connect_timeout = SecondsParameter(default="10s")
Aleksey Shirokih's avatar
Aleksey Shirokih committed
149
        default_merge_tree_granularity = IntParameter(default=8192)
150
        encoding = StringParameter(default="", choices=["", "deflate", "gzip"])
151
152
        # Enable LowCardinality fileds
        enable_low_cardinality = BooleanParameter(default=False)
Dmitry Volodin's avatar
Dmitry Volodin committed
153
        # Cluster name for sharded/replicated configuration
154
        # Matches appropriative <remote_servers> part
Dmitry Volodin's avatar
Dmitry Volodin committed
155
        cluster = StringParameter()
156
157
158
159
160
161
162
163
164
165
166
167
168
        # Cluster topology
        # Expression in form
        # <topology> ::= <shard> | <shard>,<topology>
        # <shard> ::= [<weight>]<replicas>
        # <weight> := <DIGITS>
        # <replicas> := <DIGITS>
        # Examples:
        # 1 - non-replicated, non-sharded configuration
        # 1,1 - 2 shards, non-replicated
        # 2,2 - 2 shards, 2 replicas in each
        # 3:2,2 - first shard has 2 replicas an weight 3,
        #   second shard has 2 replicas and weight 1
        cluster_topology = StringParameter(default="1")
169

Dmitry Volodin's avatar
Dmitry Volodin committed
170
171
172
    class collections(ConfigSection):
        allow_sharing = BooleanParameter(default=True)

173
174
175
176
177
178
179
    class consul(ConfigSection):
        token = SecretParameter()
        connect_timeout = SecondsParameter(default="5s")
        request_timeout = SecondsParameter(default="1h")
        near_retry_timeout = IntParameter(default=1)
        host = StringParameter(default="consul")
        port = IntParameter(default=8500)
180
        check_interval = SecondsParameter(default="10s")
181
182
183
        check_timeout = SecondsParameter(default="1s")
        release = SecondsParameter(default="1M")
        session_ttl = SecondsParameter(default="10s")
184
        lock_delay = SecondsParameter(default="20s")
185
186
187
188
189
190
191
192
193
        retry_timeout = SecondsParameter(default="1s")
        keepalive_attempts = IntParameter(default=5)
        base = StringParameter(default="noc", help="kv lookup base")

    class correlator(ConfigSection):
        max_threads = IntParameter(default=20)
        topology_rca_window = IntParameter(default=0)
        oo_close_delay = SecondsParameter(default="20s")
        discovery_delay = SecondsParameter(default="10M")
194
        auto_escalation = BooleanParameter(default=True)
195

196
    class customization(ConfigSection):
197
198
        favicon_url = StringParameter(default="/ui/web/img/logo_24x24_deep_azure.png")
        logo_url = StringParameter(default="/ui/web/img/logo_white.svg")
199
200
        logo_width = IntParameter(default=24)
        logo_height = IntParameter(default=24)
201
202
        branding_color = StringParameter(default="#ffffff")
        branding_background_color = StringParameter(default="#34495e")
203
        preview_theme = StringParameter(default="midnight")
204

205
206
207
208
209
210
211
212
213
    class date_time_formats(StringParameter):
        date_format = StringParameter(default="d.m.Y")
        datetime_format = StringParameter(default="d.m.Y H:i:s")
        month_day_format = StringParameter(default="F j")
        time_format = StringParameter(default="H:i:s")
        year_month_format = StringParameter(default="F Y")

    class dcs(ConfigSection):
        resolution_timeout = SecondsParameter(default="5M")
Dmitry Volodin's avatar
Dmitry Volodin committed
214
        resolver_expiration_timeout = SecondsParameter(default="10M")
215
216
217

    class discovery(ConfigSection):
        max_threads = IntParameter(default=20)
Dmitry Volodin's avatar
Dmitry Volodin committed
218
        sample = IntParameter(default=0)
219
220
221
222
223

    class dns(ConfigSection):
        warn_before_expired = SecondsParameter(default="30d")

    class escalator(ConfigSection):
224
        max_threads = IntParameter(default=5)
225
226
227
228
        retry_timeout = SecondsParameter(default="60s")
        tt_escalation_limit = IntParameter(default=10)
        ets = SecondsParameter(default="60s")
        wait_tt_check_interval = SecondsParameter(default="60s")
Dmitry Volodin's avatar
Dmitry Volodin committed
229
        sample = IntParameter(default=0)
230

Dmitry Volodin's avatar
Dmitry Volodin committed
231
232
233
    class etl(ConfigSection):
        compression = StringParameter(choices=["plain", "gzip", "bz2", "lzma"], default="gzip")

234
    class features(ConfigSection):
235
        use_uvloop = BooleanParameter(default=False)
236
237
238
239
        cp = BooleanParameter(default=True)
        sentry = BooleanParameter(default=False)
        traefik = BooleanParameter(default=False)
        cpclient = BooleanParameter(default=False)
240
241
242
243
244
245
246
247
248
        telemetry = BooleanParameter(
            default=False, help="Enable internal telemetry export to Clickhouse"
        )
        consul_healthchecks = BooleanParameter(
            default=True, help="While registering serive in consul also register health check"
        )
        service_registration = BooleanParameter(
            default=True, help="Permit consul self registration"
        )
Dmitry Volodin's avatar
Dmitry Volodin committed
249
        forensic = BooleanParameter(default=False)
250
251
252
253
254
255

    class fm(ConfigSection):
        active_window = SecondsParameter(default="1d")
        keep_events_wo_alarm = IntParameter(default=0)
        keep_events_with_alarm = IntParameter(default=-1)
        alarm_close_retries = IntParameter(default=5)
256
        outage_refresh = SecondsParameter(default="60s")
257
258
259
260
261
        total_outage_refresh = SecondsParameter(default="60s")

    class geocoding(ConfigSection):
        order = StringParameter(default="yandex,google")
        yandex_key = SecretParameter(default="")
262
        yandex_apikey = SecretParameter(default="")
263
264
        google_key = SecretParameter(default="")
        google_language = StringParameter(default="en")
265
        negative_ttl = SecondsParameter(default="7d", help="Period then saving bad result")
266
267

    class gis(ConfigSection):
Aleksey Shirokih's avatar
Aleksey Shirokih committed
268
        ellipsoid = StringParameter(default="PZ-90")
269
270
271
        enable_osm = BooleanParameter(default=True)
        enable_google_sat = BooleanParameter(default=False)
        enable_google_roadmap = BooleanParameter(default=False)
Aleksey Shirokih's avatar
Aleksey Shirokih committed
272
273
        tile_size = IntParameter(default=256, help="Tile size 256x256")
        tilecache_padding = IntParameter(default=0)
274

275
    global_n_instances = IntParameter(default=1)
276

277
278
    class grafanads(ConfigSection):
        db_threads = IntParameter(default=10)
Aleksey Shirokih's avatar
Aleksey Shirokih committed
279

280
281
282
283
284
285
    class http_client(ConfigSection):
        connect_timeout = SecondsParameter(default="10s")
        request_timeout = SecondsParameter(default="1h")
        user_agent = StringParameter(default="noc")
        buffer_size = IntParameter(default=128 * 1024)
        max_redirects = IntParameter(default=5)
286

287
288
289
290
291
292
293
        ns_cache_size = IntParameter(default=1000)
        resolver_ttl = SecondsParameter(default="3s")

        http_port = IntParameter(default=80)
        https_port = IntParameter(default=443)
        validate_certs = BooleanParameter(default=False, help="Have to be set as True")

294
295
296
297
298
    class initial(ConfigSection):
        admin_user_name = StringParameter(default="admin")
        admin_password = StringParameter(default="admin")
        admin_email = StringParameter(default="test@example.com")

299
    installation_name = StringParameter(default="Unconfigured installation")
300
    installation_id = StringParameter(default="")
301
302
303

    instance = IntParameter(default=0)

Dmitry Volodin's avatar
Dmitry Volodin committed
304
305
306
307
308
309
310
311
312
313
314
    class kafkasender(ConfigSection):
        bootstrap_servers = StringParameter()
        username = StringParameter()
        password = SecretParameter()
        sasl_mechanism = StringParameter(
            choices=["PLAIN", "GSSAPI", "SCRAM-SHA-256", "SCRAM-SHA-512"], default="PLAIN"
        )
        security_protocol = StringParameter(
            choices=["PLAINTEXT", "SASL_PLAINTEXT", "SSL", "SASL_SSL"], default="PLAINTEXT"
        )

315
    language = StringParameter(default="en")
Dmitry Volodin's avatar
Dmitry Volodin committed
316
    language_code = StringParameter(default="en")
317

318
319
320
321
322
323
324
    class layout(ConfigSection):
        ring_ring_edge = IntParameter(default=150)
        ring_chain_edge = IntParameter(default=150)
        ring_chain_spacing = IntParameter(default=100)
        tree_horizontal_step = IntParameter(default=100)
        tree_vertical_step = IntParameter(default=100)
        tree_max_levels = IntParameter(default=4)
325
326
327
328
329
        spring_propulsion_force = FloatParameter(default=1.5)
        spring_edge_force = FloatParameter(default=1.2)
        spring_bubble_force = FloatParameter(default=2.0)
        spring_edge_spacing = IntParameter(default=190)
        spring_iterations = IntParameter(default=50)
330

Dmitry Volodin's avatar
Dmitry Volodin committed
331
332
    class liftbridge(ConfigSection):
        addresses = ServiceParameter(service="liftbridge", wait=True, near=True, full_result=False)
333
        max_message_size = IntParameter(default=-1, help="Max message size for GRPC client")
Dmitry Volodin's avatar
Dmitry Volodin committed
334

335
336
    listen = StringParameter(default="auto:0")

337
    log_format = StringParameter(default="%(asctime)s [%(name)s] %(message)s")
338

339
    thread_stack_size = IntParameter(default=0)
340
    version_format = StringParameter(default="%(version)s+%(branch)s.%(number)s.%(changeset)s")
341

342
343
344
345
    class logging(ConfigSection):
        log_api_calls = BooleanParameter(default=False)
        log_sql_statements = BooleanParameter(default=False)

346
347
348
349
350
351
352
353
354
355
356
    class login(ConfigSection):
        methods = StringParameter(default="local")
        session_ttl = SecondsParameter(default="7d")
        language = StringParameter(default="en")
        restrict_to_group = StringParameter(default="")
        single_session_group = StringParameter(default="")
        mutual_exclusive_group = StringParameter(default="")
        idle_timeout = SecondsParameter(default="1w")
        pam_service = StringParameter(default="noc")
        radius_secret = SecretParameter(default="noc")
        radius_server = StringParameter()
357
        register_last_login = BooleanParameter(default=True)
Dmitry Volodin's avatar
Dmitry Volodin committed
358
359
        jwt_cookie_name = StringParameter(default="noc_jwt")
        jwt_algorithm = StringParameter(default="HS256", choices=["HS256", "HS384", "HS512"])
360
361
362
363
364
365
366
367
368
369
370

    class mailsender(ConfigSection):
        smtp_server = StringParameter()
        smtp_port = IntParameter(default=25)
        use_tls = BooleanParameter(default=False)
        helo_hostname = StringParameter(default="noc")
        from_address = StringParameter(default="noc@example.com")
        smtp_user = StringParameter()
        smtp_password = SecretParameter()

    class memcached(ConfigSection):
371
        addresses = ServiceParameter(service="memcached", wait=True, full_result=True)
372
373
374
        pool_size = IntParameter(default=8)
        default_ttl = SecondsParameter(default="1d")

Dmitry Volodin's avatar
Dmitry Volodin committed
375
376
377
378
    class message(ConfigSection):
        enable_alarm = BooleanParameter(default=False)
        enable_managedobject = BooleanParameter(default=False)

379
380
381
382
383
384
385
386
    class mongo(ConfigSection):
        addresses = ServiceParameter(service="mongo", wait=True)
        db = StringParameter(default="noc")
        user = StringParameter()
        password = SecretParameter()
        rs = StringParameter()
        retries = IntParameter(default=20)
        timeout = SecondsParameter(default="3s")
387
388
        retry_writes = BooleanParameter(default=False)
        app_name = StringParameter()
389
        max_idle_time = SecondsParameter(default="60s")
390
391
392

    class mrt(ConfigSection):
        max_concurrency = IntParameter(default=50)
393
        enable_command_logging = BooleanParameter(default=False)
394
395
396

    node = socket.gethostname()

Dmitry Volodin's avatar
Dmitry Volodin committed
397
398
399
400
    class nbi(ConfigSection):
        max_threads = IntParameter(default=10)
        objectmetrics_max_interval = SecondsParameter(default="3h")

401
    class nsqd(ConfigSection):
402
403
404
405
        addresses = ServiceParameter(service="nsqd", wait=True, near=True, full_result=False)
        http_addresses = ServiceParameter(
            service="nsqdhttp", wait=True, near=True, full_result=False
        )
Dmitry Volodin's avatar
Dmitry Volodin committed
406
407
408
409
410
        pub_retries = IntParameter(default=5)
        pub_retry_delay = FloatParameter(default=1)
        mpub_messages = IntParameter(default=10000)
        mpub_size = IntParameter(default=1048576)
        topic_mpub_rate = IntParameter(default=10)
411
412
413
        ch_chunk_size = IntParameter(default=4000)
        connect_timeout = SecondsParameter(default="3s")
        request_timeout = SecondsParameter(default="30s")
414
        reconnect_interval = IntParameter(default=15)
415
        compression = StringParameter(choices=["", "deflate", "snappy"], default="")
416
        compression_level = IntParameter(default=6)
417
        max_in_flight = IntParameter(default=1)
418
419

    class nsqlookupd(ConfigSection):
420
        addresses = ServiceParameter(service="nsqlookupd", wait=True, near=True, full_result=False)
421
        http_addresses = ServiceParameter(service="nsqlookupdhttp", wait=True, full_result=False)
422

423
424
425
426
    class path(ConfigSection):
        smilint = StringParameter()
        smidump = StringParameter()
        dig = StringParameter()
Aleksey Shirokih's avatar
Aleksey Shirokih committed
427
428
        vcs_path = StringParameter(default="/usr/local/bin/hg")
        repo = StringParameter(default="/var/repo")
Aleksey Shirokih's avatar
Aleksey Shirokih committed
429
        backup_dir = StringParameter(default="/var/backup")
430
431
432
433
        etl_import = StringParameter(default="/var/lib/noc/import")
        ssh_key_prefix = StringParameter(default="etc/noc_ssh")
        cp_new = StringParameter(default="/var/lib/noc/cp/crashinfo/new")
        bi_data_prefix = StringParameter(default="/var/lib/noc/bi")
Aleksey Shirokih's avatar
Aleksey Shirokih committed
434
435
436
        babel_cfg = StringParameter(default="etc/babel.cfg")
        babel = StringParameter(default="./bin/pybabel")
        pojson = StringParameter(default="./bin/pojson")
Aleksey Shirokih's avatar
Aleksey Shirokih committed
437
        collection_fm_mibs = StringParameter(default="collections/fm.mibs/")
438
439
440
        supervisor_cfg = StringParameter(default="etc/noc_services.conf")
        legacy_config = StringParameter(default="etc/noc.yml")
        cythonize = StringParameter(default="./bin/cythonize")
441
        npkg_root = StringParameter(default="/var/lib/noc/var/pkg")
442
443
        card_template_path = StringParameter(default="services/card/templates/card.html.j2")
        pm_templates = StringParameter(default="templates/ddash/")
444
        custom_path = StringParameter()
Dmitry Volodin's avatar
Dmitry Volodin committed
445
        mib_path = StringParameter(default="/var/mib")
446

447
    class pg(ConfigSection):
448
        addresses = ServiceParameter(service="postgres", wait=True, near=True, full_result=False)
449
450
451
        db = StringParameter(default="noc")
        user = StringParameter()
        password = SecretParameter()
452
        connect_timeout = IntParameter(default=5)
453
454

    class ping(ConfigSection):
455
456
        throttle_threshold = FloatParameter()
        restore_threshold = FloatParameter()
457
        tos = IntParameter(min=0, max=255, default=0)
Aleksey Shirokih's avatar
Aleksey Shirokih committed
458
459
460
461
        # Recommended send buffer size, 4M by default
        send_buffer = IntParameter(default=4 * 1048576)
        # Recommended receive buffer size, 4M by default
        receive_buffer = IntParameter(default=4 * 1048576)
Dmitry Volodin's avatar
Dmitry Volodin committed
462
463
        # DataStream request limit
        ds_limit = IntParameter(default=1000)
464
465

    class pmwriter(ConfigSection):
466
467
        batch_size = IntParameter(default=2500)
        metrics_buffer = IntParameter(default=50000)
468
        read_from = StringParameter(default="pmwriter")
Aleksey Shirokih's avatar
Aleksey Shirokih committed
469
470
        write_to = StringParameter(default="influxdb")
        write_to_port = IntParameter(default=8086)
471
        max_delay = FloatParameter(default="1.0")
472

473
474
475
476
    class proxy(ConfigSection):
        http_proxy = StringParameter(default=os.environ.get("http_proxy"))
        https_proxy = StringParameter(default=os.environ.get("https_proxy"))
        ftp_proxy = StringParameter(default=os.environ.get("ftp_proxy"))
477

478
    pool = StringParameter(default=os.environ.get("NOC_POOL", ""))
479

Dmitry Volodin's avatar
Dmitry Volodin committed
480
481
482
483
484
    class redis(ConfigSection):
        addresses = ServiceParameter(service="redis", wait=True, full_result=True)
        db = IntParameter(default=0)
        default_ttl = SecondsParameter(default="1d")

485
    class rpc(ConfigSection):
486
        retry_timeout = StringParameter(default="0.1,0.5,1,3,10,30")
487
488
489
490
491
492
493
494
495
496
497
498
        sync_connect_timeout = SecondsParameter(default="20s")
        sync_request_timeout = SecondsParameter(default="1h")
        sync_retry_timeout = FloatParameter(default=1.0)
        sync_retry_delta = FloatParameter(default=2.0)
        sync_retries = IntParameter(default=5)
        async_connect_timeout = SecondsParameter(default="20s")
        async_request_timeout = SecondsParameter(default="1h")

    class sae(ConfigSection):
        db_threads = IntParameter(default=20)
        activator_resolution_retries = IntParameter(default=5)
        activator_resolution_timeout = SecondsParameter(default="2s")
499
500
501

    class scheduler(ConfigSection):
        max_threads = IntParameter(default=20)
Aleksey Shirokih's avatar
Aleksey Shirokih committed
502
503
504
        submit_threshold_factor = IntParameter(default=10)
        max_chunk_factor = IntParameter(default=1)
        updates_per_check = IntParameter(default=4)
Aleksey Shirokih's avatar
Aleksey Shirokih committed
505
        cache_default_ttl = SecondsParameter(default="1d")
Aleksey Shirokih's avatar
Aleksey Shirokih committed
506
507
        autointervaljob_interval = SecondsParameter(default="1d")
        autointervaljob_initial_submit_interval = SecondsParameter(default="1d")
508

509
    class script(ConfigSection):
510
        timeout = SecondsParameter(default="2M", help="default sa script script timeout")
511
        session_idle_timeout = SecondsParameter(default="1M", help="default session timeout")
512
        caller_timeout = SecondsParameter(default="1M")
513
        calling_service = StringParameter(default="script")
514

515
    secret_key = StringParameter(default="12345")
516

Dmitry Volodin's avatar
Dmitry Volodin committed
517
518
519
    class selfmon(ConfigSection):
        enable_managedobject = BooleanParameter(default=True)
        managedobject_ttl = IntParameter(default=30)
520
521
522
523
524
525
        enable_task = BooleanParameter(default=False)
        task_ttl = IntParameter(default=30)
        enable_inventory = BooleanParameter(default=False)
        inventory_ttl = IntParameter(default=30)
        enable_fm = BooleanParameter(default=False)
        fm_ttl = IntParameter(default=30)
Dmitry Volodin's avatar
Dmitry Volodin committed
526

527
528
    class sentry(ConfigSection):
        url = StringParameter(default="")
MaksimSmile13's avatar
MaksimSmile13 committed
529
530
531
532
        shutdown_timeout = IntParameter(min=1, max=10, default=2)
        default_integrations = BooleanParameter(default=False)
        debug = BooleanParameter(default=False)
        max_breadcrumbs = IntParameter(min=1, max=100, default=10)
533

534
535
    class syslogcollector(ConfigSection):
        listen = StringParameter(default="0.0.0.0:514")
536
537
        enable_reuseport = BooleanParameter(default=True)
        enable_freebind = BooleanParameter(default=False)
538
539
        # DataStream request limit
        ds_limit = IntParameter(default=1000)
Aleksey Shirokih's avatar
Aleksey Shirokih committed
540

541
542
543
544
545
    class icqsender(ConfigSection):
        token = SecretParameter()
        retry_timeout = IntParameter(default=2)
        use_proxy = BooleanParameter(default=False)

Aleksey Shirokih's avatar
Aleksey Shirokih committed
546
547
    class tgsender(ConfigSection):
        token = SecretParameter()
548
549
        retry_timeout = IntParameter(default=2)
        use_proxy = BooleanParameter(default=False)
Aleksey Shirokih's avatar
Aleksey Shirokih committed
550

551
552
553
    class threadpool(ConfigSection):
        idle_timeout = SecondsParameter(default="30s")
        shutdown_timeout = SecondsParameter(default="1M")
Aleksey Shirokih's avatar
Aleksey Shirokih committed
554

555
    timezone = StringParameter(default="Europe/Moscow")
Aleksey Shirokih's avatar
Aleksey Shirokih committed
556

557
558
    class traceback(ConfigSection):
        reverse = BooleanParameter(default=True)
Aleksey Shirokih's avatar
Aleksey Shirokih committed
559

560
561
    class trapcollector(ConfigSection):
        listen = StringParameter(default="0.0.0.0:162")
562
563
        enable_reuseport = BooleanParameter(default=True)
        enable_freebind = BooleanParameter(default=False)
564
565
        # DataStream request limit
        ds_limit = IntParameter(default=1000)
566
567

    class web(ConfigSection):
Dmitry Volodin's avatar
Dmitry Volodin committed
568
        theme = StringParameter(default="gray")
569
        api_row_limit = IntParameter(default=0)
Andrey Vertiprahov's avatar
Andrey Vertiprahov committed
570
        api_unlimited_row_limit = IntParameter(default=1000)
571
        api_arch_alarm_limit = IntParameter(default=4 * 86400)
572
        max_upload_size = IntParameter(default=16777216)
573
574
575
        language = StringParameter(default="en")
        install_collection = BooleanParameter(default=False)
        max_threads = IntParameter(default=10)
576
        macdb_window = IntParameter(default=4 * 86400)
uncol's avatar
uncol committed
577
        enable_remote_system_last_extract_info = BooleanParameter(default=False)
Aleksey Shirokih's avatar
Aleksey Shirokih committed
578

579
580
581
582
583
    class datasource(ConfigSection):
        chunk_size = IntParameter(default=1000)
        max_threads = IntParameter(default=10)
        default_ttl = SecondsParameter(default="1h")

Dmitry Volodin's avatar
Dmitry Volodin committed
584
    class datastream(ConfigSection):
585
        enable_administrativedomain = BooleanParameter(default=False)
586
587
        enable_administrativedomain_wait = BooleanParameter(
            default=True,
588
589
            help="Activate Wait Mode for Adm. Domain datastream (Mongo greater 3.6 needed)",
        )
590
        enable_alarm = BooleanParameter(default=False)
Dmitry Volodin's avatar
Dmitry Volodin committed
591
        enable_alarm_wait = BooleanParameter(
592
593
            default=True, help="Activate Wait Mode for Alarm datastream (Mongo greater 3.6 needed)"
        )
Dmitry Volodin's avatar
Dmitry Volodin committed
594
        enable_cfgping = BooleanParameter(default=True)
595
596
        enable_cfgping_wait = BooleanParameter(
            default=True,
597
598
            help="Activate Wait Mode for CfgPing datastream (Mongo greater 3.6 needed)",
        )
Dmitry Volodin's avatar
PEP8    
Dmitry Volodin committed
599
        enable_cfgsyslog = BooleanParameter(default=True)
600
601
        enable_cfgsyslog_wait = BooleanParameter(
            default=True,
602
603
            help="Activate Wait Mode for CfgSyslog datastream (Mongo greater 3.6 needed)",
        )
Dmitry Volodin's avatar
Dmitry Volodin committed
604
        enable_cfgtrap = BooleanParameter(default=True)
605
606
        enable_cfgtrap_wait = BooleanParameter(
            default=True,
607
608
            help="Activate Wait Mode for CfgTrap datastream (Mongo greater 3.6 needed)",
        )
609
610
611
        enable_dnszone = BooleanParameter(default=False)
        enable_dnszone_wait = BooleanParameter(
            default=True,
612
613
            help="Activate Wait Mode for DNS Zone datastream (Mongo greater 3.6 needed)",
        )
614
        enable_managedobject = BooleanParameter(default=False)
615
616
        enable_managedobject_wait = BooleanParameter(
            default=True,
617
618
            help="Activate Wait Mode for ManagedObject datastream (Mongo greater 3.6 needed)",
        )
619
        enable_resourcegroup = BooleanParameter(default=False)
620
621
        enable_resourcegroup_wait = BooleanParameter(
            default=True,
622
623
            help="Activate Wait Mode for ResourceGroup datastream (Mongo greater 3.6 needed)",
        )
Dmitry Volodin's avatar
Dmitry Volodin committed
624
625
626
627
628
629
630
631
632
633
634
635
636
        enable_vrf = BooleanParameter(default=False)
        enable_vrf_wait = BooleanParameter(
            default=True, help="Activate Wait Mode for VRF datastream (Mongo greater 3.6 needed)"
        )
        enable_prefix = BooleanParameter(default=False)
        enable_prefix_wait = BooleanParameter(
            default=True, help="Activate Wait Mode for Prefix datastream (Mongo greater 3.6 needed)"
        )
        enable_address = BooleanParameter(default=False)
        enable_address_wait = BooleanParameter(
            default=True,
            help="Activate Wait Mode for Address datastream (Mongo greater 3.6 needed)",
        )
Dmitry Volodin's avatar
Dmitry Volodin committed
637

uncol's avatar
uncol committed
638
639
640
641
642
    class help(ConfigSection):
        base_url = StringParameter(default="https://docs.getnoc.com")
        branch = StringParameter(default="microservices")
        language = StringParameter(default="en")

Dmitry Volodin's avatar
Dmitry Volodin committed
643
    class tests(ConfigSection):
Dmitry Volodin's avatar
Dmitry Volodin committed
644
645
        # List of pyfilesystem URLs holding intial data
        fixtures_paths = ListParameter(item=StringParameter(), default=["tests/data"])
646
647
        # List of pyfilesystem URLs holding event classification samples
        events_paths = ListParameter(item=StringParameter())
Dmitry Volodin's avatar
Dmitry Volodin committed
648
649
        # List of pyfilesystem URLs holding beef test cases
        beef_paths = ListParameter(item=StringParameter())
Dmitry Volodin's avatar
Dmitry Volodin committed
650

Dmitry Volodin's avatar
Dmitry Volodin committed
651
652
653
654
655
656
657
    class peer(ConfigSection):
        enable_ripe = BooleanParameter(default=True)
        enable_arin = BooleanParameter(default=True)
        enable_radb = BooleanParameter(default=True)
        prefix_list_optimization = BooleanParameter(default=True)
        prefix_list_optimization_threshold = IntParameter(default=1000)
        max_prefix_length = IntParameter(default=24)
658
        rpsl_inverse_pref_style = BooleanParameter(default=False)
Andrey Vertiprahov's avatar
Fix.    
Andrey Vertiprahov committed
659

Dmitry Volodin's avatar
Dmitry Volodin committed
660
    class metrics(ConfigSection):
661
662
663
        default_hist = ListParameter(
            item=FloatParameter(), default=[0.001, 0.005, 0.01, 0.05, 0.5, 1.0, 5.0, 10.0]
        )
Dmitry Volodin's avatar
Dmitry Volodin committed
664
        enable_mongo_hist = BooleanParameter(default=False)
665
666
667
        mongo_hist = ListParameter(
            item=FloatParameter(), default=[0.001, 0.005, 0.01, 0.05, 0.5, 1.0, 5.0, 10.0]
        )
Dmitry Volodin's avatar
Dmitry Volodin committed
668
        enable_postgres_hist = BooleanParameter(default=False)
669
670
671
672
        postgres_hist = ListParameter(
            item=FloatParameter(), default=[0.001, 0.005, 0.01, 0.05, 0.5, 1.0, 5.0, 10.0]
        )
        default_quantiles = ListParameter(item=FloatParameter(), default=[0.5, 0.9, 0.95])
Dmitry Volodin's avatar
Dmitry Volodin committed
673
674
675
676
677
        default_quantiles_epsilon = 0.01
        default_quantiles_window = 60
        default_quantiles_buffer = 100
        enable_mongo_quantiles = BooleanParameter(default=False)
        enable_postgres_quantiles = BooleanParameter(default=False)
Dmitry Volodin's avatar
Dmitry Volodin committed
678

Andrey Vertiprahov's avatar
Andrey Vertiprahov committed
679
    # pylint: disable=super-init-not-called
680
681
682
683
684
685
686
687
688
689
    def __init__(self):
        self.setup_logging()

    @property
    def pg_connection_args(self):
        """
        PostgreSQL database connection arguments
        suitable to pass to psycopg2.connect
        """
        return {
690
691
            "host": self.pg.addresses[0].host,
            "port": self.pg.addresses[0].port,
692
693
            "database": self.pg.db,
            "user": self.pg.user,
694
            "password": self.pg.password,
695
696
697
698
699
700
701
702
703
704
705
706
        }

    @property
    def mongo_connection_args(self):
        """
        Mongo connection arguments. Suitable to pass to
        pymongo.connect and mongoengine.connect
        """
        if not hasattr(self, "_mongo_connection_args"):
            self._mongo_connection_args = {
                "db": self.mongo.db,
                "username": self.mongo.user,
707
                "password": self.mongo.password,
708
            }
709
710
711
712
            if self.mongo.app_name:
                self._mongo_connection_args["appname"] = self.mongo.app_name
            if self.mongo.retry_writes:
                self._mongo_connection_args["retryWrites"] = True
713
714
715
            has_credentials = self.mongo.user or self.mongo.password
            if has_credentials:
                self._mongo_connection_args["authentication_source"] = self.mongo.db
716
            hosts = self.mongo.addresses
717
718
            if self.mongo.rs:
                self._mongo_connection_args["replicaSet"] = self.mongo.rs
719
                self._mongo_connection_args["readPreference"] = "secondaryPreferred"
720
721
            elif len(hosts) > 1:
                raise ValueError("Replica set name must be set")
722
723
            if self.mongo.max_idle_time:
                self._mongo_connection_args["maxIdleTimeMS"] = self.mongo.max_idle_time * 1000
724
725
            url = ["mongodb://"]
            if has_credentials:
726
727
728
                url += [
                    "%s:%s@" % (urllib_quote(self.mongo.user), urllib_quote(self.mongo.password))
                ]
729
            url += [",".join(str(h) for h in hosts)]
730
731
            url += ["/%s" % self.mongo.db]
            self._mongo_connection_args["host"] = "".join(url)
Dmitry Volodin's avatar
Dmitry Volodin committed
732
733
            if self.metrics.enable_mongo_hist:
                from noc.core.mongo.monitor import MongoCommandSpan
734

Dmitry Volodin's avatar
Dmitry Volodin committed
735
                self._mongo_connection_args["event_listeners"] = [MongoCommandSpan()]
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
        return self._mongo_connection_args

    def setup_logging(self, loglevel=None):
        """
        Create new or setup existing logger
        """
        if not loglevel:
            loglevel = self.loglevel
        logger = logging.getLogger()
        if len(logger.handlers):
            # Logger is already initialized
            fmt = logging.Formatter(self.log_format, None)
            for h in logging.root.handlers:
                if isinstance(h, logging.StreamHandler):
                    h.stream = sys.stdout
                h.setFormatter(fmt)
            logging.root.setLevel(loglevel)
        else:
            # Initialize logger
755
            logging.basicConfig(stream=sys.stdout, format=self.log_format, level=loglevel)
756
757
        logging.captureWarnings(True)

758
759
760
761
762
763
764
765
766
767
768
    @property
    def ch_cluster_topology(self):
        if not hasattr(self, "_ch_cluster_topology"):
            shards = []
            for s in self.clickhouse.cluster_topology.split(","):
                s = s.strip()
                if ":" in s:
                    weight, replicas = s.split(":")
                else:
                    weight, replicas = 1, s
                shards += [CHClusterShard(int(replicas), int(weight))]
769
            self._ch_cluster_topology = shards
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
        return self._ch_cluster_topology

    def get_ch_topology_type(self):
        """
        Detect ClickHouse topology type
        :return: Any of
          * CH_UNCLUSTERED
          * CH_REPLICATED
          * CH_SHARDED
        """
        topo = self.ch_cluster_topology
        if len(topo) == 1:
            if topo[0].replicas == 1:
                return CH_UNCLUSTERED
            else:
                return CH_REPLICATED
        else:
            return CH_SHARDED

789
    def get_customized_paths(self, *args, **kwargs):
790
791
792
793
794
795
796
797
798
        """
        Check for customized path for given repo path.
        Repo path may be given in os.path.join-style components.
        Returns list of possible paths. One of elements is always repo path,
        while other may be custom counterpart, if exists.
        :param prefer_custom: True - customized path first, False - repo path first
        :param args: Path or path components in os.path.join-style
        :return: List of possible paths
        """
799
        prefer_custom = kwargs.get("prefer_custom", False)
800
801
802
803
804
805
806
807
808
809
810
        rpath = os.path.join(*args)
        if not self.path.custom_path:
            return [rpath]
        cpath = os.path.join(self.path.custom_path, *args)
        if os.path.exists(cpath):
            if prefer_custom:
                return [cpath, rpath]
            else:
                return [rpath, cpath]
        return [rpath]

Dmitry Volodin's avatar
Dmitry Volodin committed
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
    def get_hist_config(self, name):
        """
        Get configuration for hist `name`. Returns list of times or None, if hist is disabled
        :param name: Hist name
        :return: List of hist config or None
        """
        # Check hist is enabled
        if not getattr(self.metrics, "enable_%s_hist" % name, False):
            return None
        # Get config
        cfg = getattr(self.metrics, "%s_hist" % name)
        if cfg:
            return cfg
        # Fallback to defaults
        return self.metrics.default_hist or None

Dmitry Volodin's avatar
Dmitry Volodin committed
827
828
829
830
831
832
833
834
    def get_quantiles_config(self, name):
        """
        Check if quantile is enabled
        :return: True if quantile is enabled
        """
        # Check quantiles is enabled
        return getattr(self.metrics, "enable_%s_quantiles" % name, False)

835
836
837
838
839

CHClusterShard = namedtuple("CHClusterShard", ["replicas", "weight"])
CH_UNCLUSTERED = 0
CH_REPLICATED = 1
CH_SHARDED = 2
840
841
842

config = Config()
config.load()
843
config.setup_logging()