Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in / Register
Toggle navigation
Menu
Open sidebar
noc
noc
Commits
17e82ce7
Commit
17e82ce7
authored
Jul 07, 2022
by
Andrey Vertiprahov
Browse files
Add send syslog message to mx.
parent
820ace7e
Pipeline
#36737
passed with stages
in 45 minutes and 30 seconds
Changes
10
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
config.py
View file @
17e82ce7
...
...
@@ -467,6 +467,7 @@ class Config(BaseConfig):
enable_metrics
=
BooleanParameter
(
default
=
False
)
# Comma-separated list of metric scopes
enable_metric_scopes
=
ListParameter
(
item
=
StringParameter
(),
default
=
[])
enable_syslog
=
BooleanParameter
(
default
=
False
)
class
mongo
(
ConfigSection
):
addresses
=
ServiceParameter
(
service
=
"mongo"
,
wait
=
True
)
...
...
core/mx.py
View file @
17e82ce7
...
...
@@ -30,7 +30,7 @@ MX_PROFILE_ID = "Profile-Id"
MX_LABELS
=
"Labels"
MX_H_VALUE_SPLITTER
=
";"
# Available message types
MESSAGE_TYPES
=
{
"alarm"
,
"managedobject"
,
"reboot"
,
"metrics"
,
"notification"
}
MESSAGE_TYPES
=
{
"alarm"
,
"managedobject"
,
"reboot"
,
"metrics"
,
"notification"
,
"syslog"
}
MESSAGE_HEADERS
=
{
MX_SHARDING_KEY
,
MX_TO
,
...
...
docs/en/docs/user/reference/mx/syslog.md
0 → 100644
View file @
17e82ce7
# syslog MX Message
`syslog`
message is generated by
[
syslogcollector
](
../../../admin/reference/services/syslogcollector.md
)
service when managed object Syslog is received.
## Message Headers
Message-Type
: Type of message. Always
`syslog`
.
Sharding-Key
: Key for consistent sharding.
Labels
: Managed Object's effective labels.
## Message Format
Message contains JSON array, containing objects of following structure
| Name | Type | Description |
| ---------- | -------- | -------------------------------------------------------------------- |
| timestamp | DateTime | ISO 8601 timestamp (i.e.
`YYYY-MM-DDTHH:MM:SS`
) of collected metrics |
| uuid | String | Global
`UUIDv4`
|
| collector_type | String | Always
`syslog`
|
| collector | String | Collector Pool |
| address | String | SNMP Trap Source Address |
| object | Object {{ complex }} | Managed Object |
| {{ tab }} id | String | Managed Object's ID |
| {{ tab }} bi_id | Integer | Managed Object's BI ID |
| {{ tab }} name | String | Managed Object's name |
| {{ tab }} labels | Array of String | Managed Object's labels |
| {{ tab }} administrative_domain | Object {{ complex }} | Managed Object Administrative Domain's |
| {{ tab2 }} remote_system | Object {{ complex }} | Managed Object Administrative Domain's
[
Remote System
](
../../../user/reference/concepts/remote-system/index.md
)
(
if
imported) |
| {{ tab4 }} id | String | Remote System's ID |
| {{ tab4 }} name | String | Remote System's Name |
| {{ tab2 }} id | Integer | Managed Object Administrative Domain's ID |
| {{ tab2 }} name | String | Managed Object Administrative Domain's Name |
| {{ tab2 }} remote_id | String | Managed Object Administrative Domain's ID in Remote System (if any) |
| {{ tab }} remote_system | Object {{ complex }} | Managed Object's
[
Remote System
](
../../../user/reference/concepts/remote-system/index.md
)
(
if
imported) |
| {{ tab2 }} id | String | Remote System's ID |
| {{ tab2 }} name | String | Remote System's Name |
| {{ tab }} remote_id | String | Managed Object's ID in Remote System (if any) |
| syslog | Object {{ complex }} | Syslog message body content |
| {{ tab }} facility | String | Syslog facility |
| {{ tab }} severity | String | Syslog severity |
| {{ tab }} message | String | Syslog message |
## Example
```
json
{
"timestamp"
:
"2022-06-20T10:07:15"
,
"uuid"
:
"3d6a2405-447c-44e7-b885-7a2deb43a26a"
,
"collector_type"
:
"snmptrap"
,
"collector"
:
"syslog"
,
"address"
:
"127.0.0.1"
,
"managed_object"
:
{
"id"
:
"450"
,
"bi_id"
:
7602684790455147111
,
"name"
:
"device-1"
,
"administrative_domain"
:
{
"id"
:
11
,
"name"
:
"default"
,
"remote_system"
:
{
"id"
:
"596e715fc165cf1e082ea14c"
,
"name"
:
"TEST"
},
"remote_id"
:
"1"
},
"labels"
:
[],
"remote_system"
:
{
"id"
:
"596e715fc165cf1e082ea14c"
,
"name"
:
"TEST"
},
"remote_id"
:
"22"
},
"syslog"
:
{
"severity"
:
1
,
"facility"
:
2
,
"message"
:
"Message 1"
,
}
}
\ No newline at end of file
docs/en/mkdocs.yml
View file @
17e82ce7
...
...
@@ -1292,6 +1292,7 @@ nav:
-
managedobject
:
user/reference/mx/managedobject.md
-
metrics
:
user/reference/mx/metrics.md
-
reboot
:
user/reference/mx/reboot.md
-
syslog
:
user/reference/mx/syslog.md
-
Technologies
:
-
Group
:
user/reference/technology/group.md
-
"
Network
|
CGNAT
Termination"
:
user/reference/technology/Network/cgnat-termination.md
...
...
docs/ru/docs/user/reference/mx/syslog.md
0 → 100644
View file @
17e82ce7
# syslog MX Message
`syslog`
message is generated by
[
syslogcollector
](
../../../admin/reference/services/syslogcollector.md
)
service when managed object Syslog is received.
## Message Headers
Message-Type
: Type of message. Always
`syslog`
.
Sharding-Key
: Key for consistent sharding.
Labels
: Managed Object's effective labels.
## Message Format
Message contains JSON array, containing objects of following structure
| Name | Type | Description |
| ---------- | -------- | -------------------------------------------------------------------- |
| timestamp | DateTime | ISO 8601 timestamp (i.e.
`YYYY-MM-DDTHH:MM:SS`
) of collected metrics |
| uuid | String | Global
`UUIDv4`
|
| collector_type | String | Always
`syslog`
|
| collector | String | Collector Pool |
| address | String | SNMP Trap Source Address |
| object | Object {{ complex }} | Managed Object |
| {{ tab }} id | String | Managed Object's ID |
| {{ tab }} bi_id | Integer | Managed Object's BI ID |
| {{ tab }} name | String | Managed Object's name |
| {{ tab }} labels | Array of String | Managed Object's labels |
| {{ tab }} administrative_domain | Object {{ complex }} | Managed Object Administrative Domain's |
| {{ tab2 }} remote_system | Object {{ complex }} | Managed Object Administrative Domain's
[
Remote System
](
../../../user/reference/concepts/remote-system/index.md
)
(
if
imported) |
| {{ tab4 }} id | String | Remote System's ID |
| {{ tab4 }} name | String | Remote System's Name |
| {{ tab2 }} id | Integer | Managed Object Administrative Domain's ID |
| {{ tab2 }} name | String | Managed Object Administrative Domain's Name |
| {{ tab2 }} remote_id | String | Managed Object Administrative Domain's ID in Remote System (if any) |
| {{ tab }} remote_system | Object {{ complex }} | Managed Object's
[
Remote System
](
../../../user/reference/concepts/remote-system/index.md
)
(
if
imported) |
| {{ tab2 }} id | String | Remote System's ID |
| {{ tab2 }} name | String | Remote System's Name |
| {{ tab }} remote_id | String | Managed Object's ID in Remote System (if any) |
| syslog | Object {{ complex }} | Syslog message body content |
| {{ tab }} facility | String | Syslog facility |
| {{ tab }} severity | String | Syslog severity |
| {{ tab }} message | String | Syslog message |
## Example
```
json
{
"timestamp"
:
"2022-06-20T10:07:15"
,
"uuid"
:
"3d6a2405-447c-44e7-b885-7a2deb43a26a"
,
"collector_type"
:
"snmptrap"
,
"collector"
:
"syslog"
,
"address"
:
"127.0.0.1"
,
"managed_object"
:
{
"id"
:
"450"
,
"bi_id"
:
7602684790455147111
,
"name"
:
"device-1"
,
"administrative_domain"
:
{
"id"
:
11
,
"name"
:
"default"
,
"remote_system"
:
{
"id"
:
"596e715fc165cf1e082ea14c"
,
"name"
:
"TEST"
},
"remote_id"
:
"1"
},
"labels"
:
[],
"remote_system"
:
{
"id"
:
"596e715fc165cf1e082ea14c"
,
"name"
:
"TEST"
},
"remote_id"
:
"22"
},
"syslog"
:
{
"severity"
:
1
,
"facility"
:
2
,
"message"
:
"Message 1"
,
}
}
\ No newline at end of file
docs/ru/mkdocs.yml
View file @
17e82ce7
...
...
@@ -1303,6 +1303,7 @@ nav:
-
managedobject
:
user/reference/mx/managedobject.md
-
metrics
:
user/reference/mx/metrics.md
-
reboot
:
user/reference/mx/reboot.md
-
syslog
:
user/reference/mx/syslog.md
-
Technologies
:
-
Group
:
user/reference/technology/group.md
-
"
Network
|
CGNAT
Termination"
:
user/reference/technology/Network/cgnat-termination.md
...
...
services/datastream/streams/cfgsyslog.py
View file @
17e82ce7
...
...
@@ -8,6 +8,8 @@
# NOC modules
from
noc.core.datastream.base
import
DataStream
from
noc.main.models.pool
import
Pool
from
noc.main.models.label
import
Label
from
noc.main.models.remotesystem
import
RemoteSystem
from
noc.sa.models.managedobject
import
ManagedObject
...
...
@@ -19,11 +21,20 @@ class CfgSyslogDataStream(DataStream):
def
get_object
(
cls
,
id
):
mo
=
ManagedObject
.
objects
.
filter
(
id
=
id
).
values_list
(
"id"
,
"name"
,
"bi_id"
,
"is_managed"
,
"pool"
,
"fm_pool"
,
"administrative_domain"
,
"administrative_domain__name"
,
"administrative_domain__remote_system"
,
"administrative_domain__remote_id"
,
"remote_system"
,
"remote_id"
,
"address"
,
"labels"
,
"effective_labels"
,
"syslog_source_ip"
,
"syslog_source_type"
,
"event_processing_policy"
,
...
...
@@ -35,11 +46,20 @@ class CfgSyslogDataStream(DataStream):
raise
KeyError
()
(
mo_id
,
name
,
bi_id
,
is_managed
,
pool
,
fm_pool
,
adm_domain
,
adm_domain_name
,
adm_domain_remote_system
,
adm_domain_remote_id
,
remote_system
,
remote_id
,
address
,
labels
,
effective_labels
,
syslog_source_ip
,
syslog_source_type
,
event_processing_policy
,
...
...
@@ -69,13 +89,38 @@ class CfgSyslogDataStream(DataStream):
pool
=
str
(
Pool
.
get_by_id
(
pool
).
name
)
r
=
{
"id"
:
str
(
mo_id
),
"bi_id"
:
str
(
bi_id
)
,
"bi_id"
:
bi_id
,
"pool"
:
pool
,
"fm_pool"
:
str
(
Pool
.
get_by_id
(
fm_pool
).
name
)
if
fm_pool
else
pool
,
"addresses"
:
[],
"process_events"
:
effective_epp
,
"archive_events"
:
effective_sap
,
"managed_object"
:
{
"id"
:
str
(
mo_id
),
"bi_id"
:
str
(
bi_id
),
"name"
:
name
,
"administrative_domain"
:
{
"id"
:
adm_domain
,
"name"
:
adm_domain_name
},
"labels"
:
[
cls
.
qs
(
ll
)
for
ll
in
Label
.
objects
.
filter
(
name__in
=
labels
,
expose_datastream
=
True
).
values_list
(
"name"
)
],
},
"effective_labels"
:
effective_labels
,
"name"
:
name
,
}
if
remote_system
:
rs
=
RemoteSystem
.
get_by_id
(
remote_system
)
r
[
"managed_object"
][
"remote_system"
]
=
{
"id"
:
str
(
rs
.
id
),
"name"
:
rs
.
name
}
r
[
"managed_object"
][
"remote_id"
]
=
remote_id
if
adm_domain_remote_system
:
rs
=
RemoteSystem
.
get_by_id
(
adm_domain_remote_system
)
r
[
"managed_object"
][
"administrative_domain"
][
"remote_system"
]
=
{
"id"
:
str
(
rs
.
id
),
"name"
:
rs
.
name
,
}
r
[
"managed_object"
][
"administrative_domain"
][
"remote_id"
]
=
adm_domain_remote_id
if
syslog_source_type
==
"m"
and
address
:
# Managed Object's address
r
[
"addresses"
]
+=
[
str
(
address
)]
...
...
services/syslogcollector/service.py
View file @
17e82ce7
...
...
@@ -8,8 +8,10 @@
# Python modules
import
datetime
from
collections
import
defaultdict
import
asyncio
import
uuid
from
collections
import
defaultdict
from
dataclasses
import
asdict
from
typing
import
Optional
,
Dict
# Third-party modules
...
...
@@ -20,10 +22,19 @@ from noc.config import config
from
noc.core.error
import
NOCError
from
noc.core.service.fastapi
import
FastAPIService
from
noc.core.perf
import
metrics
from
noc.core.mx
import
(
MX_STREAM
,
get_mx_partitions
,
MX_MESSAGE_TYPE
,
MX_SHARDING_KEY
,
MX_LABELS
,
MX_H_VALUE_SPLITTER
,
)
from
noc.core.ioloop.timers
import
PeriodicCallback
from
noc.services.syslogcollector.syslogserver
import
SyslogServer
from
noc.services.syslogcollector.datastream
import
SysologDataStreamClient
from
noc.services.syslogcollector.sourceconfig
import
SourceConfig
from
noc.core.
ioloop.timers
import
PeriodicCallback
from
noc.services.syslogcollector.sourceconfig
import
SourceConfig
,
ManagedObjectData
from
noc.core.
comp
import
DEFAULT_ENCODING
class
SyslogCollectorService
(
FastAPIService
):
...
...
@@ -80,7 +91,13 @@ class SyslogCollectorService(FastAPIService):
return
None
def
register_message
(
self
,
cfg
:
SourceConfig
,
timestamp
:
int
,
message
:
str
,
facility
:
int
,
severity
:
int
self
,
cfg
:
SourceConfig
,
timestamp
:
int
,
message
:
str
,
facility
:
int
,
severity
:
int
,
source_address
:
str
=
None
,
)
->
None
:
"""
Spool message to be sent
...
...
@@ -99,18 +116,16 @@ class SyslogCollectorService(FastAPIService):
stream
=
cfg
.
stream
,
partition
=
cfg
.
partition
,
)
now
=
datetime
.
datetime
.
now
().
replace
(
microsecond
=
0
)
if
cfg
.
archive_events
and
cfg
.
bi_id
:
# Archive message
metrics
[
"events_archived"
]
+=
1
now
=
datetime
.
datetime
.
now
()
ts
=
now
.
strftime
(
"%Y-%m-%d %H:%M:%S"
)
date
=
ts
.
split
(
" "
)[
0
]
self
.
register_metrics
(
"syslog"
,
[
{
"date"
:
date
,
"ts"
:
ts
,
"date"
:
now
.
date
()
,
"ts"
:
now
.
isoformat
(
sep
=
" "
)
,
"managed_object"
:
cfg
.
bi_id
,
"facility"
:
facility
,
"severity"
:
severity
,
...
...
@@ -118,6 +133,36 @@ class SyslogCollectorService(FastAPIService):
}
],
)
if
config
.
message
.
enable_snmptrap
:
metrics
[
"events_message"
]
+=
1
n_partitions
=
get_mx_partitions
()
now
=
datetime
.
datetime
.
now
()
self
.
publish
(
value
=
orjson
.
dumps
(
{
"timestamp"
:
now
.
replace
(
microsecond
=
0
),
"uuid"
:
uuid
.
uuid4
(),
"collector_type"
:
"syslog"
,
"collector"
:
config
.
pool
,
"address"
:
source_address
,
"managed_object"
:
asdict
(
cfg
.
managed_object
),
"syslog"
:
{
"facility"
:
facility
,
"severity"
:
severity
,
"message"
:
message
,
},
}
),
stream
=
MX_STREAM
,
partition
=
int
(
cfg
.
id
)
%
n_partitions
,
headers
=
{
MX_MESSAGE_TYPE
:
b
"syslog"
,
MX_LABELS
:
MX_H_VALUE_SPLITTER
.
join
(
cfg
.
effective_labels
).
encode
(
DEFAULT_ENCODING
),
MX_SHARDING_KEY
:
str
(
cfg
.
id
).
encode
(
DEFAULT_ENCODING
),
},
)
async
def
get_object_mappings
(
self
):
"""
...
...
@@ -131,7 +176,7 @@ class SyslogCollectorService(FastAPIService):
try
:
await
client
.
query
(
limit
=
config
.
syslogcollector
.
ds_limit
,
filters
=
[
"pool(
%s)"
%
config
.
pool
],
filters
=
[
f
"pool(
{
config
.
pool
}
)"
],
block
=
True
,
filter_policy
=
"delete"
,
)
...
...
@@ -170,9 +215,12 @@ class SyslogCollectorService(FastAPIService):
bi_id
=
data
.
get
(
"bi_id"
),
# For backward compatibility
process_events
=
data
.
get
(
"process_events"
,
True
),
# For backward compatibility
archive_events
=
data
.
get
(
"archive_events"
,
False
),
stream
=
"events.
%s"
%
fm_pool
,
stream
=
f
"events.
{
fm_pool
}
"
,
partition
=
int
(
data
[
"id"
])
%
num_partitions
,
effective_labels
=
data
.
get
(
"effective_labels"
,
[]),
)
if
config
.
message
.
enable_syslog
and
"managed_object"
in
data
:
cfg
.
managed_object
=
ManagedObjectData
(
**
data
[
"managed_object"
])
new_addresses
=
set
(
cfg
.
addresses
)
# Add new addresses, update remaining
for
addr
in
new_addresses
:
...
...
services/syslogcollector/sourceconfig.py
View file @
17e82ce7
...
...
@@ -7,7 +7,31 @@
# Python modules
from
dataclasses
import
dataclass
from
typing
import
Tuple
from
typing
import
Tuple
,
Optional
,
List
@
dataclass
class
RemoteSystemData
(
object
):
id
:
str
name
:
str
@
dataclass
class
AdministrativeDomainData
(
object
):
id
:
int
name
:
str
remote_system
:
Optional
[
RemoteSystemData
]
=
None
@
dataclass
class
ManagedObjectData
(
object
):
id
:
str
bi_id
:
int
name
:
str
administrative_domain
:
AdministrativeDomainData
labels
:
List
[
str
]
=
None
remote_system
:
Optional
[
RemoteSystemData
]
=
None
remote_id
:
Optional
[
str
]
=
None
@
dataclass
...
...
@@ -19,3 +43,6 @@ class SourceConfig(object):
archive_events
:
bool
stream
:
str
partition
:
int
name
:
Optional
[
str
]
=
None
effective_labels
:
List
[
str
]
=
None
managed_object
:
Optional
[
ManagedObjectData
]
=
None
services/syslogcollector/syslogserver.py
View file @
17e82ce7
...
...
@@ -51,4 +51,6 @@ class SyslogServer(UDPServer):
# Get timestamp
ts
=
int
(
time
.
time
())
#
self
.
service
.
register_message
(
cfg
,
ts
,
data
,
facility
=
priority
>>
3
,
severity
=
priority
&
7
)
self
.
service
.
register_message
(
cfg
,
ts
,
data
,
facility
=
priority
>>
3
,
severity
=
priority
&
7
,
source_address
=
address
[
0
]
)
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment