Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in / Register
Toggle navigation
Menu
Open sidebar
arthur-zzz
noc
Commits
60dccdbb
Commit
60dccdbb
authored
Jun 22, 2021
by
EKbfh
🐼
Browse files
Merge branch 'noc-1603' into 'master'
noc/noc#1603
Fix upload MIB on WEB. See merge request
noc/noc!5455
parents
aee8b033
95712963
Changes
15
Hide whitespace changes
Inline
Side-by-side
ansible/noc_roles/mib/defaults/main.yml
0 → 100644
View file @
60dccdbb
---
mib_path
:
/var/lib/noc/mibs/
ansible/noc_roles/mib/tasks/main.yml
View file @
60dccdbb
---
-
include_tasks
:
"
os/{{
ansible_distribution
}}_{{
ansible_distribution_major_version
}}/main.yml"
-
name
:
Create directory for imported mibs
become
:
"
True"
file
:
state
:
directory
path
:
"
{{
item
}}"
owner
:
root
group
:
wheel
mode
:
0777
with_items
:
-
"
{{
mib_path
}}"
-
name
:
Install mibs for working MIB-import
command
:
"
{{
noc_root
}}/scripts/deploy/install-packages
requirements/mib.json"
args
:
chdir
:
"
{{
noc_root
}}"
register
:
s
changed_when
:
"
'CHANGED'
in
s.stdout"
environment
:
http_proxy
:
"
{{
http_proxy
}}"
https_proxy
:
"
{{
http_proxy
}}"
tags
:
-
requirements
ansible/noc_roles/mib/tasks/os/CentOS_7/main.yml
View file @
60dccdbb
---
-
name
:
Install smilint/smidump packages
yum
:
name
:
"
{{
packages
}}"
update_cache
:
"
True"
vars
:
packages
:
-
libsmi
environment
:
https_proxy
:
"
{{
http_proxy
}}"
http_proxy
:
"
{{
http_proxy
}}"
tags
:
-
requirements
ansible/noc_roles/mib/tasks/os/Debian_10/main.yml
View file @
60dccdbb
---
-
name
:
Install smilint/smidump packages
apt
:
name
:
"
{{
packages
}}"
update_cache
:
"
True"
cache_valid_time
:
"
{{
apt_cache_valid_time
|
default
(3600)
}}"
vars
:
packages
:
-
smitools
environment
:
https_proxy
:
"
{{
http_proxy
}}"
http_proxy
:
"
{{
http_proxy
}}"
tags
:
-
requirements
ansible/noc_roles/mib/tasks/os/Debian_9/main.yml
View file @
60dccdbb
---
-
name
:
Install smilint/smidump packages
apt
:
name
:
"
{{
packages
}}"
update_cache
:
"
True"
cache_valid_time
:
"
{{
apt_cache_valid_time
|
default
(3600)
}}"
vars
:
packages
:
-
smitools
environment
:
https_proxy
:
"
{{
http_proxy
}}"
http_proxy
:
"
{{
http_proxy
}}"
tags
:
-
requirements
ansible/noc_roles/mib/tasks/os/OracleLinux_7/main.yml
View file @
60dccdbb
---
-
name
:
Install smilint/smidump packages
yum
:
name
:
"
{{
packages
}}"
update_cache
:
"
True"
vars
:
packages
:
-
libsmi
environment
:
https_proxy
:
"
{{
http_proxy
}}"
http_proxy
:
"
{{
http_proxy
}}"
tags
:
-
requirements
ansible/noc_roles/mib/tasks/os/RedHat_7/main.yml
View file @
60dccdbb
---
-
name
:
Install smilint/smidump packages
yum
:
name
:
"
{{
packages
}}"
update_cache
:
"
True"
vars
:
packages
:
-
libsmi
environment
:
https_proxy
:
"
{{
http_proxy
}}"
http_proxy
:
"
{{
http_proxy
}}"
tags
:
-
requirements
ansible/noc_roles/mib/tasks/os/Ubuntu_16/main.yml
View file @
60dccdbb
---
-
name
:
Install smilint/smidump packages
apt
:
name
:
"
{{
packages
}}"
update_cache
:
"
True"
cache_valid_time
:
"
{{
apt_cache_valid_time
|
default
(3600)
}}"
vars
:
packages
:
-
smitools
environment
:
https_proxy
:
"
{{
http_proxy
}}"
http_proxy
:
"
{{
http_proxy
}}"
tags
:
-
requirements
ansible/noc_roles/mib/tasks/os/Ubuntu_18/main.yml
View file @
60dccdbb
---
-
name
:
Install smilint/smidump packages
apt
:
name
:
"
{{
packages
}}"
update_cache
:
"
True"
cache_valid_time
:
"
{{
apt_cache_valid_time
|
default
(3600)
}}"
vars
:
packages
:
-
smitools
environment
:
https_proxy
:
"
{{
http_proxy
}}"
http_proxy
:
"
{{
http_proxy
}}"
tags
:
-
requirements
commands/mib.py
View file @
60dccdbb
...
...
@@ -268,7 +268,7 @@ class Command(BaseCommand):
with
open
(
path
,
"rb"
)
as
f
:
data
=
f
.
read
()
try
:
r
=
self
.
svc
.
compile
(
data
)
r
=
self
.
svc
.
compile
(
MIB
.
guess_encoding
(
data
)
)
if
r
.
get
(
"status"
):
return
True
if
r
.
get
(
"code"
)
==
ERR_MIB_MISSED
:
...
...
config.py
View file @
60dccdbb
...
...
@@ -512,7 +512,7 @@ class Config(BaseConfig):
card_template_path
=
StringParameter
(
default
=
"services/card/templates/card.html.j2"
)
pm_templates
=
StringParameter
(
default
=
"templates/ddash/"
)
custom_path
=
StringParameter
()
mib_path
=
StringParameter
(
default
=
"/var/mib"
)
mib_path
=
StringParameter
(
default
=
"/var/
lib/noc/
mib
s/
"
)
class
pg
(
ConfigSection
):
addresses
=
ServiceParameter
(
service
=
"postgres"
,
wait
=
True
,
near
=
True
,
full_result
=
False
)
...
...
fm/models/mib.py
View file @
60dccdbb
...
...
@@ -8,6 +8,7 @@
# Python modules
import
re
import
threading
from
typing
import
Optional
,
List
,
Dict
import
operator
# Third-party modules
...
...
@@ -32,6 +33,8 @@ id_lock = threading.Lock()
# Regular expression patterns
rx_tailing_numbers
=
re
.
compile
(
r
"^(\S+?)((?:\.\d+)*)$"
)
TRY_ENCODINGS
=
[
"utf-8"
,
"big5"
]
@
on_delete_check
(
check
=
[(
"fm.MIBData"
,
"mib"
)])
class
MIB
(
Document
):
...
...
@@ -283,7 +286,7 @@ class MIB(Document):
o
.
save
()
@
classmethod
def
resolve_vars
(
cls
,
vars
):
def
resolve_vars
(
cls
,
vars
:
Dict
[
str
,
bytes
]
):
"""
Resolve FM key -> value dict according to MIBs
...
...
@@ -352,6 +355,23 @@ class MIB(Document):
r
[
rk
]
=
rv
return
r
@
classmethod
def
guess_encoding
(
cls
,
s
:
bytes
,
encodings
:
Optional
[
List
[
str
]]
=
None
)
->
str
:
"""
Try to guess encoding
:param s:
:param encodings:
:return:
"""
encodings
=
encodings
or
TRY_ENCODINGS
exc
=
None
for
enc
in
encodings
:
try
:
return
s
.
decode
(
enc
)
except
UnicodeDecodeError
as
e
:
exc
=
e
raise
exc
# Avoid circular references
from
.mibdata
import
MIBData
services/mib/api/mib.py
View file @
60dccdbb
...
...
@@ -7,10 +7,12 @@
# Python modules
import
os
import
shutil
import
subprocess
import
re
from
importlib.machinery
import
SourceFileLoader
import
datetime
from
shutil
import
which
# Third-party modules
from
typing
import
Optional
,
List
...
...
@@ -35,7 +37,6 @@ class MIBAPI(API):
rx_illegal_subtype
=
re
.
compile
(
b
"{subtype-enumeration-illegal}.*`([^']+)'"
)
rx_object_identifier_unknown
=
re
.
compile
(
b
"{object-identifier-unknown}.*`([^']+)'"
)
rx_oid
=
re
.
compile
(
r
"^\d+(\.\d+)+"
)
TRY_ENCODINGS
=
[
"utf-8"
,
"big5"
]
SMI_ENV
=
{
"SMIPATH"
:
config
.
path
.
mib_path
}
...
...
@@ -60,22 +61,6 @@ class MIBAPI(API):
return
{
"status"
:
True
,
"data"
:
f
.
read
()}
return
{
"status"
:
False
,
"msg"
:
"Not found"
,
"code"
:
ERR_MIB_NOT_FOUND
}
def
guess_encoding
(
self
,
s
:
bytes
,
encodings
:
Optional
[
List
[
str
]]
=
None
)
->
str
:
"""
Try to guess encoding
:param s:
:param encodings:
:return:
"""
encodings
=
encodings
or
self
.
TRY_ENCODINGS
exc
=
None
for
enc
in
encodings
:
try
:
return
s
.
decode
(
enc
)
except
UnicodeDecodeError
as
e
:
exc
=
e
raise
exc
@
api
def
compile
(
self
,
data
):
"""
...
...
@@ -83,31 +68,42 @@ class MIBAPI(API):
:param data: MIB text
:return:
"""
if
not
config
.
path
.
smilint
or
not
os
.
path
.
exists
(
config
.
path
.
smilint
):
return
{
"status"
:
False
,
"msg"
:
"smilint is missed"
,
"error"
:
ERR_MIB_TOOL_MISSED
}
if
not
config
.
path
.
smilint
or
not
os
.
path
.
exists
(
config
.
path
.
smidump
):
return
{
"status"
:
False
,
"msg"
:
"smidump is missed"
,
"error"
:
ERR_MIB_TOOL_MISSED
}
if
config
.
path
.
smilint
and
os
.
path
.
exists
(
config
.
path
.
smilint
):
smilint_path
=
config
.
path
.
smilint
else
:
smilint_path
=
shutil
.
which
(
"smilint"
)
if
not
smilint_path
:
self
.
logger
.
error
(
"Can't find smilint executable"
)
return
{
"status"
:
False
,
"msg"
:
"smilint is missed"
,
"error"
:
ERR_MIB_TOOL_MISSED
}
if
config
.
path
.
smidump
and
os
.
path
.
exists
(
config
.
path
.
smidump
):
smidump_path
=
config
.
path
.
smidump
else
:
smidump_path
=
shutil
.
which
(
"smidump"
)
if
not
smidump_path
:
self
.
logger
.
error
(
"Can't find smidump executable"
)
return
{
"status"
:
False
,
"msg"
:
"smidump is missed"
,
"error"
:
ERR_MIB_TOOL_MISSED
}
# Normalize input
if
isinstance
(
data
,
bytes
):
data
=
self
.
guess_encoding
(
data
)
data
=
MIB
.
guess_encoding
(
data
)
# Put data to temporary file
with
temporary_file
(
data
)
as
tmp_path
:
# Pass MIB through smilint to detect missed modules
self
.
logger
.
debug
(
"Pass MIB through smilint to detect missed modules"
)
self
.
logger
.
info
(
"Pass MIB through smilint to detect missed modules"
)
f
=
subprocess
.
Popen
(
[
config
.
path
.
smilint
,
"-m"
,
tmp_path
],
stderr
=
subprocess
.
PIPE
,
env
=
self
.
SMI_ENV
[
smilint
_path
,
"-m"
,
tmp_path
],
stderr
=
subprocess
.
PIPE
,
env
=
self
.
SMI_ENV
).
stderr
for
l
in
f
:
match
=
self
.
rx_module_not_found
.
search
(
l
.
strip
())
for
l
ine
in
f
:
match
=
self
.
rx_module_not_found
.
search
(
l
ine
.
strip
())
if
match
:
self
.
logger
.
error
(
"Required MIB missed: %s"
,
smart_text
(
match
.
group
(
1
)))
return
{
"status"
:
False
,
"msg"
:
"Required MIB missed: %s"
%
smart_text
(
match
.
group
(
1
)),
"code"
:
ERR_MIB_MISSED
,
}
match
=
self
.
rx_macro_not_imported
.
search
(
l
.
strip
())
match
=
self
.
rx_macro_not_imported
.
search
(
l
ine
.
strip
())
if
match
:
self
.
logger
.
debug
(
self
.
logger
.
error
(
"Macro '%s' (%s) has not been imported"
,
smart_text
(
match
.
group
(
1
)),
smart_text
(
match
.
group
(
2
)),
...
...
@@ -118,14 +114,14 @@ class MIBAPI(API):
# smart_text(match.group(1)), smart_text(match.group(2))),
# "code": ERR_MIB_MISSED,
# }
match
=
self
.
rx_illegal_subtype
.
search
(
l
.
strip
())
match
=
self
.
rx_illegal_subtype
.
search
(
l
ine
.
strip
())
if
match
:
return
{
"status"
:
False
,
"msg"
:
"Illegal subtype: %s"
%
smart_text
(
match
.
group
(
1
)),
"code"
:
ERR_MIB_MISSED
,
}
match
=
self
.
rx_object_identifier_unknown
.
search
(
l
.
strip
())
match
=
self
.
rx_object_identifier_unknown
.
search
(
l
ine
.
strip
())
if
match
:
self
.
logger
.
warning
(
"Object Identifier unknown: %s"
%
smart_text
(
match
.
group
(
1
))
...
...
@@ -139,7 +135,7 @@ class MIBAPI(API):
# Convert MIB to python module and load
with
temporary_file
()
as
py_path
:
subprocess
.
check_call
(
[
config
.
path
.
smidump
,
"-k"
,
"-q"
,
"-f"
,
"python"
,
"-o"
,
py_path
,
tmp_path
],
[
smidump
_path
,
"-k"
,
"-q"
,
"-f"
,
"python"
,
"-o"
,
py_path
,
tmp_path
],
env
=
self
.
SMI_ENV
,
)
with
open
(
py_path
)
as
f
:
...
...
services/web/apps/fm/mib/views.py
View file @
60dccdbb
...
...
@@ -100,10 +100,10 @@ class MIBApplication(ExtDocApplication):
errors
=
{}
while
len
(
left
):
n
=
len
(
left
)
for
name
in
l
eft
:
for
name
in
l
ist
(
left
.
keys
())
:
try
:
svc
=
open_sync_rpc
(
"mib"
)
r
=
svc
.
compile
(
left
[
name
].
read
())
r
=
svc
.
compile
(
MIB
.
guess_encoding
(
left
[
name
].
read
())
)
if
r
.
get
(
"status"
):
del
left
[
name
]
if
name
in
errors
:
...
...
@@ -115,8 +115,9 @@ class MIBApplication(ExtDocApplication):
if
len
(
left
)
==
n
:
# Failed to upload anything, stopping
break
r
=
{
"success"
:
len
(
left
)
==
0
,
"errors"
:
errors
}
return
r
return
self
.
render_json
(
{
"success"
:
len
(
left
)
==
0
,
"message"
:
f
"ERROR:
{
errors
}
"
},
status
=
self
.
OK
)
@
view
(
url
=
"^(?P<id>[0-9a-f]{24})/text/$"
,
method
=
[
"GET"
],
access
=
"launch"
,
api
=
True
)
def
api_text
(
self
,
request
,
id
):
...
...
ui/web/fm/mib/MIBUpload.js
View file @
60dccdbb
...
...
@@ -65,8 +65,13 @@ Ext.define("NOC.fm.mib.MIBUpload", {
NOC
.
info
(
__
(
"
MIBs has been uploaded
"
));
me
.
close
();
},
failure
:
function
()
{
NOC
.
error
(
__
(
"
Failed to upload MIB
"
));
failure
:
function
(
arg1
,
arg2
)
{
var
message
=
__
(
"
Failed to upload MIB
"
);
var
_arg2
=
Ext
.
decode
(
arg2
.
response
.
responseText
);
if
(
_arg2
&&
_arg2
.
message
)
{
message
=
_arg2
.
message
;
}
NOC
.
error
(
message
);
}
});
}
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment