Commit 95712963 authored by Andrey Vertiprahov's avatar Andrey Vertiprahov Committed by EKbfh
Browse files

noc/noc#1603 Fix upload MIB on WEB.

parent 56eae3b7
---
mib_path: /var/lib/noc/mibs/
---
- include_tasks: "os/{{ ansible_distribution }}_{{ ansible_distribution_major_version }}/main.yml"
- name: Create directory for imported mibs
become: "True"
file:
state: directory
path: "{{ item }}"
owner: root
group: wheel
mode: 0777
with_items:
- "{{ mib_path }}"
- name: Install mibs for working MIB-import
command: "{{ noc_root }}/scripts/deploy/install-packages requirements/mib.json"
args:
chdir: "{{ noc_root }}"
register: s
changed_when: "'CHANGED' in s.stdout"
environment:
http_proxy: "{{ http_proxy }}"
https_proxy: "{{ http_proxy }}"
tags:
- requirements
---
- name: Install smilint/smidump packages
yum:
name: "{{ packages }}"
update_cache: "True"
vars:
packages:
- libsmi
environment:
https_proxy: "{{ http_proxy }}"
http_proxy: "{{ http_proxy }}"
tags:
- requirements
---
- name: Install smilint/smidump packages
apt:
name: "{{ packages }}"
update_cache: "True"
cache_valid_time: "{{ apt_cache_valid_time | default (3600) }}"
vars:
packages:
- smitools
environment:
https_proxy: "{{ http_proxy }}"
http_proxy: "{{ http_proxy }}"
tags:
- requirements
---
- name: Install smilint/smidump packages
apt:
name: "{{ packages }}"
update_cache: "True"
cache_valid_time: "{{ apt_cache_valid_time | default (3600) }}"
vars:
packages:
- smitools
environment:
https_proxy: "{{ http_proxy }}"
http_proxy: "{{ http_proxy }}"
tags:
- requirements
---
- name: Install smilint/smidump packages
yum:
name: "{{ packages }}"
update_cache: "True"
vars:
packages:
- libsmi
environment:
https_proxy: "{{ http_proxy }}"
http_proxy: "{{ http_proxy }}"
tags:
- requirements
---
- name: Install smilint/smidump packages
yum:
name: "{{ packages }}"
update_cache: "True"
vars:
packages:
- libsmi
environment:
https_proxy: "{{ http_proxy }}"
http_proxy: "{{ http_proxy }}"
tags:
- requirements
---
- name: Install smilint/smidump packages
apt:
name: "{{ packages }}"
update_cache: "True"
cache_valid_time: "{{ apt_cache_valid_time | default (3600) }}"
vars:
packages:
- smitools
environment:
https_proxy: "{{ http_proxy }}"
http_proxy: "{{ http_proxy }}"
tags:
- requirements
---
- name: Install smilint/smidump packages
apt:
name: "{{ packages }}"
update_cache: "True"
cache_valid_time: "{{ apt_cache_valid_time | default (3600) }}"
vars:
packages:
- smitools
environment:
https_proxy: "{{ http_proxy }}"
http_proxy: "{{ http_proxy }}"
tags:
- requirements
......@@ -268,7 +268,7 @@ class Command(BaseCommand):
with open(path, "rb") as f:
data = f.read()
try:
r = self.svc.compile(data)
r = self.svc.compile(MIB.guess_encoding(data))
if r.get("status"):
return True
if r.get("code") == ERR_MIB_MISSED:
......
......@@ -512,7 +512,7 @@ class Config(BaseConfig):
card_template_path = StringParameter(default="services/card/templates/card.html.j2")
pm_templates = StringParameter(default="templates/ddash/")
custom_path = StringParameter()
mib_path = StringParameter(default="/var/mib")
mib_path = StringParameter(default="/var/lib/noc/mibs/")
class pg(ConfigSection):
addresses = ServiceParameter(service="postgres", wait=True, near=True, full_result=False)
......
......@@ -8,6 +8,7 @@
# Python modules
import re
import threading
from typing import Optional, List, Dict
import operator
# Third-party modules
......@@ -32,6 +33,8 @@ id_lock = threading.Lock()
# Regular expression patterns
rx_tailing_numbers = re.compile(r"^(\S+?)((?:\.\d+)*)$")
TRY_ENCODINGS = ["utf-8", "big5"]
@on_delete_check(check=[("fm.MIBData", "mib")])
class MIB(Document):
......@@ -283,7 +286,7 @@ class MIB(Document):
o.save()
@classmethod
def resolve_vars(cls, vars):
def resolve_vars(cls, vars: Dict[str, bytes]):
"""
Resolve FM key -> value dict according to MIBs
......@@ -352,6 +355,23 @@ class MIB(Document):
r[rk] = rv
return r
@classmethod
def guess_encoding(cls, s: bytes, encodings: Optional[List[str]] = None) -> str:
"""
Try to guess encoding
:param s:
:param encodings:
:return:
"""
encodings = encodings or TRY_ENCODINGS
exc = None
for enc in encodings:
try:
return s.decode(enc)
except UnicodeDecodeError as e:
exc = e
raise exc
# Avoid circular references
from .mibdata import MIBData
......@@ -7,10 +7,12 @@
# Python modules
import os
import shutil
import subprocess
import re
from importlib.machinery import SourceFileLoader
import datetime
from shutil import which
# Third-party modules
from typing import Optional, List
......@@ -35,7 +37,6 @@ class MIBAPI(API):
rx_illegal_subtype = re.compile(b"{subtype-enumeration-illegal}.*`([^']+)'")
rx_object_identifier_unknown = re.compile(b"{object-identifier-unknown}.*`([^']+)'")
rx_oid = re.compile(r"^\d+(\.\d+)+")
TRY_ENCODINGS = ["utf-8", "big5"]
SMI_ENV = {"SMIPATH": config.path.mib_path}
......@@ -60,22 +61,6 @@ class MIBAPI(API):
return {"status": True, "data": f.read()}
return {"status": False, "msg": "Not found", "code": ERR_MIB_NOT_FOUND}
def guess_encoding(self, s: bytes, encodings: Optional[List[str]] = None) -> str:
"""
Try to guess encoding
:param s:
:param encodings:
:return:
"""
encodings = encodings or self.TRY_ENCODINGS
exc = None
for enc in encodings:
try:
return s.decode(enc)
except UnicodeDecodeError as e:
exc = e
raise exc
@api
def compile(self, data):
"""
......@@ -83,31 +68,42 @@ class MIBAPI(API):
:param data: MIB text
:return:
"""
if not config.path.smilint or not os.path.exists(config.path.smilint):
if config.path.smilint and os.path.exists(config.path.smilint):
smilint_path = config.path.smilint
else:
smilint_path = shutil.which("smilint")
if not smilint_path:
self.logger.error("Can't find smilint executable")
return {"status": False, "msg": "smilint is missed", "error": ERR_MIB_TOOL_MISSED}
if not config.path.smilint or not os.path.exists(config.path.smidump):
if config.path.smidump and os.path.exists(config.path.smidump):
smidump_path = config.path.smidump
else:
smidump_path = shutil.which("smidump")
if not smidump_path:
self.logger.error("Can't find smidump executable")
return {"status": False, "msg": "smidump is missed", "error": ERR_MIB_TOOL_MISSED}
# Normalize input
if isinstance(data, bytes):
data = self.guess_encoding(data)
data = MIB.guess_encoding(data)
# Put data to temporary file
with temporary_file(data) as tmp_path:
# Pass MIB through smilint to detect missed modules
self.logger.debug("Pass MIB through smilint to detect missed modules")
self.logger.info("Pass MIB through smilint to detect missed modules")
f = subprocess.Popen(
[config.path.smilint, "-m", tmp_path], stderr=subprocess.PIPE, env=self.SMI_ENV
[smilint_path, "-m", tmp_path], stderr=subprocess.PIPE, env=self.SMI_ENV
).stderr
for l in f:
match = self.rx_module_not_found.search(l.strip())
for line in f:
match = self.rx_module_not_found.search(line.strip())
if match:
self.logger.error("Required MIB missed: %s", smart_text(match.group(1)))
return {
"status": False,
"msg": "Required MIB missed: %s" % smart_text(match.group(1)),
"code": ERR_MIB_MISSED,
}
match = self.rx_macro_not_imported.search(l.strip())
match = self.rx_macro_not_imported.search(line.strip())
if match:
self.logger.debug(
self.logger.error(
"Macro '%s' (%s) has not been imported",
smart_text(match.group(1)),
smart_text(match.group(2)),
......@@ -118,14 +114,14 @@ class MIBAPI(API):
# smart_text(match.group(1)), smart_text(match.group(2))),
# "code": ERR_MIB_MISSED,
# }
match = self.rx_illegal_subtype.search(l.strip())
match = self.rx_illegal_subtype.search(line.strip())
if match:
return {
"status": False,
"msg": "Illegal subtype: %s" % smart_text(match.group(1)),
"code": ERR_MIB_MISSED,
}
match = self.rx_object_identifier_unknown.search(l.strip())
match = self.rx_object_identifier_unknown.search(line.strip())
if match:
self.logger.warning(
"Object Identifier unknown: %s" % smart_text(match.group(1))
......@@ -139,7 +135,7 @@ class MIBAPI(API):
# Convert MIB to python module and load
with temporary_file() as py_path:
subprocess.check_call(
[config.path.smidump, "-k", "-q", "-f", "python", "-o", py_path, tmp_path],
[smidump_path, "-k", "-q", "-f", "python", "-o", py_path, tmp_path],
env=self.SMI_ENV,
)
with open(py_path) as f:
......
......@@ -100,10 +100,10 @@ class MIBApplication(ExtDocApplication):
errors = {}
while len(left):
n = len(left)
for name in left:
for name in list(left.keys()):
try:
svc = open_sync_rpc("mib")
r = svc.compile(left[name].read())
r = svc.compile(MIB.guess_encoding(left[name].read()))
if r.get("status"):
del left[name]
if name in errors:
......@@ -115,8 +115,9 @@ class MIBApplication(ExtDocApplication):
if len(left) == n:
# Failed to upload anything, stopping
break
r = {"success": len(left) == 0, "errors": errors}
return r
return self.render_json(
{"success": len(left) == 0, "message": f"ERROR: {errors}"}, status=self.OK
)
@view(url="^(?P<id>[0-9a-f]{24})/text/$", method=["GET"], access="launch", api=True)
def api_text(self, request, id):
......
......@@ -65,8 +65,13 @@ Ext.define("NOC.fm.mib.MIBUpload", {
NOC.info(__("MIBs has been uploaded"));
me.close();
},
failure: function() {
NOC.error(__("Failed to upload MIB"));
failure: function(arg1, arg2) {
var message = __("Failed to upload MIB");
var _arg2 = Ext.decode(arg2.response.responseText);
if(_arg2 && _arg2.message) {
message = _arg2.message;
}
NOC.error(message);
}
});
}
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment