Commit 95712963 authored by Andrey Vertiprahov's avatar Andrey Vertiprahov Committed by EKbfh
Browse files

noc/noc#1603 Fix upload MIB on WEB.

parent 56eae3b7
---
mib_path: /var/lib/noc/mibs/
--- ---
- include_tasks: "os/{{ ansible_distribution }}_{{ ansible_distribution_major_version }}/main.yml" - include_tasks: "os/{{ ansible_distribution }}_{{ ansible_distribution_major_version }}/main.yml"
- name: Create directory for imported mibs
become: "True"
file:
state: directory
path: "{{ item }}"
owner: root
group: wheel
mode: 0777
with_items:
- "{{ mib_path }}"
- name: Install mibs for working MIB-import
command: "{{ noc_root }}/scripts/deploy/install-packages requirements/mib.json"
args:
chdir: "{{ noc_root }}"
register: s
changed_when: "'CHANGED' in s.stdout"
environment:
http_proxy: "{{ http_proxy }}"
https_proxy: "{{ http_proxy }}"
tags:
- requirements
--- ---
- name: Install smilint/smidump packages
yum:
name: "{{ packages }}"
update_cache: "True"
vars:
packages:
- libsmi
environment:
https_proxy: "{{ http_proxy }}"
http_proxy: "{{ http_proxy }}"
tags:
- requirements
--- ---
- name: Install smilint/smidump packages
apt:
name: "{{ packages }}"
update_cache: "True"
cache_valid_time: "{{ apt_cache_valid_time | default (3600) }}"
vars:
packages:
- smitools
environment:
https_proxy: "{{ http_proxy }}"
http_proxy: "{{ http_proxy }}"
tags:
- requirements
--- ---
- name: Install smilint/smidump packages
apt:
name: "{{ packages }}"
update_cache: "True"
cache_valid_time: "{{ apt_cache_valid_time | default (3600) }}"
vars:
packages:
- smitools
environment:
https_proxy: "{{ http_proxy }}"
http_proxy: "{{ http_proxy }}"
tags:
- requirements
--- ---
- name: Install smilint/smidump packages
yum:
name: "{{ packages }}"
update_cache: "True"
vars:
packages:
- libsmi
environment:
https_proxy: "{{ http_proxy }}"
http_proxy: "{{ http_proxy }}"
tags:
- requirements
--- ---
- name: Install smilint/smidump packages
yum:
name: "{{ packages }}"
update_cache: "True"
vars:
packages:
- libsmi
environment:
https_proxy: "{{ http_proxy }}"
http_proxy: "{{ http_proxy }}"
tags:
- requirements
--- ---
- name: Install smilint/smidump packages
apt:
name: "{{ packages }}"
update_cache: "True"
cache_valid_time: "{{ apt_cache_valid_time | default (3600) }}"
vars:
packages:
- smitools
environment:
https_proxy: "{{ http_proxy }}"
http_proxy: "{{ http_proxy }}"
tags:
- requirements
--- ---
- name: Install smilint/smidump packages
apt:
name: "{{ packages }}"
update_cache: "True"
cache_valid_time: "{{ apt_cache_valid_time | default (3600) }}"
vars:
packages:
- smitools
environment:
https_proxy: "{{ http_proxy }}"
http_proxy: "{{ http_proxy }}"
tags:
- requirements
...@@ -268,7 +268,7 @@ class Command(BaseCommand): ...@@ -268,7 +268,7 @@ class Command(BaseCommand):
with open(path, "rb") as f: with open(path, "rb") as f:
data = f.read() data = f.read()
try: try:
r = self.svc.compile(data) r = self.svc.compile(MIB.guess_encoding(data))
if r.get("status"): if r.get("status"):
return True return True
if r.get("code") == ERR_MIB_MISSED: if r.get("code") == ERR_MIB_MISSED:
......
...@@ -512,7 +512,7 @@ class Config(BaseConfig): ...@@ -512,7 +512,7 @@ class Config(BaseConfig):
card_template_path = StringParameter(default="services/card/templates/card.html.j2") card_template_path = StringParameter(default="services/card/templates/card.html.j2")
pm_templates = StringParameter(default="templates/ddash/") pm_templates = StringParameter(default="templates/ddash/")
custom_path = StringParameter() custom_path = StringParameter()
mib_path = StringParameter(default="/var/mib") mib_path = StringParameter(default="/var/lib/noc/mibs/")
class pg(ConfigSection): class pg(ConfigSection):
addresses = ServiceParameter(service="postgres", wait=True, near=True, full_result=False) addresses = ServiceParameter(service="postgres", wait=True, near=True, full_result=False)
......
...@@ -8,6 +8,7 @@ ...@@ -8,6 +8,7 @@
# Python modules # Python modules
import re import re
import threading import threading
from typing import Optional, List, Dict
import operator import operator
# Third-party modules # Third-party modules
...@@ -32,6 +33,8 @@ id_lock = threading.Lock() ...@@ -32,6 +33,8 @@ id_lock = threading.Lock()
# Regular expression patterns # Regular expression patterns
rx_tailing_numbers = re.compile(r"^(\S+?)((?:\.\d+)*)$") rx_tailing_numbers = re.compile(r"^(\S+?)((?:\.\d+)*)$")
TRY_ENCODINGS = ["utf-8", "big5"]
@on_delete_check(check=[("fm.MIBData", "mib")]) @on_delete_check(check=[("fm.MIBData", "mib")])
class MIB(Document): class MIB(Document):
...@@ -283,7 +286,7 @@ class MIB(Document): ...@@ -283,7 +286,7 @@ class MIB(Document):
o.save() o.save()
@classmethod @classmethod
def resolve_vars(cls, vars): def resolve_vars(cls, vars: Dict[str, bytes]):
""" """
Resolve FM key -> value dict according to MIBs Resolve FM key -> value dict according to MIBs
...@@ -352,6 +355,23 @@ class MIB(Document): ...@@ -352,6 +355,23 @@ class MIB(Document):
r[rk] = rv r[rk] = rv
return r return r
@classmethod
def guess_encoding(cls, s: bytes, encodings: Optional[List[str]] = None) -> str:
"""
Try to guess encoding
:param s:
:param encodings:
:return:
"""
encodings = encodings or TRY_ENCODINGS
exc = None
for enc in encodings:
try:
return s.decode(enc)
except UnicodeDecodeError as e:
exc = e
raise exc
# Avoid circular references # Avoid circular references
from .mibdata import MIBData from .mibdata import MIBData
...@@ -7,10 +7,12 @@ ...@@ -7,10 +7,12 @@
# Python modules # Python modules
import os import os
import shutil
import subprocess import subprocess
import re import re
from importlib.machinery import SourceFileLoader from importlib.machinery import SourceFileLoader
import datetime import datetime
from shutil import which
# Third-party modules # Third-party modules
from typing import Optional, List from typing import Optional, List
...@@ -35,7 +37,6 @@ class MIBAPI(API): ...@@ -35,7 +37,6 @@ class MIBAPI(API):
rx_illegal_subtype = re.compile(b"{subtype-enumeration-illegal}.*`([^']+)'") rx_illegal_subtype = re.compile(b"{subtype-enumeration-illegal}.*`([^']+)'")
rx_object_identifier_unknown = re.compile(b"{object-identifier-unknown}.*`([^']+)'") rx_object_identifier_unknown = re.compile(b"{object-identifier-unknown}.*`([^']+)'")
rx_oid = re.compile(r"^\d+(\.\d+)+") rx_oid = re.compile(r"^\d+(\.\d+)+")
TRY_ENCODINGS = ["utf-8", "big5"]
SMI_ENV = {"SMIPATH": config.path.mib_path} SMI_ENV = {"SMIPATH": config.path.mib_path}
...@@ -60,22 +61,6 @@ class MIBAPI(API): ...@@ -60,22 +61,6 @@ class MIBAPI(API):
return {"status": True, "data": f.read()} return {"status": True, "data": f.read()}
return {"status": False, "msg": "Not found", "code": ERR_MIB_NOT_FOUND} return {"status": False, "msg": "Not found", "code": ERR_MIB_NOT_FOUND}
def guess_encoding(self, s: bytes, encodings: Optional[List[str]] = None) -> str:
"""
Try to guess encoding
:param s:
:param encodings:
:return:
"""
encodings = encodings or self.TRY_ENCODINGS
exc = None
for enc in encodings:
try:
return s.decode(enc)
except UnicodeDecodeError as e:
exc = e
raise exc
@api @api
def compile(self, data): def compile(self, data):
""" """
...@@ -83,31 +68,42 @@ class MIBAPI(API): ...@@ -83,31 +68,42 @@ class MIBAPI(API):
:param data: MIB text :param data: MIB text
:return: :return:
""" """
if not config.path.smilint or not os.path.exists(config.path.smilint): if config.path.smilint and os.path.exists(config.path.smilint):
return {"status": False, "msg": "smilint is missed", "error": ERR_MIB_TOOL_MISSED} smilint_path = config.path.smilint
if not config.path.smilint or not os.path.exists(config.path.smidump): else:
return {"status": False, "msg": "smidump is missed", "error": ERR_MIB_TOOL_MISSED} smilint_path = shutil.which("smilint")
if not smilint_path:
self.logger.error("Can't find smilint executable")
return {"status": False, "msg": "smilint is missed", "error": ERR_MIB_TOOL_MISSED}
if config.path.smidump and os.path.exists(config.path.smidump):
smidump_path = config.path.smidump
else:
smidump_path = shutil.which("smidump")
if not smidump_path:
self.logger.error("Can't find smidump executable")
return {"status": False, "msg": "smidump is missed", "error": ERR_MIB_TOOL_MISSED}
# Normalize input # Normalize input
if isinstance(data, bytes): if isinstance(data, bytes):
data = self.guess_encoding(data) data = MIB.guess_encoding(data)
# Put data to temporary file # Put data to temporary file
with temporary_file(data) as tmp_path: with temporary_file(data) as tmp_path:
# Pass MIB through smilint to detect missed modules # Pass MIB through smilint to detect missed modules
self.logger.debug("Pass MIB through smilint to detect missed modules") self.logger.info("Pass MIB through smilint to detect missed modules")
f = subprocess.Popen( f = subprocess.Popen(
[config.path.smilint, "-m", tmp_path], stderr=subprocess.PIPE, env=self.SMI_ENV [smilint_path, "-m", tmp_path], stderr=subprocess.PIPE, env=self.SMI_ENV
).stderr ).stderr
for l in f: for line in f:
match = self.rx_module_not_found.search(l.strip()) match = self.rx_module_not_found.search(line.strip())
if match: if match:
self.logger.error("Required MIB missed: %s", smart_text(match.group(1)))
return { return {
"status": False, "status": False,
"msg": "Required MIB missed: %s" % smart_text(match.group(1)), "msg": "Required MIB missed: %s" % smart_text(match.group(1)),
"code": ERR_MIB_MISSED, "code": ERR_MIB_MISSED,
} }
match = self.rx_macro_not_imported.search(l.strip()) match = self.rx_macro_not_imported.search(line.strip())
if match: if match:
self.logger.debug( self.logger.error(
"Macro '%s' (%s) has not been imported", "Macro '%s' (%s) has not been imported",
smart_text(match.group(1)), smart_text(match.group(1)),
smart_text(match.group(2)), smart_text(match.group(2)),
...@@ -118,14 +114,14 @@ class MIBAPI(API): ...@@ -118,14 +114,14 @@ class MIBAPI(API):
# smart_text(match.group(1)), smart_text(match.group(2))), # smart_text(match.group(1)), smart_text(match.group(2))),
# "code": ERR_MIB_MISSED, # "code": ERR_MIB_MISSED,
# } # }
match = self.rx_illegal_subtype.search(l.strip()) match = self.rx_illegal_subtype.search(line.strip())
if match: if match:
return { return {
"status": False, "status": False,
"msg": "Illegal subtype: %s" % smart_text(match.group(1)), "msg": "Illegal subtype: %s" % smart_text(match.group(1)),
"code": ERR_MIB_MISSED, "code": ERR_MIB_MISSED,
} }
match = self.rx_object_identifier_unknown.search(l.strip()) match = self.rx_object_identifier_unknown.search(line.strip())
if match: if match:
self.logger.warning( self.logger.warning(
"Object Identifier unknown: %s" % smart_text(match.group(1)) "Object Identifier unknown: %s" % smart_text(match.group(1))
...@@ -139,7 +135,7 @@ class MIBAPI(API): ...@@ -139,7 +135,7 @@ class MIBAPI(API):
# Convert MIB to python module and load # Convert MIB to python module and load
with temporary_file() as py_path: with temporary_file() as py_path:
subprocess.check_call( subprocess.check_call(
[config.path.smidump, "-k", "-q", "-f", "python", "-o", py_path, tmp_path], [smidump_path, "-k", "-q", "-f", "python", "-o", py_path, tmp_path],
env=self.SMI_ENV, env=self.SMI_ENV,
) )
with open(py_path) as f: with open(py_path) as f:
......
...@@ -100,10 +100,10 @@ class MIBApplication(ExtDocApplication): ...@@ -100,10 +100,10 @@ class MIBApplication(ExtDocApplication):
errors = {} errors = {}
while len(left): while len(left):
n = len(left) n = len(left)
for name in left: for name in list(left.keys()):
try: try:
svc = open_sync_rpc("mib") svc = open_sync_rpc("mib")
r = svc.compile(left[name].read()) r = svc.compile(MIB.guess_encoding(left[name].read()))
if r.get("status"): if r.get("status"):
del left[name] del left[name]
if name in errors: if name in errors:
...@@ -115,8 +115,9 @@ class MIBApplication(ExtDocApplication): ...@@ -115,8 +115,9 @@ class MIBApplication(ExtDocApplication):
if len(left) == n: if len(left) == n:
# Failed to upload anything, stopping # Failed to upload anything, stopping
break break
r = {"success": len(left) == 0, "errors": errors} return self.render_json(
return r {"success": len(left) == 0, "message": f"ERROR: {errors}"}, status=self.OK
)
@view(url="^(?P<id>[0-9a-f]{24})/text/$", method=["GET"], access="launch", api=True) @view(url="^(?P<id>[0-9a-f]{24})/text/$", method=["GET"], access="launch", api=True)
def api_text(self, request, id): def api_text(self, request, id):
......
...@@ -65,8 +65,13 @@ Ext.define("NOC.fm.mib.MIBUpload", { ...@@ -65,8 +65,13 @@ Ext.define("NOC.fm.mib.MIBUpload", {
NOC.info(__("MIBs has been uploaded")); NOC.info(__("MIBs has been uploaded"));
me.close(); me.close();
}, },
failure: function() { failure: function(arg1, arg2) {
NOC.error(__("Failed to upload MIB")); var message = __("Failed to upload MIB");
var _arg2 = Ext.decode(arg2.response.responseText);
if(_arg2 && _arg2.message) {
message = _arg2.message;
}
NOC.error(message);
} }
}); });
} }
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment