Merge branch 'noc-1603' into 'master'

noc/noc#1603 Fix upload MIB on WEB.

See merge request noc/noc!5455

ansible/noc_roles/mib/defaults/main.yml

+---
+mib_path: /var/lib/noc/mibs/

ansible/noc_roles/mib/tasks/main.yml

 ---
 - include_tasks: "os/{{ ansible_distribution }}_{{ ansible_distribution_major_version }}/main.yml"
+
+- name: Create directory for imported mibs
+  become: "True"
+  file:
+    state: directory
+    path: "{{ item }}"
+    owner: root
+    group: wheel
+    mode: 0777
+  with_items:
+    - "{{ mib_path }}"
+
+- name: Install mibs for working MIB-import
+  command: "{{ noc_root }}/scripts/deploy/install-packages requirements/mib.json"
+  args:
+    chdir: "{{ noc_root }}"
+  register: s
+  changed_when: "'CHANGED' in s.stdout"
+  environment:
+    http_proxy: "{{ http_proxy }}"
+    https_proxy: "{{ http_proxy }}"
+  tags:
+    - requirements

ansible/noc_roles/mib/tasks/os/CentOS_7/main.yml

 ---
+- name: Install smilint/smidump packages
+  yum:
+    name: "{{ packages }}"
+    update_cache: "True"
+  vars:
+    packages:
+      - libsmi
+  environment:
+    https_proxy: "{{ http_proxy }}"
+    http_proxy: "{{ http_proxy }}"
+  tags:
+    - requirements

ansible/noc_roles/mib/tasks/os/Debian_10/main.yml

 ---
+- name: Install smilint/smidump packages
+  apt:
+    name: "{{ packages }}"
+    update_cache: "True"
+    cache_valid_time: "{{ apt_cache_valid_time | default (3600) }}"
+  vars:
+    packages:
+      - smitools
+  environment:
+    https_proxy: "{{ http_proxy }}"
+    http_proxy: "{{ http_proxy }}"
+  tags:
+    - requirements

ansible/noc_roles/mib/tasks/os/Debian_9/main.yml

 ---
+- name: Install smilint/smidump packages
+  apt:
+    name: "{{ packages }}"
+    update_cache: "True"
+    cache_valid_time: "{{ apt_cache_valid_time | default (3600) }}"
+  vars:
+    packages:
+      - smitools
+  environment:
+    https_proxy: "{{ http_proxy }}"
+    http_proxy: "{{ http_proxy }}"
+  tags:
+    - requirements

ansible/noc_roles/mib/tasks/os/OracleLinux_7/main.yml

 ---
+- name: Install smilint/smidump packages
+  yum:
+    name: "{{ packages }}"
+    update_cache: "True"
+  vars:
+    packages:
+      - libsmi
+  environment:
+    https_proxy: "{{ http_proxy }}"
+    http_proxy: "{{ http_proxy }}"
+  tags:
+    - requirements

ansible/noc_roles/mib/tasks/os/RedHat_7/main.yml

 ---
+- name: Install smilint/smidump packages
+  yum:
+    name: "{{ packages }}"
+    update_cache: "True"
+  vars:
+    packages:
+      - libsmi
+  environment:
+    https_proxy: "{{ http_proxy }}"
+    http_proxy: "{{ http_proxy }}"
+  tags:
+    - requirements

ansible/noc_roles/mib/tasks/os/Ubuntu_16/main.yml

 ---
+- name: Install smilint/smidump packages
+  apt:
+    name: "{{ packages }}"
+    update_cache: "True"
+    cache_valid_time: "{{ apt_cache_valid_time | default (3600) }}"
+  vars:
+    packages:
+      - smitools
+  environment:
+    https_proxy: "{{ http_proxy }}"
+    http_proxy: "{{ http_proxy }}"
+  tags:
+    - requirements

ansible/noc_roles/mib/tasks/os/Ubuntu_18/main.yml

 ---
+- name: Install smilint/smidump packages
+  apt:
+    name: "{{ packages }}"
+    update_cache: "True"
+    cache_valid_time: "{{ apt_cache_valid_time | default (3600) }}"
+  vars:
+    packages:
+      - smitools
+  environment:
+    https_proxy: "{{ http_proxy }}"
+    http_proxy: "{{ http_proxy }}"
+  tags:
+    - requirements

commands/mib.py

@@ -268,7 +268,7 @@ class Command(BaseCommand):
         with open(path, "rb") as f:
             data = f.read()
         try:
-            r = self.svc.compile(data)
+            r = self.svc.compile(MIB.guess_encoding(data))
             if r.get("status"):
                 return True
             if r.get("code") == ERR_MIB_MISSED:

config.py

@@ -512,7 +512,7 @@ class Config(BaseConfig):
         card_template_path = StringParameter(default="services/card/templates/card.html.j2")
         pm_templates = StringParameter(default="templates/ddash/")
         custom_path = StringParameter()
-        mib_path = StringParameter(default="/var/mib")
+        mib_path = StringParameter(default="/var/lib/noc/mibs/")
 
     class pg(ConfigSection):
         addresses = ServiceParameter(service="postgres", wait=True, near=True, full_result=False)

fm/models/mib.py

@@ -8,6 +8,7 @@
 # Python modules
 import re
 import threading
+from typing import Optional, List, Dict
 import operator
 
 # Third-party modules
@@ -32,6 +33,8 @@ id_lock = threading.Lock()
 # Regular expression patterns
 rx_tailing_numbers = re.compile(r"^(\S+?)((?:\.\d+)*)$")
 
+TRY_ENCODINGS = ["utf-8", "big5"]
+
 
 @on_delete_check(check=[("fm.MIBData", "mib")])
 class MIB(Document):
@@ -283,7 +286,7 @@ class MIB(Document):
             o.save()
 
     @classmethod
-    def resolve_vars(cls, vars):
+    def resolve_vars(cls, vars: Dict[str, bytes]):
         """
         Resolve FM key -> value dict according to MIBs
 
@@ -352,6 +355,23 @@ class MIB(Document):
                 r[rk] = rv
         return r
 
+    @classmethod
+    def guess_encoding(cls, s: bytes, encodings: Optional[List[str]] = None) -> str:
+        """
+        Try to guess encoding
+        :param s:
+        :param encodings:
+        :return:
+        """
+        encodings = encodings or TRY_ENCODINGS
+        exc = None
+        for enc in encodings:
+            try:
+                return s.decode(enc)
+            except UnicodeDecodeError as e:
+                exc = e
+        raise exc
+
 
 # Avoid circular references
 from .mibdata import MIBData

services/mib/api/mib.py

@@ -7,10 +7,12 @@
 
 # Python modules
 import os
+import shutil
 import subprocess
 import re
 from importlib.machinery import SourceFileLoader
 import datetime
+from shutil import which
 
 # Third-party modules
 from typing import Optional, List
@@ -35,7 +37,6 @@ class MIBAPI(API):
     rx_illegal_subtype = re.compile(b"{subtype-enumeration-illegal}.*`([^']+)'")
     rx_object_identifier_unknown = re.compile(b"{object-identifier-unknown}.*`([^']+)'")
     rx_oid = re.compile(r"^\d+(\.\d+)+")
-    TRY_ENCODINGS = ["utf-8", "big5"]
 
     SMI_ENV = {"SMIPATH": config.path.mib_path}
 
@@ -60,22 +61,6 @@ class MIBAPI(API):
                 return {"status": True, "data": f.read()}
         return {"status": False, "msg": "Not found", "code": ERR_MIB_NOT_FOUND}
 
-    def guess_encoding(self, s: bytes, encodings: Optional[List[str]] = None) -> str:
-        """
-        Try to guess encoding
-        :param s:
-        :param encodings:
-        :return:
-        """
-        encodings = encodings or self.TRY_ENCODINGS
-        exc = None
-        for enc in encodings:
-            try:
-                return s.decode(enc)
-            except UnicodeDecodeError as e:
-                exc = e
-        raise exc
-
     @api
     def compile(self, data):
         """
@@ -83,31 +68,42 @@ class MIBAPI(API):
         :param data: MIB text
         :return:
         """
-        if not config.path.smilint or not os.path.exists(config.path.smilint):
-            return {"status": False, "msg": "smilint is missed", "error": ERR_MIB_TOOL_MISSED}
-        if not config.path.smilint or not os.path.exists(config.path.smidump):
-            return {"status": False, "msg": "smidump is missed", "error": ERR_MIB_TOOL_MISSED}
+        if config.path.smilint and os.path.exists(config.path.smilint):
+            smilint_path = config.path.smilint
+        else:
+            smilint_path = shutil.which("smilint")
+            if not smilint_path:
+                self.logger.error("Can't find smilint executable")
+                return {"status": False, "msg": "smilint is missed", "error": ERR_MIB_TOOL_MISSED}
+        if config.path.smidump and os.path.exists(config.path.smidump):
+            smidump_path = config.path.smidump
+        else:
+            smidump_path = shutil.which("smidump")
+            if not smidump_path:
+                self.logger.error("Can't find smidump executable")
+                return {"status": False, "msg": "smidump is missed", "error": ERR_MIB_TOOL_MISSED}
         # Normalize input
         if isinstance(data, bytes):
-            data = self.guess_encoding(data)
+            data = MIB.guess_encoding(data)
         # Put data to temporary file
         with temporary_file(data) as tmp_path:
             # Pass MIB through smilint to detect missed modules
-            self.logger.debug("Pass MIB through smilint to detect missed modules")
+            self.logger.info("Pass MIB through smilint to detect missed modules")
             f = subprocess.Popen(
-                [config.path.smilint, "-m", tmp_path], stderr=subprocess.PIPE, env=self.SMI_ENV
+                [smilint_path, "-m", tmp_path], stderr=subprocess.PIPE, env=self.SMI_ENV
             ).stderr
-            for l in f:
-                match = self.rx_module_not_found.search(l.strip())
+            for line in f:
+                match = self.rx_module_not_found.search(line.strip())
                 if match:
+                    self.logger.error("Required MIB missed: %s", smart_text(match.group(1)))
                     return {
                         "status": False,
                         "msg": "Required MIB missed: %s" % smart_text(match.group(1)),
                         "code": ERR_MIB_MISSED,
                     }
-                match = self.rx_macro_not_imported.search(l.strip())
+                match = self.rx_macro_not_imported.search(line.strip())
                 if match:
-                    self.logger.debug(
+                    self.logger.error(
                         "Macro '%s' (%s) has not been imported",
                         smart_text(match.group(1)),
                         smart_text(match.group(2)),
@@ -118,14 +114,14 @@ class MIBAPI(API):
                     #         smart_text(match.group(1)), smart_text(match.group(2))),
                     #     "code": ERR_MIB_MISSED,
                     # }
-                match = self.rx_illegal_subtype.search(l.strip())
+                match = self.rx_illegal_subtype.search(line.strip())
                 if match:
                     return {
                         "status": False,
                         "msg": "Illegal subtype: %s" % smart_text(match.group(1)),
                         "code": ERR_MIB_MISSED,
                     }
-                match = self.rx_object_identifier_unknown.search(l.strip())
+                match = self.rx_object_identifier_unknown.search(line.strip())
                 if match:
                     self.logger.warning(
                         "Object Identifier unknown: %s" % smart_text(match.group(1))
@@ -139,7 +135,7 @@ class MIBAPI(API):
             # Convert MIB to python module and load
             with temporary_file() as py_path:
                 subprocess.check_call(
-                    [config.path.smidump, "-k", "-q", "-f", "python", "-o", py_path, tmp_path],
+                    [smidump_path, "-k", "-q", "-f", "python", "-o", py_path, tmp_path],
                     env=self.SMI_ENV,
                 )
                 with open(py_path) as f:

services/web/apps/fm/mib/views.py

@@ -100,10 +100,10 @@ class MIBApplication(ExtDocApplication):
         errors = {}
         while len(left):
             n = len(left)
-            for name in left:
+            for name in list(left.keys()):
                 try:
                     svc = open_sync_rpc("mib")
-                    r = svc.compile(left[name].read())
+                    r = svc.compile(MIB.guess_encoding(left[name].read()))
                     if r.get("status"):
                         del left[name]
                         if name in errors:
@@ -115,8 +115,9 @@ class MIBApplication(ExtDocApplication):
             if len(left) == n:
                 # Failed to upload anything, stopping
                 break
-        r = {"success": len(left) == 0, "errors": errors}
-        return r
+        return self.render_json(
+            {"success": len(left) == 0, "message": f"ERROR: {errors}"}, status=self.OK
+        )
 
     @view(url="^(?P<id>[0-9a-f]{24})/text/$", method=["GET"], access="launch", api=True)
     def api_text(self, request, id):

ui/web/fm/mib/MIBUpload.js

@@ -65,8 +65,13 @@ Ext.define("NOC.fm.mib.MIBUpload", {
                     NOC.info(__("MIBs has been uploaded"));
                     me.close();
                 },
-                failure: function() {
-                    NOC.error(__("Failed to upload MIB"));
+                failure: function(arg1, arg2) {
+                    var message = __("Failed to upload MIB");
+                    var _arg2 = Ext.decode(arg2.response.responseText);
+                    if(_arg2 && _arg2.message) {
+                        message = _arg2.message;
+                    }
+                    NOC.error(message);
                 }
             });
         }