Merged with feature/microservices

--HG--
branch : feature/strizh

lib/app/extmodelapplication.py

@@ -16,6 +16,7 @@ from django.db.models.fields import (
     DateField, DateTimeField, related)
 from django.db.models import Q
 from django.db.utils import IntegrityError
+from django.core.exceptions import ValidationError
 import six
 # NOC modules
 from extapplication import ExtApplication, view
@@ -400,13 +401,24 @@ class ExtModelApplication(ExtApplication):
                 },
                 status=self.CONFLICT)
         except self.model.MultipleObjectsReturned:
-            return self.render_json(
-                    {
-                    "status": False,
-                    "message": "Duplicated record"
+            return self.render_json({
+                "status": False,
+                "message": "Duplicated record"
                 }, status=self.CONFLICT)
         except self.model.DoesNotExist:
             o = self.model(**attrs)
+            # Run models validators
+            try:
+                o.clean_fields()
+            except ValidationError as e:
+                e_msg = []
+                for f in e.message_dict:
+                    e_msg += ["%s: %s" % (f, "; ".join(e.message_dict[f]))]
+                return self.render_json({
+                    "status": False,
+                    "message": "Validation error: %s" % " | ".join(e_msg)
+                }, status=self.BAD_REQUEST)
+            # Check for duplicates
             try:
                 o.save()
                 if m2m_attrs:
@@ -462,12 +474,26 @@ class ExtModelApplication(ExtApplication):
                     "message": "Bad request",
                     "traceback": str(e)
                 }, status=self.BAD_REQUEST)
+        # Find object
         try:
             o = self.queryset(request).get(**{self.pk: int(id)})
         except self.model.DoesNotExist:
             return HttpResponse("", status=self.NOT_FOUND)
+        # Update attributes
         for k, v in attrs.items():
             setattr(o, k, v)
+        # Run models validators
+        try:
+            o.clean_fields()
+        except ValidationError as e:
+            e_msg = []
+            for f in e.message_dict:
+                e_msg += ["%s: %s" % (f, "; ".join(e.message_dict[f]))]
+            return self.render_json({
+                "status": False,
+                "message": "Validation error: %s" % " | ".join(e_msg)
+            }, status=self.BAD_REQUEST)
+        # Save
         try:
             o.save()
             if m2m_attrs:

sa/migrations/0152_managed_object_platform.py

@@ -33,7 +33,8 @@ class Migration:
         versions = set()  # profile, version
         for profile, vendor, platform, version in data:
             platform = platform.strip()
-            if not platform:
+            vendor = vendor.strip()
+            if not platform or not vendor:
                 continue
             platforms.add((vendor, platform))
             versions.add((profile, vendor, version))
@@ -114,6 +115,10 @@ class Migration:
         )
         #
         for profile, vendor, platform, version in data:
+            platform = platform.strip()
+            vendor = vendor.strip()
+            if not platform or not vendor:
+                continue
             db.execute("""
                 UPDATE sa_managedobject
                 SET

services/card/cards/path.py

@@ -6,6 +6,8 @@
 # See LICENSE for details
 # ----------------------------------------------------------------------
 
+# Third-party modules
+import ujson
 # NOC modules
 from base import BaseCard
 from noc.sa.models.managedobject import ManagedObject
@@ -15,17 +17,41 @@ from noc.core.topology.path import get_shortest_path
 class PathCard(BaseCard):
     name = "path"
     default_template_name = "path"
+    card_css = [
+        "/ui/pkg/leaflet/leaflet.css",
+        "/ui/card/css/path.css"
+    ]
+    card_js = [
+        "/ui/pkg/leaflet/leaflet.js",
+        "/ui/card/js/path.js"
+    ]
 
     def get_data(self):
         mo1, mo2 = [ManagedObject.get_by_id(int(x)) for x in self.id.split("-")]
-        path = []
-        for mo in get_shortest_path(mo1, mo2):
-            path += [{
-                "object": mo
-            }]
+        s_path = get_shortest_path(mo1, mo2)
+        if not s_path:
+            s_path = [mo1, mo2]
 
+        path = []
+        for mo in s_path:
+            if not mo.x or not mo.y:
+                continue
+            if not path or mo.x != path[-1]["x"] or mo.y != path[-1]["y"]:
+                path += [{
+                    "x": mo.x,
+                    "y": mo.y,
+                    "objects": [{
+                        "id": mo.id,
+                        "name": mo.name
+                    }]
+                }]
+            else:
+                path[-1]["objects"] += [{
+                    "id": mo.id,
+                    "name": mo.name
+                }]
         return {
             "mo1": mo1,
             "mo2": mo2,
-            "path": path
+            "path": ujson.dumps(path)
         }

services/card/templates/path.html.j2

-<table class="table table-condensed table-hover">
-<tbody>
-<tr>
-    <th scope="row" colspan="2">
-        {{ _("Path") }}: {{ mo1.name }} ({{mo1.address}}) - {{ mo2.name }} ({{ mo2.address }})
-    </th>
-</tr>
-<tr>
-    <th>{{ _("Name") }}</th>
-    <th>{{ _("Address") }}</th>
-</tr>
-{% for p in path %}
-<tr>
-    <td>{{ p.object.name }}</td>
-    <td>{{ p.object.address }}</td>
-</tr>
-{% endfor %}
-</tbody>
-</table>
+<div id="map"></div>
+<div id="summary"><i class="fa fa-spinner fa-spin"></i> {{ _("Loading ...") }}</div>
+<script>
+$(document).ready(function() {
+    var map = new PathMap();
+    map.run({{ path }});
+});
+</script>

services/classifier/service.py

@@ -606,30 +606,33 @@ class ClassifierService(Service):
                     return
         # Finally dispose event to further processing by correlator
         if disposable and rule.to_dispose:
-            self.logger.info(
-                "[%s|%s|%s] Disposing",
-                event.id, event.managed_object.name,
-                event.managed_object.address
-            )
-            # Heat up cache
-            cache.set(
-                "activeent-%s" % event.id,
-                event,
-                ttl=900
-            )
-            # @todo: Use config.pool instead
-            self.pub(
-                "correlator.dispose.%s" % event.managed_object.pool.name,
-                {"event_id": str(event.id)}
-            )
+            self.dispose_event(event)
             metrics[CR_CLASSIFIED] += 1
-            metrics[CR_DISPOSED] += 1
         elif rule.is_unknown:
             metrics[CR_UNKNOWN] += 1
         else:
             metrics[CR_CLASSIFIED] += 1
         return
 
+    def dispose_event(self, event):
+        self.logger.info(
+            "[%s|%s|%s] Disposing",
+            event.id, event.managed_object.name,
+            event.managed_object.address
+        )
+        # Heat up cache
+        cache.set(
+            "activeent-%s" % event.id,
+            event,
+            ttl=900
+        )
+        # @todo: Use config.pool instead
+        self.pub(
+            "correlator.dispose.%s" % event.managed_object.pool.name,
+            {"event_id": str(event.id)}
+        )
+        metrics[CR_DISPOSED] += 1
+
     def find_duplicated_event(self, event, event_class, vars):
         """
         Returns duplicated event if exists
@@ -691,6 +694,8 @@ class ClassifierService(Service):
             e.vars = e.raw_vars
             e.save()
             metrics[CR_PREPROCESSED] += 1
+            if len(event_class.disposition) > 0:
+                self.dispose_event(e)
         else:
             # Classify event
             try:

ui/card/css/path.css

+#map {
+    position: absolute;
+    left: 0px;
+    top: 0px;
+    right: 0px;
+    bottom: 0px;
+}
+
+#summary {
+    position: absolute;
+    top: 0px;
+    right: 0px;
+    padding: 4pt;
+    height: auto;
+    width: auto;
+    text-align: right;
+    padding: 8px;
+    background-color: rgba(255, 255, 255, 0.6);
+    font-size: 24pt;
+    border-bottom-left-radius: 12pt;
+    z-index: 10000;
+}
+
+#summary .badge {
+    font-size: 24pt;
+    border-radius: 12pt;
+}

ui/card/js/path.js

+//----------------------------------------------------------------------
+//  Path map
+//----------------------------------------------------------------------
+// Copyright (C) 2007-2017 The NOC Project
+// See LICENSE for details
+//----------------------------------------------------------------------
+var PathMap = function () {
+    this.map = null;
+};
+
+PathMap.prototype.run = function(path) {
+    var i;
+    // Get bounding box
+    var x0 = 180.0, y0 = 180.0, x1 = -180.0, y1 = -180.0;
+    for(i = 0; i < path.length; i++) {
+        var item = path[i];
+        x0 = Math.min(x0, item.x);
+        x1 = Math.max(x1, item.x);
+        y0 = Math.min(y0, item.y);
+        y1 = Math.max(y1, item.y);
+    }
+    // Get center point (@todo: Center of mass?)
+    var xc = (x0 + x1) / 2.0,
+        yc = (y0 + y1) / 2.0;
+    var scale = 12;
+    //
+    this.map = L.map("map");
+    // Set up OSM layer
+    L.tileLayer(
+        "https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png", {
+        }).addTo(this.map);
+    // Links layer
+    var linkFeatures = [];
+    for(i = 0; i < path.length - 1; i++) {
+        linkFeatures.push({
+            type: "Feature",
+            geometry: {
+                type: "LineString",
+                coordinates: [
+                    [path[i].x, path[i].y],
+                    [path[i + 1].x, path[i + 1].y]
+                ]
+            }
+        });
+    }
+    L.geoJSON({
+        type: "FeatureCollection",
+        features: linkFeatures
+    }).addTo(this.map);
+    // PoPs layer
+    var popFeatures = [];
+    for(i = 0; i < path.length; i ++) {
+        var item = path[i];
+        // Build popup
+        var popup = ["Managed Objects:"];
+        for(var j = 0; j < item.objects.length; j++) {
+            popup.push("<a target=_ href='/api/card/view/managedobject/" + item.objects[j].id + "/'>" + item.objects[j].name + "</a>")
+        }
+        // Place PoP
+        popFeatures.push({
+            type: "Feature",
+            geometry: {
+                type: "Point",
+                coordinates: [item.x, item.y]
+            },
+            properties: {
+                popupContent: popup.join("<br>")
+            }
+        });
+    }
+    L.geoJSON({
+        type: "FeatureCollection",
+        features: popFeatures
+    }, {
+        onEachFeature: function(feature, layer) {
+            if(feature.properties && feature.properties.popupContent) {
+                layer.bindPopup(feature.properties.popupContent)
+            }
+        }
+    }).addTo(this.map);
+    // Set position
+    this.map.fitBounds([
+        [y0, x0],
+        [y1, x1]
+    ]);
+    // Remove "loading" spinner
+    $("#summary").html("");
+};