Rsst

sa/profiles/Generic/Object/__init__.py

+# -*- coding: utf-8 -*-
+# ---------------------------------------------------------------------
+# Generic.Host
+# Dummb profile to allow managed object creating
+# ---------------------------------------------------------------------
+# Copyright (C) 2007-2015 The NOC Project
+# See LICENSE for details
+# ---------------------------------------------------------------------
+
+# Python modules
+import re
+import json
+# NOC modules
+from noc.core.profile.base import BaseProfile
+from noc.core.http.client import fetch_sync
+
+class Profile(BaseProfile):
+    name = "Generic.Object"
+
+
+
+    class http(object):
+        """Switch context manager to use with "with" statement"""
+
+        rx_coockie = re.compile("session_id=(?P<id>\S+);", re.MULTILINE)
+
+        FORT_ACTIONS = {
+            "connect":
+                "%s/api/integration/v1/connect?login=%s&password=%s&lang=%s&timezone=%s",
+            "disconnect":
+                "%s/api/integration/v1/disconnect"
+        }
+
+        def __init__(self, script):
+            self.script = script
+            self.host = script.credentials.get("address", "")
+            self.login = script.credentials.get("user", "")
+            self.password = script.credentials.get("password", "")
+
+        def __enter__(self):
+            """Enter switch context"""
+            user = self.login
+            login = user.split("#")[0]
+            code, headers, body = fetch_sync(
+                self.FORT_ACTIONS["connect"] % (self.host, login, self.password, "ru-ru", +3),
+                follow_redirects=True,
+                allow_proxy=False
+            )
+            if code != 200:
+                raise IOError("Invalid HTTP response: %s" % code)
+            if self.rx_coockie.search(dict(headers)["Set-Cookie"]):
+                h = self.rx_coockie.findall(dict(headers)["Set-Cookie"])
+                self.coockies = {"Cookie": "SGUID=session_id=%s;" % h[0]}
+
+            return {"coockies":self.coockies, "id": user.split("#")[1]}
+
+        def __exit__(self, exc_type, exc_val, exc_tb):
+            """Leave switch context"""
+            if exc_type is None:
+                dcode, dheaders, dbody = fetch_sync(
+                    self.FORT_ACTIONS["disconnect"] % self.host,
+                    headers=self.coockies,
+                    follow_redirects=True,
+                    allow_proxy=False
+                )
+                if dcode != 200:
+                    raise IOError("Invalid HTTP response: %s" % dcode)
\ No newline at end of file

sa/profiles/Generic/Object/get_version.py

+# -*- coding: utf-8 -*-
+# ---------------------------------------------------------------------
+# Hikvision.DS2CD.get_version
+# ---------------------------------------------------------------------
+# Copyright (C) 2007-2017 The NOC Project
+# See LICENSE for details
+# ---------------------------------------------------------------------
+
+import json
+# NOC modules
+from noc.core.http.client import fetch_sync
+from noc.core.script.base import BaseScript
+from noc.sa.interfaces.igetversion import IGetVersion
+
+
+class Script(BaseScript):
+    name = "Generic.Object.get_version"
+    cache = True
+    interface = IGetVersion
+
+    def execute(self):
+        host = self.credentials.get("address", "")
+        with self.profile.http(self) as data:
+            url = "%s/api/integration/v1/fullobjinfo?oid=%s" % (host, data["id"])
+            code, headers, body = fetch_sync(
+                url,
+                headers=data["coockies"],
+                follow_redirects=True,
+                allow_proxy=False
+            )
+            if code != 200:
+                raise IOError("Invalid HTTP response: %s" % code)
+            if json.loads(body)["result"] == "NoAuth":
+                raise IOError("Not Authorized")
+            print body
+            res = json.loads(body)
+            vendor = "None"
+            platform = "None"
+            version = "None"
+            for r in res["properties"]:
+                if r["name"] == u"Марка ТС":
+                    vendor = r["val"]
+                if r["name"] == u"Модель ТС":
+                    platform = r["val"]
+                if r["name"] == u"VIN-код":
+                    version = r["val"]
+            return {
+                    "vendor": vendor,
+                    "platform": platform,
+                    "version": version
+                }

services/gpslogcollector/__init__.py

+# -*- coding: utf-8 -*-

services/gpslogcollector/gpslogserver.py

+# -*- coding: utf-8 -*-
+# ---------------------------------------------------------------------
+# Syslog server
+# ---------------------------------------------------------------------
+# Copyright (C) 2007-2015 The NOC Project
+# See LICENSE for details
+# ---------------------------------------------------------------------
+
+# Python modules
+import logging
+import time
+# NOC modules
+from noc.core.ioloop.udpserver import UDPServer
+
+logger = logging.getLogger(__name__)
+
+
+class GPSlogServer(UDPServer):
+    def __init__(self, service, io_loop=None):
+        super(GPSlogServer, self).__init__(io_loop)
+        self.service = service
+
+    def on_read(self, data, address):
+        self.service.perf_metrics["gpslog_msg_in"] += 1
+        object = self.service.lookup_object(address[0])
+        if not object:
+            return  # Invalid event source
+        # Convert data to valid UTF8
+        data = unicode(data, "utf8", "ignore").encode("utf8")
+        # Parse priority
+        priority = 0
+        if data.startswith("<"):
+            idx = data.find(">")
+            if idx == -1:
+                return
+            try:
+                priority = int(data[1:idx])
+            except ValueError:
+                pass
+            data = data[idx + 1:].strip()
+        # Get timestamp
+        ts = int(time.time())
+        #
+        self.service.register_message(
+            object, ts, data,
+            facility=priority >> 3,
+            severity=priority & 7
+        )

services/gpslogcollector/service.py

+#!./bin/python
+# -*- coding: utf-8 -*-
+# ---------------------------------------------------------------------
+# Syslog Collector service
+# ---------------------------------------------------------------------
+# Copyright (C) 2007-2016 The NOC Project
+# See LICENSE for details
+# ---------------------------------------------------------------------
+
+# Python modules
+import socket
+from collections import defaultdict
+# Third-party modules
+import tornado.ioloop
+import tornado.gen
+# NOC modules
+from noc.config import config
+from noc.core.service.base import Service
+from noc.services.gpslogcollector.gpslogserver import GPSlogServer
+
+
+class GPSlogCollectorService(Service):
+    name = "gpslogcollector"    #
+    leader_group_name = "gpslogcollector-%(dc)s-%(node)s"
+    pooled = True
+    require_nsq_writer = True
+    process_name = "noc-%(name).10s-%(pool).5s"
+
+    def __init__(self):
+        super(GPSlogCollectorService, self).__init__()
+        self.messages = []
+        self.send_callback = None
+        self.mappings_callback = None
+        self.report_invalid_callback = None
+        self.source_map = {}
+        self.invalid_sources = defaultdict(int)  # ip -> count
+        self.omap = None
+        self.fmwriter = None
+
+    @tornado.gen.coroutine
+    def on_activate(self):
+        # Register RPC aliases
+        self.omap = self.open_rpc("omap")
+        self.fmwriter = self.open_rpc("fmwriter", pool=config.pool)
+        # Set event listeners
+        # self.subscribe("objmapchange.%(pool)s",
+        #                self.on_object_map_change)
+        # Listen sockets
+        server = GPSlogServer(service=self)
+        a = ("0.0.0.0", "12374")
+        #for l in config.gpslogcollector.listen.split(","):
+        for l in a:
+            if ":" in l:
+                addr, port = l.split(":")
+            else:
+                addr, port = "", l
+            self.logger.info("Starting gpslog server at %s:%s",
+                             addr, port)
+            try:
+                server.listen(port, addr)
+            except socket.error as e:
+                self.perf_metrics["error", ("type", "socket_listen_error")] += 1
+                self.logger.error(
+                    "Failed to start gpslog server at %s:%s: %s",
+                    addr, port, e
+                )
+        server.start()
+        # Send spooled messages every 250ms
+        self.logger.debug("Stating message sender task")
+        self.send_callback = tornado.ioloop.PeriodicCallback(
+            self.send_messages,
+            250,
+            self.ioloop
+        )
+        self.send_callback.start()
+        # Get object mappings every 300s
+        self.logger.debug("Stating object mapping task")
+        self.mappings_callback = tornado.ioloop.PeriodicCallback(
+            self.get_object_mappings,
+            300000,
+            self.ioloop
+        )
+        self.mappings_callback.start()
+        self.ioloop.add_callback(self.get_object_mappings)
+        # Report invalid sources every 60 seconds
+        self.logger.info("Stating invalid sources reporting task")
+        self.report_invalid_callback = tornado.ioloop.PeriodicCallback(
+            self.report_invalid_sources,
+            60000,
+            self.ioloop
+        )
+        self.report_invalid_callback.start()
+
+    def lookup_object(self, address):
+        """
+        Returns object id for given address or None when
+        unknown source
+        """
+        obj_id = self.source_map.get(address)
+        if not obj_id:
+            # Register invalid event source
+            if self.source_map:
+                self.invalid_sources[address] += 1
+            self.perf_metrics["error", ("type", "object_not_found")] += 1
+            return None
+        return obj_id
+
+    def register_message(self, object, timestamp, message,
+                         facility, severity):
+        """
+        Spool message to be sent
+        """
+        self.perf_metrics["events_out"] += 1
+        self.messages += [{
+            "ts": timestamp,
+            "object": object,
+            "data": {
+                "source": "syslog",
+                "collector": config.pool,
+                "message": message
+            }
+        }]
+
+    @tornado.gen.coroutine
+    def send_messages(self):
+        """
+        Periodic task to send collected messages to fmwriter
+        """
+        if self.messages:
+            messages, self.messages = self.messages, []
+            self.mpub("events.%s" % config.pool, messages)
+
+    @tornado.gen.coroutine
+    def get_object_mappings(self):
+        """
+        Periodic task to request object mappings
+        """
+        self.logger.debug("Requesting object mappings")
+        sm = yield self.omap.get_syslog_mappings(config.pool)
+        if sm != self.source_map:
+            self.logger.debug("Setting object mappings to: %s", sm)
+            self.source_map = sm
+
+    @tornado.gen.coroutine
+    def report_invalid_sources(self):
+        """
+        Report invalid event sources
+        """
+        if not self.invalid_sources:
+            return
+        total = sum(self.invalid_sources[s] for s in self.invalid_sources)
+        self.logger.info(
+            "Dropping %d messages with invalid sources: %s",
+            total,
+            ", ".join("%s: %s" % (s, self.invalid_sources[s])
+                      for s in self.invalid_sources)
+        )
+        self.invalid_sources = defaultdict(int)
+
+    def on_object_map_change(self, topic):
+        self.logger.info("Object mappings changed. Rerequesting")
+        self.ioloop.add_callback(self.get_object_mappings)
+
+
+if __name__ == "__main__":
+    GPSlogCollectorService().start()