From a216afd32dd0588ea7e43c9415007e149f183177 Mon Sep 17 00:00:00 2001 From: Vladimir Komarov Date: Mon, 25 Jan 2021 16:10:07 +0700 Subject: [PATCH 01/22] vak-908 first draft of netflow service. --- services/netflowcollector/__init__.py | 0 services/netflowcollector/datastream.py | 17 ++ services/netflowcollector/netflowserver.py | 54 ++++++ services/netflowcollector/service.py | 199 +++++++++++++++++++++ services/netflowcollector/sourceconfig.py | 21 +++ 5 files changed, 291 insertions(+) create mode 100644 services/netflowcollector/__init__.py create mode 100644 services/netflowcollector/datastream.py create mode 100644 services/netflowcollector/netflowserver.py create mode 100755 services/netflowcollector/service.py create mode 100644 services/netflowcollector/sourceconfig.py diff --git a/services/netflowcollector/__init__.py b/services/netflowcollector/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/services/netflowcollector/datastream.py b/services/netflowcollector/datastream.py new file mode 100644 index 0000000000..c47561c29b --- /dev/null +++ b/services/netflowcollector/datastream.py @@ -0,0 +1,17 @@ +# ---------------------------------------------------------------------- +# Netflow DataStream client +# ---------------------------------------------------------------------- +# Copyright (C) 2007-2021 The NOC Project +# See LICENSE for details +# ---------------------------------------------------------------------- + +# NOC modules +from noc.core.datastream.client import DataStreamClient + + +class NetflowDataStreamClient(DataStreamClient): + async def on_change(self, data): + await self.service.update_source(data) + + async def on_delete(self, data): + await self.service.delete_source(data["id"]) diff --git a/services/netflowcollector/netflowserver.py b/services/netflowcollector/netflowserver.py new file mode 100644 index 0000000000..e83389c117 --- /dev/null +++ b/services/netflowcollector/netflowserver.py @@ -0,0 +1,54 @@ +# --------------------------------------------------------------------- +# Netflow server +# --------------------------------------------------------------------- +# Copyright (C) 2007-2021 The NOC Project +# See LICENSE for details +# --------------------------------------------------------------------- + +# Python modules +import logging +import time +from typing import Tuple + +# NOC modules +from noc.config import config +from noc.core.perf import metrics +from noc.core.ioloop.udpserver import UDPServer +from noc.core.comp import smart_text + +logger = logging.getLogger(__name__) + + +class NetflowServer(UDPServer): + def __init__(self, service): + super().__init__() + self.service = service + + def enable_reuseport(self): + return config.syslogcollector.enable_reuseport + + def enable_freebind(self): + return config.syslogcollector.enable_freebind + + def on_read(self, data: bytes, address: Tuple[str, int]): + metrics["syslog_msg_in"] += 1 + cfg = self.service.lookup_config(address[0]) + if not cfg: + return # Invalid event source + # Convert data to valid UTF8 + data = smart_text(data, errors="ignore") + # Parse priority + priority = 0 + if data.startswith("<"): + idx = data.find(">") + if idx == -1: + return + try: + priority = int(data[1:idx]) + except ValueError: + pass + data = data[idx + 1 :].strip() + # Get timestamp + ts = int(time.time()) + # + self.service.register_message(cfg, ts, data, facility=priority >> 3, severity=priority & 7) diff --git a/services/netflowcollector/service.py b/services/netflowcollector/service.py new file mode 100755 index 0000000000..7b1459e145 --- /dev/null +++ b/services/netflowcollector/service.py @@ -0,0 +1,199 @@ +#!./bin/python +# --------------------------------------------------------------------- +# Netflow Collector service +# --------------------------------------------------------------------- +# Copyright (C) 2007-2021 The NOC Project +# See LICENSE for details +# --------------------------------------------------------------------- + +# Python modules +import datetime +from collections import defaultdict +import asyncio +from typing import Optional, Dict + +# Third-party modules +import orjson + +# NOC modules +from noc.config import config +from noc.core.error import NOCError +from noc.core.service.tornado import TornadoService +from noc.core.perf import metrics +from noc.services.netflowcollector.netflowserver import NetflowServer +from noc.services.netflowcollector.datastream import NetflowDataStreamClient +from noc.services.netflowcollector.sourceconfig import SourceConfig +from noc.core.ioloop.timers import PeriodicCallback + + +class NetflowCollectorService(TornadoService): + name = "netflowcollector" + leader_group_name = "netflowcollector-%(dc)s-%(node)s" + pooled = True + process_name = "noc-%(name).10s-%(pool).5s" + + def __init__(self): + super().__init__() + self.mappings_callback = None + self.report_invalid_callback = None + self.source_configs = {} # id -> SourceConfig + self.address_configs = {} # address -> SourceConfig + self.invalid_sources = defaultdict(int) # ip -> count + self.pool_partitions: Dict[str, int] = {} + + async def on_activate(self): + # Listen sockets + server = NetflowServer(service=self) + for addr, port in server.iter_listen(config.netflowcollector.listen): + self.logger.info("Starting syslog server at %s:%s", addr, port) + try: + server.listen(port, addr) + except OSError as e: + metrics["error", ("type", "socket_listen_error")] += 1 + self.logger.error("Failed to start syslog server at %s:%s: %s", addr, port, e) + server.start() + # Report invalid sources every 60 seconds + self.logger.info("Stating invalid sources reporting task") + self.report_invalid_callback = PeriodicCallback(self.report_invalid_sources, 60000) + self.report_invalid_callback.start() + # Start tracking changes + asyncio.get_running_loop().create_task(self.get_object_mappings()) + + async def get_pool_partitions(self, pool: str) -> int: + parts = self.pool_partitions.get(pool) + if not parts: + parts = await self.get_stream_partitions("events.%s" % pool) + self.pool_partitions[pool] = parts + return parts + + def lookup_config(self, address: str) -> Optional[SourceConfig]: + """ + Returns object id for given address or None when + unknown source + """ + cfg = self.address_configs.get(address) + if cfg: + return cfg + # Register invalid event source + if self.address_configs: + self.invalid_sources[address] += 1 + metrics["error", ("type", "object_not_found")] += 1 + return None + + def register_message( + self, cfg: SourceConfig, timestamp: int, message: str, facility: int, severity: int + ) -> None: + """ + Spool message to be sent + """ + if cfg.process_events: + # Send to classifier + metrics["events_out"] += 1 + self.publish( + orjson.dumps( + { + "ts": timestamp, + "object": cfg.id, + "data": {"source": "netflow", "collector": config.pool, "message": message}, + } + ), + stream=cfg.stream, + partition=cfg.partition, + ) + if cfg.archive_events and cfg.bi_id: + # Archive message + metrics["events_archived"] += 1 + now = datetime.datetime.now() + ts = now.strftime("%Y-%m-%d %H:%M:%S") + date = ts.split(" ")[0] + self.register_metrics( + "syslog", + [ + { + "date": date, + "ts": ts, + "managed_object": cfg.bi_id, + "facility": facility, + "severity": severity, + "message": message, + } + ], + ) + + async def get_object_mappings(self): + """ + Subscribe and track datastream changes + """ + # Register RPC aliases + client = NetflowDataStreamClient("cfgnetflow", service=self) + # Track stream changes + while True: + self.logger.info("Starting to track object mappings") + try: + await client.query( + limit=config.syslogcollector.ds_limit, + filters=["pool(%s)" % config.pool], + block=1, + ) + except NOCError as e: + self.logger.info("Failed to get object mappings: %s", e) + await asyncio.sleep(1) + + async def report_invalid_sources(self): + """ + Report invalid event sources + """ + if not self.invalid_sources: + return + total = sum(self.invalid_sources[s] for s in self.invalid_sources) + self.logger.info( + "Dropping %d messages with invalid sources: %s", + total, + ", ".join("%s: %s" % (s, self.invalid_sources[s]) for s in self.invalid_sources), + ) + self.invalid_sources = defaultdict(int) + + async def update_source(self, data): + # Get old config + old_cfg = self.source_configs.get(data["id"]) + if old_cfg: + old_addresses = set(old_cfg.addresses) + else: + old_addresses = set() + # Get pool and sharding information + fm_pool = data.get("fm_pool", None) or config.pool + num_partitions = await self.get_pool_partitions(fm_pool) + # Build new config + cfg = SourceConfig( + id=data["id"], + addresses=tuple(data["addresses"]), + bi_id=data.get("bi_id"), # For backward compatibility + process_events=data.get("process_events", True), # For backward compatibility + archive_events=data.get("archive_events", False), + stream="events.%s" % fm_pool, + partition=int(data["id"]) % num_partitions, + ) + new_addresses = set(cfg.addresses) + # Add new addresses, update remaining + for addr in new_addresses: + self.address_configs[addr] = cfg + # Revoke stale addresses + for addr in old_addresses - new_addresses: + del self.address_configs[addr] + # Update configs + self.source_configs[data["id"]] = cfg + # Update metrics + metrics["sources_changed"] += 1 + + async def delete_source(self, id): + cfg = self.source_configs.get(id) + if not cfg: + return + for addr in cfg.addresses: + del self.address_configs[addr] + del self.source_configs[id] + metrics["sources_deleted"] += 1 + + +if __name__ == "__main__": + NetflowCollectorService().start() diff --git a/services/netflowcollector/sourceconfig.py b/services/netflowcollector/sourceconfig.py new file mode 100644 index 0000000000..7ae5a7f483 --- /dev/null +++ b/services/netflowcollector/sourceconfig.py @@ -0,0 +1,21 @@ +# ---------------------------------------------------------------------- +# SourceConfig +# ---------------------------------------------------------------------- +# Copyright (C) 2007-2020 The NOC Project +# See LICENSE for details +# ---------------------------------------------------------------------- + +# Python modules +from dataclasses import dataclass +from typing import Tuple + + +@dataclass +class SourceConfig(object): + id: str + addresses: Tuple[str, ...] + bi_id: int + process_events: bool + archive_events: bool + stream: str + partition: int -- GitLab From c0f2bf99ef3ba0ec1901f1520eb0aabdec3d08da Mon Sep 17 00:00:00 2001 From: Vladimir Komarov Date: Mon, 25 Jan 2021 19:51:44 +0700 Subject: [PATCH 02/22] vak-908 created config.netflowcollector --- config.py | 7 +++++++ services/netflowcollector/netflowserver.py | 6 +++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/config.py b/config.py index 1085524e35..aeea6b6513 100644 --- a/config.py +++ b/config.py @@ -550,6 +550,13 @@ class Config(BaseConfig): # DataStream request limit ds_limit = IntParameter(default=1000) + class netflowcollector(ConfigSection): + listen = StringParameter(default="0.0.0.0:2055") + enable_reuseport = BooleanParameter(default=True) + enable_freebind = BooleanParameter(default=False) + # DataStream request limit + ds_limit = IntParameter(default=1000) + class icqsender(ConfigSection): token = SecretParameter() retry_timeout = IntParameter(default=2) diff --git a/services/netflowcollector/netflowserver.py b/services/netflowcollector/netflowserver.py index e83389c117..071d8ea5b6 100644 --- a/services/netflowcollector/netflowserver.py +++ b/services/netflowcollector/netflowserver.py @@ -25,13 +25,13 @@ class NetflowServer(UDPServer): self.service = service def enable_reuseport(self): - return config.syslogcollector.enable_reuseport + return config.netflowcollector.enable_reuseport def enable_freebind(self): - return config.syslogcollector.enable_freebind + return config.netflowcollector.enable_freebind def on_read(self, data: bytes, address: Tuple[str, int]): - metrics["syslog_msg_in"] += 1 + metrics["netflow_msg_in"] += 1 cfg = self.service.lookup_config(address[0]) if not cfg: return # Invalid event source -- GitLab From 5eec4ffc707c3e9a3331584c923a2c1624df1047 Mon Sep 17 00:00:00 2001 From: Vladimir Komarov Date: Tue, 26 Jan 2021 16:09:11 +0700 Subject: [PATCH 03/22] vak-908 changed some messages: from syslog to netflow --- services/netflowcollector/service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/netflowcollector/service.py b/services/netflowcollector/service.py index 7b1459e145..6fde17ee1d 100755 --- a/services/netflowcollector/service.py +++ b/services/netflowcollector/service.py @@ -45,12 +45,12 @@ class NetflowCollectorService(TornadoService): # Listen sockets server = NetflowServer(service=self) for addr, port in server.iter_listen(config.netflowcollector.listen): - self.logger.info("Starting syslog server at %s:%s", addr, port) + self.logger.info("Starting netflow server at %s:%s", addr, port) try: server.listen(port, addr) except OSError as e: metrics["error", ("type", "socket_listen_error")] += 1 - self.logger.error("Failed to start syslog server at %s:%s: %s", addr, port, e) + self.logger.error("Failed to start netflow server at %s:%s: %s", addr, port, e) server.start() # Report invalid sources every 60 seconds self.logger.info("Stating invalid sources reporting task") -- GitLab From 383a7bb3b73cea57f1c97d206c2b69eaebd293a2 Mon Sep 17 00:00:00 2001 From: Vladimir Komarov Date: Mon, 1 Feb 2021 13:42:28 +0700 Subject: [PATCH 04/22] vak-1501 created exeptions classes. --- services/login/exceptions.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 services/login/exceptions.py diff --git a/services/login/exceptions.py b/services/login/exceptions.py new file mode 100644 index 0000000000..fb938f20f0 --- /dev/null +++ b/services/login/exceptions.py @@ -0,0 +1,33 @@ +# --------------------------------------------------------------------- +# Additional exceptions +# --------------------------------------------------------------------- +# Copyright (C) 2021 The NOC Project +# See LICENSE for details +# --------------------------------------------------------------------- + +import http + +from typing import Any, Dict, Optional + + +class StarletteHTTPException(Exception): + def __init__(self, status_code: int, error: str = None) -> None: + if error is None: + error = http.HTTPStatus(status_code).phrase + self.status_code = status_code + self.error = error + + def __repr__(self) -> str: + class_name = self.__class__.__name__ + return f"{class_name}(status_code={self.status_code!r}, detail={self.error!r})" + + +class HTTPException(StarletteHTTPException): + def __init__( + self, + status_code: int, + error: Any = None, + headers: Optional[Dict[str, Any]] = None, + ) -> None: + super().__init__(status_code=status_code, error=error) + self.headers = headers -- GitLab From 070490d72553c02a98f1846d7b2ba1e9bc830fdd Mon Sep 17 00:00:00 2001 From: Vladimir Komarov Date: Mon, 1 Feb 2021 14:55:05 +0700 Subject: [PATCH 05/22] vak-1501 changed messages exceptions of method /api/login/token --- services/login/exceptions.py | 2 +- services/login/paths/token.py | 15 ++++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/services/login/exceptions.py b/services/login/exceptions.py index fb938f20f0..4310caf97f 100644 --- a/services/login/exceptions.py +++ b/services/login/exceptions.py @@ -19,7 +19,7 @@ class StarletteHTTPException(Exception): def __repr__(self) -> str: class_name = self.__class__.__name__ - return f"{class_name}(status_code={self.status_code!r}, detail={self.error!r})" + return f"{class_name}(status_code={self.status_code!r}, error={self.error!r})" class HTTPException(StarletteHTTPException): diff --git a/services/login/paths/token.py b/services/login/paths/token.py index 34f62119cd..206215f23b 100644 --- a/services/login/paths/token.py +++ b/services/login/paths/token.py @@ -11,13 +11,14 @@ from typing import Optional, Dict import codecs # Third-party modules -from fastapi import APIRouter, Request, HTTPException, Header +from fastapi import APIRouter, Request, Header # NOC modules from noc.config import config from noc.core.comp import smart_text, smart_bytes from ..models.token import TokenRequest, TokenResponse from ..auth import authenticate, get_jwt_token, get_user_from_jwt, revoke_token, is_revoked +from ..exceptions import HTTPException router = APIRouter() @@ -32,11 +33,11 @@ async def token( if req.grant_type == "refresh_token": # Refresh token if is_revoked(req.refresh_token): - raise HTTPException(detail="Token is expired", status_code=HTTPStatus.FORBIDDEN) + raise HTTPException(error="invalid_grant", status_code=HTTPStatus.FORBIDDEN) try: user = get_user_from_jwt(req.refresh_token, audience="refresh") except ValueError as e: - raise HTTPException(detail="Access denied (%s)" % e, status_code=HTTPStatus.FORBIDDEN) + raise HTTPException(error="unauthorized_client (%s)" % e, status_code=HTTPStatus.FORBIDDEN) revoke_token(req.refresh_token) return get_token_response(user) elif req.grant_type == "password": @@ -47,23 +48,23 @@ async def token( schema, data = authorization.split(" ", 1) if schema != "Basic": raise HTTPException( - detail="Basic authorization header required", status_code=HTTPStatus.BAD_REQUEST + error="unsupported_grant_type", status_code=HTTPStatus.BAD_REQUEST ) auth_data = smart_text(codecs.decode(smart_bytes(data), "base64")) if ":" not in auth_data: raise HTTPException( - detail="Invalid basic auth header", status_code=HTTPStatus.BAD_REQUEST + error="invalid_request", status_code=HTTPStatus.BAD_REQUEST ) user, password = auth_data.split(":", 1) auth_req = {"user": user, "password": password, "ip": request.client.host} else: - raise HTTPException(detail="Invalid grant type", status_code=HTTPStatus.BAD_REQUEST) + raise HTTPException(error="unsupported_grant_type", status_code=HTTPStatus.BAD_REQUEST) # Authenticate if auth_req: user = authenticate(auth_req) if user: return get_token_response(user) - raise HTTPException(detail="Access denied", status_code=HTTPStatus.BAD_REQUEST) + raise HTTPException(error="invalid_scope", status_code=HTTPStatus.BAD_REQUEST) def get_token_response(user: str) -> TokenResponse: -- GitLab From 5c03ea8c540263b152e173e3af4b94225a632058 Mon Sep 17 00:00:00 2001 From: Vladimir Komarov Date: Mon, 1 Feb 2021 14:59:14 +0700 Subject: [PATCH 06/22] vak-1501 fixed branch's mistake. --- services/netflowcollector/__init__.py | 0 services/netflowcollector/datastream.py | 17 -- services/netflowcollector/netflowserver.py | 54 ------ services/netflowcollector/service.py | 199 --------------------- services/netflowcollector/sourceconfig.py | 21 --- 5 files changed, 291 deletions(-) delete mode 100644 services/netflowcollector/__init__.py delete mode 100644 services/netflowcollector/datastream.py delete mode 100644 services/netflowcollector/netflowserver.py delete mode 100755 services/netflowcollector/service.py delete mode 100644 services/netflowcollector/sourceconfig.py diff --git a/services/netflowcollector/__init__.py b/services/netflowcollector/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/services/netflowcollector/datastream.py b/services/netflowcollector/datastream.py deleted file mode 100644 index c47561c29b..0000000000 --- a/services/netflowcollector/datastream.py +++ /dev/null @@ -1,17 +0,0 @@ -# ---------------------------------------------------------------------- -# Netflow DataStream client -# ---------------------------------------------------------------------- -# Copyright (C) 2007-2021 The NOC Project -# See LICENSE for details -# ---------------------------------------------------------------------- - -# NOC modules -from noc.core.datastream.client import DataStreamClient - - -class NetflowDataStreamClient(DataStreamClient): - async def on_change(self, data): - await self.service.update_source(data) - - async def on_delete(self, data): - await self.service.delete_source(data["id"]) diff --git a/services/netflowcollector/netflowserver.py b/services/netflowcollector/netflowserver.py deleted file mode 100644 index 071d8ea5b6..0000000000 --- a/services/netflowcollector/netflowserver.py +++ /dev/null @@ -1,54 +0,0 @@ -# --------------------------------------------------------------------- -# Netflow server -# --------------------------------------------------------------------- -# Copyright (C) 2007-2021 The NOC Project -# See LICENSE for details -# --------------------------------------------------------------------- - -# Python modules -import logging -import time -from typing import Tuple - -# NOC modules -from noc.config import config -from noc.core.perf import metrics -from noc.core.ioloop.udpserver import UDPServer -from noc.core.comp import smart_text - -logger = logging.getLogger(__name__) - - -class NetflowServer(UDPServer): - def __init__(self, service): - super().__init__() - self.service = service - - def enable_reuseport(self): - return config.netflowcollector.enable_reuseport - - def enable_freebind(self): - return config.netflowcollector.enable_freebind - - def on_read(self, data: bytes, address: Tuple[str, int]): - metrics["netflow_msg_in"] += 1 - cfg = self.service.lookup_config(address[0]) - if not cfg: - return # Invalid event source - # Convert data to valid UTF8 - data = smart_text(data, errors="ignore") - # Parse priority - priority = 0 - if data.startswith("<"): - idx = data.find(">") - if idx == -1: - return - try: - priority = int(data[1:idx]) - except ValueError: - pass - data = data[idx + 1 :].strip() - # Get timestamp - ts = int(time.time()) - # - self.service.register_message(cfg, ts, data, facility=priority >> 3, severity=priority & 7) diff --git a/services/netflowcollector/service.py b/services/netflowcollector/service.py deleted file mode 100755 index 6fde17ee1d..0000000000 --- a/services/netflowcollector/service.py +++ /dev/null @@ -1,199 +0,0 @@ -#!./bin/python -# --------------------------------------------------------------------- -# Netflow Collector service -# --------------------------------------------------------------------- -# Copyright (C) 2007-2021 The NOC Project -# See LICENSE for details -# --------------------------------------------------------------------- - -# Python modules -import datetime -from collections import defaultdict -import asyncio -from typing import Optional, Dict - -# Third-party modules -import orjson - -# NOC modules -from noc.config import config -from noc.core.error import NOCError -from noc.core.service.tornado import TornadoService -from noc.core.perf import metrics -from noc.services.netflowcollector.netflowserver import NetflowServer -from noc.services.netflowcollector.datastream import NetflowDataStreamClient -from noc.services.netflowcollector.sourceconfig import SourceConfig -from noc.core.ioloop.timers import PeriodicCallback - - -class NetflowCollectorService(TornadoService): - name = "netflowcollector" - leader_group_name = "netflowcollector-%(dc)s-%(node)s" - pooled = True - process_name = "noc-%(name).10s-%(pool).5s" - - def __init__(self): - super().__init__() - self.mappings_callback = None - self.report_invalid_callback = None - self.source_configs = {} # id -> SourceConfig - self.address_configs = {} # address -> SourceConfig - self.invalid_sources = defaultdict(int) # ip -> count - self.pool_partitions: Dict[str, int] = {} - - async def on_activate(self): - # Listen sockets - server = NetflowServer(service=self) - for addr, port in server.iter_listen(config.netflowcollector.listen): - self.logger.info("Starting netflow server at %s:%s", addr, port) - try: - server.listen(port, addr) - except OSError as e: - metrics["error", ("type", "socket_listen_error")] += 1 - self.logger.error("Failed to start netflow server at %s:%s: %s", addr, port, e) - server.start() - # Report invalid sources every 60 seconds - self.logger.info("Stating invalid sources reporting task") - self.report_invalid_callback = PeriodicCallback(self.report_invalid_sources, 60000) - self.report_invalid_callback.start() - # Start tracking changes - asyncio.get_running_loop().create_task(self.get_object_mappings()) - - async def get_pool_partitions(self, pool: str) -> int: - parts = self.pool_partitions.get(pool) - if not parts: - parts = await self.get_stream_partitions("events.%s" % pool) - self.pool_partitions[pool] = parts - return parts - - def lookup_config(self, address: str) -> Optional[SourceConfig]: - """ - Returns object id for given address or None when - unknown source - """ - cfg = self.address_configs.get(address) - if cfg: - return cfg - # Register invalid event source - if self.address_configs: - self.invalid_sources[address] += 1 - metrics["error", ("type", "object_not_found")] += 1 - return None - - def register_message( - self, cfg: SourceConfig, timestamp: int, message: str, facility: int, severity: int - ) -> None: - """ - Spool message to be sent - """ - if cfg.process_events: - # Send to classifier - metrics["events_out"] += 1 - self.publish( - orjson.dumps( - { - "ts": timestamp, - "object": cfg.id, - "data": {"source": "netflow", "collector": config.pool, "message": message}, - } - ), - stream=cfg.stream, - partition=cfg.partition, - ) - if cfg.archive_events and cfg.bi_id: - # Archive message - metrics["events_archived"] += 1 - now = datetime.datetime.now() - ts = now.strftime("%Y-%m-%d %H:%M:%S") - date = ts.split(" ")[0] - self.register_metrics( - "syslog", - [ - { - "date": date, - "ts": ts, - "managed_object": cfg.bi_id, - "facility": facility, - "severity": severity, - "message": message, - } - ], - ) - - async def get_object_mappings(self): - """ - Subscribe and track datastream changes - """ - # Register RPC aliases - client = NetflowDataStreamClient("cfgnetflow", service=self) - # Track stream changes - while True: - self.logger.info("Starting to track object mappings") - try: - await client.query( - limit=config.syslogcollector.ds_limit, - filters=["pool(%s)" % config.pool], - block=1, - ) - except NOCError as e: - self.logger.info("Failed to get object mappings: %s", e) - await asyncio.sleep(1) - - async def report_invalid_sources(self): - """ - Report invalid event sources - """ - if not self.invalid_sources: - return - total = sum(self.invalid_sources[s] for s in self.invalid_sources) - self.logger.info( - "Dropping %d messages with invalid sources: %s", - total, - ", ".join("%s: %s" % (s, self.invalid_sources[s]) for s in self.invalid_sources), - ) - self.invalid_sources = defaultdict(int) - - async def update_source(self, data): - # Get old config - old_cfg = self.source_configs.get(data["id"]) - if old_cfg: - old_addresses = set(old_cfg.addresses) - else: - old_addresses = set() - # Get pool and sharding information - fm_pool = data.get("fm_pool", None) or config.pool - num_partitions = await self.get_pool_partitions(fm_pool) - # Build new config - cfg = SourceConfig( - id=data["id"], - addresses=tuple(data["addresses"]), - bi_id=data.get("bi_id"), # For backward compatibility - process_events=data.get("process_events", True), # For backward compatibility - archive_events=data.get("archive_events", False), - stream="events.%s" % fm_pool, - partition=int(data["id"]) % num_partitions, - ) - new_addresses = set(cfg.addresses) - # Add new addresses, update remaining - for addr in new_addresses: - self.address_configs[addr] = cfg - # Revoke stale addresses - for addr in old_addresses - new_addresses: - del self.address_configs[addr] - # Update configs - self.source_configs[data["id"]] = cfg - # Update metrics - metrics["sources_changed"] += 1 - - async def delete_source(self, id): - cfg = self.source_configs.get(id) - if not cfg: - return - for addr in cfg.addresses: - del self.address_configs[addr] - del self.source_configs[id] - metrics["sources_deleted"] += 1 - - -if __name__ == "__main__": - NetflowCollectorService().start() diff --git a/services/netflowcollector/sourceconfig.py b/services/netflowcollector/sourceconfig.py deleted file mode 100644 index 7ae5a7f483..0000000000 --- a/services/netflowcollector/sourceconfig.py +++ /dev/null @@ -1,21 +0,0 @@ -# ---------------------------------------------------------------------- -# SourceConfig -# ---------------------------------------------------------------------- -# Copyright (C) 2007-2020 The NOC Project -# See LICENSE for details -# ---------------------------------------------------------------------- - -# Python modules -from dataclasses import dataclass -from typing import Tuple - - -@dataclass -class SourceConfig(object): - id: str - addresses: Tuple[str, ...] - bi_id: int - process_events: bool - archive_events: bool - stream: str - partition: int -- GitLab From 19525bfebfd20dff13bffbfe8a629662f745a859 Mon Sep 17 00:00:00 2001 From: Vladimir Komarov Date: Mon, 1 Feb 2021 15:00:52 +0700 Subject: [PATCH 07/22] vak-1501 fixed branch's mistake. --- config.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/config.py b/config.py index aeea6b6513..1085524e35 100644 --- a/config.py +++ b/config.py @@ -550,13 +550,6 @@ class Config(BaseConfig): # DataStream request limit ds_limit = IntParameter(default=1000) - class netflowcollector(ConfigSection): - listen = StringParameter(default="0.0.0.0:2055") - enable_reuseport = BooleanParameter(default=True) - enable_freebind = BooleanParameter(default=False) - # DataStream request limit - ds_limit = IntParameter(default=1000) - class icqsender(ConfigSection): token = SecretParameter() retry_timeout = IntParameter(default=2) -- GitLab From 7d69a3111edd859d1edf53986b297afd28a057bb Mon Sep 17 00:00:00 2001 From: Vladimir Komarov Date: Mon, 1 Feb 2021 15:13:18 +0700 Subject: [PATCH 08/22] vak-1501 fixed black's mistake. --- services/login/paths/token.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/services/login/paths/token.py b/services/login/paths/token.py index 206215f23b..408a942b74 100644 --- a/services/login/paths/token.py +++ b/services/login/paths/token.py @@ -47,14 +47,10 @@ async def token( # CCGrantRequest + Basic auth header schema, data = authorization.split(" ", 1) if schema != "Basic": - raise HTTPException( - error="unsupported_grant_type", status_code=HTTPStatus.BAD_REQUEST - ) + raise HTTPException(error="unsupported_grant_type", status_code=HTTPStatus.BAD_REQUEST) auth_data = smart_text(codecs.decode(smart_bytes(data), "base64")) if ":" not in auth_data: - raise HTTPException( - error="invalid_request", status_code=HTTPStatus.BAD_REQUEST - ) + raise HTTPException(error="invalid_request", status_code=HTTPStatus.BAD_REQUEST) user, password = auth_data.split(":", 1) auth_req = {"user": user, "password": password, "ip": request.client.host} else: -- GitLab From ca315a0601b7682f97efc63257a8b9b378d69e17 Mon Sep 17 00:00:00 2001 From: Vladimir Komarov Date: Mon, 1 Feb 2021 20:07:09 +0700 Subject: [PATCH 09/22] vak-1501 error messages of /api/login/revoke. --- services/login/models/status.py | 6 +++++- services/login/paths/revoke.py | 10 ++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/services/login/models/status.py b/services/login/models/status.py index 4465ddb2a3..4ad696a93a 100644 --- a/services/login/models/status.py +++ b/services/login/models/status.py @@ -12,6 +12,10 @@ from typing import Optional from pydantic import BaseModel -class StatusResponse(BaseModel): +class StatusResponseOk(BaseModel): status: bool message: Optional[str] + + +class StatusResponse(BaseModel): + error: Optional[str] diff --git a/services/login/paths/revoke.py b/services/login/paths/revoke.py index 0e2b62b7d9..55b9a1b567 100644 --- a/services/login/paths/revoke.py +++ b/services/login/paths/revoke.py @@ -11,7 +11,7 @@ from fastapi import APIRouter # NOC modules from ..auth import revoke_token, get_user_from_jwt from ..models.revoke import RevokeRequest -from ..models.status import StatusResponse +from ..models.status import StatusResponse, StatusResponseOk router = APIRouter() @@ -22,12 +22,14 @@ async def revoke(req: RevokeRequest): try: get_user_from_jwt(req.access_token, audience="auth") except ValueError: - return StatusResponse(status=False, message="Invalid access token") + return StatusResponse(error="unauthorized_client") revoke_token(req.access_token) if req.refresh_token: try: get_user_from_jwt(req.refresh_token, audience="auth") except ValueError: - return StatusResponse(status=False, message="Invalid refresh token") + return StatusResponse(error="invalid_request") revoke_token(req.refresh_token) - return StatusResponse(status=True, message="Ok") + if not req.access_token and not req.refresh_token: + return StatusResponse(error="invalid_request") + return StatusResponseOk(status=True, message="Ok") -- GitLab From 12af0bb78597ff9bcd43bcbedd48f4141f0a5919 Mon Sep 17 00:00:00 2001 From: Vladimir Komarov Date: Mon, 1 Feb 2021 21:11:07 +0700 Subject: [PATCH 10/22] vak-1501 fixed /api/login/revoke. --- services/login/models/status.py | 2 +- services/login/paths/revoke.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/services/login/models/status.py b/services/login/models/status.py index 4ad696a93a..e87dada9fc 100644 --- a/services/login/models/status.py +++ b/services/login/models/status.py @@ -18,4 +18,4 @@ class StatusResponseOk(BaseModel): class StatusResponse(BaseModel): - error: Optional[str] + error: Optional[str] = None diff --git a/services/login/paths/revoke.py b/services/login/paths/revoke.py index 55b9a1b567..1083104bc5 100644 --- a/services/login/paths/revoke.py +++ b/services/login/paths/revoke.py @@ -16,7 +16,7 @@ from ..models.status import StatusResponse, StatusResponseOk router = APIRouter() -@router.post("/api/login/revoke", response_model=StatusResponse, tags=["login"]) +@router.post("/api/login/revoke", tags=["login"]) async def revoke(req: RevokeRequest): if req.access_token: try: -- GitLab From bc1c56e6faa7d7dbf8f11d640981e3ccd7cea3c1 Mon Sep 17 00:00:00 2001 From: Vladimir Komarov Date: Tue, 2 Feb 2021 20:14:13 +0700 Subject: [PATCH 11/22] vak-1501 changed exception_handlers in FastAPIService. --- core/service/fastapi.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/core/service/fastapi.py b/core/service/fastapi.py index 2b65ab8485..39b3ceeb34 100644 --- a/core/service/fastapi.py +++ b/core/service/fastapi.py @@ -12,8 +12,10 @@ from typing import Optional, Tuple, Dict # Third-party modules import uvicorn from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware -from starlette.responses import Response, PlainTextResponse +from starlette.responses import Response, PlainTextResponse, JSONResponse from fastapi import FastAPI +from fastapi.exceptions import RequestValidationError +from fastapi.encoders import jsonable_encoder # NOC modules from noc.core.version import version @@ -44,6 +46,18 @@ class FastAPIService(BaseService): error_report(logger=self.logger) return PlainTextResponse("Internal Server Error", status_code=500) + async def request_validation_error_handler(self, request, exc) -> Response: + """ + Handle request validation and customize response + :param request: + :param exc: + :return: + """ + return JSONResponse( + status_code=400, + content={"error": "bad_request", "error_detail": jsonable_encoder(exc.errors())}, + ) + async def init_api(self): # Build tags docs openapi_tags = [] @@ -60,7 +74,10 @@ class FastAPIService(BaseService): docs_url="/api/%s/docs" % self.name, redoc_url="/api/%s/redoc" % self.name, openapi_tags=openapi_tags, - exception_handlers={Exception: self.error_handler}, + exception_handlers={ + Exception: self.error_handler, + RequestValidationError: self.request_validation_error_handler, + }, ) self.app.add_middleware(ProxyHeadersMiddleware, trusted_hosts="*") self.app.add_middleware(LoggingMiddleware, logger=PrefixLoggerAdapter(self.logger, "api")) -- GitLab From 799b3c0d3db29df73eb362efe851613a13b08816 Mon Sep 17 00:00:00 2001 From: Vladimir Komarov Date: Tue, 2 Feb 2021 21:34:47 +0700 Subject: [PATCH 12/22] vak-1501 deleted exceptions module. --- services/login/exceptions.py | 33 --------------------------------- services/login/paths/token.py | 3 +-- 2 files changed, 1 insertion(+), 35 deletions(-) delete mode 100644 services/login/exceptions.py diff --git a/services/login/exceptions.py b/services/login/exceptions.py deleted file mode 100644 index 4310caf97f..0000000000 --- a/services/login/exceptions.py +++ /dev/null @@ -1,33 +0,0 @@ -# --------------------------------------------------------------------- -# Additional exceptions -# --------------------------------------------------------------------- -# Copyright (C) 2021 The NOC Project -# See LICENSE for details -# --------------------------------------------------------------------- - -import http - -from typing import Any, Dict, Optional - - -class StarletteHTTPException(Exception): - def __init__(self, status_code: int, error: str = None) -> None: - if error is None: - error = http.HTTPStatus(status_code).phrase - self.status_code = status_code - self.error = error - - def __repr__(self) -> str: - class_name = self.__class__.__name__ - return f"{class_name}(status_code={self.status_code!r}, error={self.error!r})" - - -class HTTPException(StarletteHTTPException): - def __init__( - self, - status_code: int, - error: Any = None, - headers: Optional[Dict[str, Any]] = None, - ) -> None: - super().__init__(status_code=status_code, error=error) - self.headers = headers diff --git a/services/login/paths/token.py b/services/login/paths/token.py index 408a942b74..adc8a8cab7 100644 --- a/services/login/paths/token.py +++ b/services/login/paths/token.py @@ -11,14 +11,13 @@ from typing import Optional, Dict import codecs # Third-party modules -from fastapi import APIRouter, Request, Header +from fastapi import APIRouter, Request, HTTPException, Header # NOC modules from noc.config import config from noc.core.comp import smart_text, smart_bytes from ..models.token import TokenRequest, TokenResponse from ..auth import authenticate, get_jwt_token, get_user_from_jwt, revoke_token, is_revoked -from ..exceptions import HTTPException router = APIRouter() -- GitLab From ec8328a8249fe23e3578b0fdcab579cedec15f2a Mon Sep 17 00:00:00 2001 From: Vladimir Komarov Date: Tue, 2 Feb 2021 21:43:37 +0700 Subject: [PATCH 13/22] vak-1501 fixed black's mistake. --- services/login/paths/token.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/services/login/paths/token.py b/services/login/paths/token.py index adc8a8cab7..3e87909114 100644 --- a/services/login/paths/token.py +++ b/services/login/paths/token.py @@ -36,7 +36,9 @@ async def token( try: user = get_user_from_jwt(req.refresh_token, audience="refresh") except ValueError as e: - raise HTTPException(error="unauthorized_client (%s)" % e, status_code=HTTPStatus.FORBIDDEN) + raise HTTPException( + error="unauthorized_client (%s)" % e, status_code=HTTPStatus.FORBIDDEN + ) revoke_token(req.refresh_token) return get_token_response(user) elif req.grant_type == "password": -- GitLab From 19e65895f5aeefe5ed025b10ee9d924369d1947a Mon Sep 17 00:00:00 2001 From: Vladimir Komarov Date: Wed, 3 Feb 2021 15:23:06 +0700 Subject: [PATCH 14/22] vak-1501 recreated exceptions module. --- core/service/fastapi.py | 2 +- services/login/exceptions.py | 33 +++++++++++++++++++++++++++++++++ services/login/paths/token.py | 3 ++- 3 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 services/login/exceptions.py diff --git a/core/service/fastapi.py b/core/service/fastapi.py index 39b3ceeb34..c76008f527 100644 --- a/core/service/fastapi.py +++ b/core/service/fastapi.py @@ -55,7 +55,7 @@ class FastAPIService(BaseService): """ return JSONResponse( status_code=400, - content={"error": "bad_request", "error_detail": jsonable_encoder(exc.errors())}, + content={"error": "bad_request"}, ) async def init_api(self): diff --git a/services/login/exceptions.py b/services/login/exceptions.py new file mode 100644 index 0000000000..4310caf97f --- /dev/null +++ b/services/login/exceptions.py @@ -0,0 +1,33 @@ +# --------------------------------------------------------------------- +# Additional exceptions +# --------------------------------------------------------------------- +# Copyright (C) 2021 The NOC Project +# See LICENSE for details +# --------------------------------------------------------------------- + +import http + +from typing import Any, Dict, Optional + + +class StarletteHTTPException(Exception): + def __init__(self, status_code: int, error: str = None) -> None: + if error is None: + error = http.HTTPStatus(status_code).phrase + self.status_code = status_code + self.error = error + + def __repr__(self) -> str: + class_name = self.__class__.__name__ + return f"{class_name}(status_code={self.status_code!r}, error={self.error!r})" + + +class HTTPException(StarletteHTTPException): + def __init__( + self, + status_code: int, + error: Any = None, + headers: Optional[Dict[str, Any]] = None, + ) -> None: + super().__init__(status_code=status_code, error=error) + self.headers = headers diff --git a/services/login/paths/token.py b/services/login/paths/token.py index 3e87909114..f79b2d6a41 100644 --- a/services/login/paths/token.py +++ b/services/login/paths/token.py @@ -11,13 +11,14 @@ from typing import Optional, Dict import codecs # Third-party modules -from fastapi import APIRouter, Request, HTTPException, Header +from fastapi import APIRouter, Request, Header # NOC modules from noc.config import config from noc.core.comp import smart_text, smart_bytes from ..models.token import TokenRequest, TokenResponse from ..auth import authenticate, get_jwt_token, get_user_from_jwt, revoke_token, is_revoked +from ..exceptions import HTTPException router = APIRouter() -- GitLab From d91dd9823316334bdeb36209265c7aaa1b8f4160 Mon Sep 17 00:00:00 2001 From: Vladimir Komarov Date: Wed, 3 Feb 2021 15:37:57 +0700 Subject: [PATCH 15/22] vak-1501 fixed flake's mistake. --- core/service/fastapi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/service/fastapi.py b/core/service/fastapi.py index c76008f527..39b3ceeb34 100644 --- a/core/service/fastapi.py +++ b/core/service/fastapi.py @@ -55,7 +55,7 @@ class FastAPIService(BaseService): """ return JSONResponse( status_code=400, - content={"error": "bad_request"}, + content={"error": "bad_request", "error_detail": jsonable_encoder(exc.errors())}, ) async def init_api(self): -- GitLab From aa5e45c047ef1cded9a93b622efa1bbd75d28bf3 Mon Sep 17 00:00:00 2001 From: Vladimir Komarov Date: Wed, 3 Feb 2021 20:08:46 +0700 Subject: [PATCH 16/22] vak-1501 changed raise HTTPException -> return JSONResponse. --- core/service/fastapi.py | 2 +- services/login/exceptions.py | 33 --------------------------------- services/login/paths/token.py | 16 ++++++++-------- 3 files changed, 9 insertions(+), 42 deletions(-) delete mode 100644 services/login/exceptions.py diff --git a/core/service/fastapi.py b/core/service/fastapi.py index 39b3ceeb34..80e485c2f3 100644 --- a/core/service/fastapi.py +++ b/core/service/fastapi.py @@ -55,7 +55,7 @@ class FastAPIService(BaseService): """ return JSONResponse( status_code=400, - content={"error": "bad_request", "error_detail": jsonable_encoder(exc.errors())}, + content={"error": "invalid_request"}, ) async def init_api(self): diff --git a/services/login/exceptions.py b/services/login/exceptions.py deleted file mode 100644 index 4310caf97f..0000000000 --- a/services/login/exceptions.py +++ /dev/null @@ -1,33 +0,0 @@ -# --------------------------------------------------------------------- -# Additional exceptions -# --------------------------------------------------------------------- -# Copyright (C) 2021 The NOC Project -# See LICENSE for details -# --------------------------------------------------------------------- - -import http - -from typing import Any, Dict, Optional - - -class StarletteHTTPException(Exception): - def __init__(self, status_code: int, error: str = None) -> None: - if error is None: - error = http.HTTPStatus(status_code).phrase - self.status_code = status_code - self.error = error - - def __repr__(self) -> str: - class_name = self.__class__.__name__ - return f"{class_name}(status_code={self.status_code!r}, error={self.error!r})" - - -class HTTPException(StarletteHTTPException): - def __init__( - self, - status_code: int, - error: Any = None, - headers: Optional[Dict[str, Any]] = None, - ) -> None: - super().__init__(status_code=status_code, error=error) - self.headers = headers diff --git a/services/login/paths/token.py b/services/login/paths/token.py index f79b2d6a41..abdc739cc8 100644 --- a/services/login/paths/token.py +++ b/services/login/paths/token.py @@ -12,13 +12,13 @@ import codecs # Third-party modules from fastapi import APIRouter, Request, Header +from starlette.responses import JSONResponse # NOC modules from noc.config import config from noc.core.comp import smart_text, smart_bytes from ..models.token import TokenRequest, TokenResponse from ..auth import authenticate, get_jwt_token, get_user_from_jwt, revoke_token, is_revoked -from ..exceptions import HTTPException router = APIRouter() @@ -33,12 +33,12 @@ async def token( if req.grant_type == "refresh_token": # Refresh token if is_revoked(req.refresh_token): - raise HTTPException(error="invalid_grant", status_code=HTTPStatus.FORBIDDEN) + return JSONResponse(content={"error": "invalid_grant"}, status_code=HTTPStatus.FORBIDDEN) try: user = get_user_from_jwt(req.refresh_token, audience="refresh") except ValueError as e: - raise HTTPException( - error="unauthorized_client (%s)" % e, status_code=HTTPStatus.FORBIDDEN + return JSONResponse( + content={"error": "unauthorized_client (%s)" % e}, status_code=HTTPStatus.FORBIDDEN ) revoke_token(req.refresh_token) return get_token_response(user) @@ -49,20 +49,20 @@ async def token( # CCGrantRequest + Basic auth header schema, data = authorization.split(" ", 1) if schema != "Basic": - raise HTTPException(error="unsupported_grant_type", status_code=HTTPStatus.BAD_REQUEST) + return JSONResponse(content={"error": "unsupported_grant_type"}, status_code=HTTPStatus.BAD_REQUEST) auth_data = smart_text(codecs.decode(smart_bytes(data), "base64")) if ":" not in auth_data: - raise HTTPException(error="invalid_request", status_code=HTTPStatus.BAD_REQUEST) + return JSONResponse(content={"error": "invalid_request"}, status_code=HTTPStatus.BAD_REQUEST) user, password = auth_data.split(":", 1) auth_req = {"user": user, "password": password, "ip": request.client.host} else: - raise HTTPException(error="unsupported_grant_type", status_code=HTTPStatus.BAD_REQUEST) + return JSONResponse(content={"error": "unsupported_grant_type"}, status_code=HTTPStatus.BAD_REQUEST) # Authenticate if auth_req: user = authenticate(auth_req) if user: return get_token_response(user) - raise HTTPException(error="invalid_scope", status_code=HTTPStatus.BAD_REQUEST) + return JSONResponse(content={"error": "invalid_scope"}, status_code=HTTPStatus.BAD_REQUEST) def get_token_response(user: str) -> TokenResponse: -- GitLab From b4ce9b5f59051d675a43dedb788788515d85dfb3 Mon Sep 17 00:00:00 2001 From: Vladimir Komarov Date: Wed, 3 Feb 2021 20:18:02 +0700 Subject: [PATCH 17/22] vak-1501 fixed flake's mistake. --- core/service/fastapi.py | 1 - services/login/paths/token.py | 12 +++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/core/service/fastapi.py b/core/service/fastapi.py index 80e485c2f3..9f4525efa8 100644 --- a/core/service/fastapi.py +++ b/core/service/fastapi.py @@ -15,7 +15,6 @@ from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware from starlette.responses import Response, PlainTextResponse, JSONResponse from fastapi import FastAPI from fastapi.exceptions import RequestValidationError -from fastapi.encoders import jsonable_encoder # NOC modules from noc.core.version import version diff --git a/services/login/paths/token.py b/services/login/paths/token.py index abdc739cc8..bbdd260317 100644 --- a/services/login/paths/token.py +++ b/services/login/paths/token.py @@ -33,7 +33,9 @@ async def token( if req.grant_type == "refresh_token": # Refresh token if is_revoked(req.refresh_token): - return JSONResponse(content={"error": "invalid_grant"}, status_code=HTTPStatus.FORBIDDEN) + return JSONResponse( + content={"error": "invalid_grant"}, status_code=HTTPStatus.FORBIDDEN + ) try: user = get_user_from_jwt(req.refresh_token, audience="refresh") except ValueError as e: @@ -49,10 +51,14 @@ async def token( # CCGrantRequest + Basic auth header schema, data = authorization.split(" ", 1) if schema != "Basic": - return JSONResponse(content={"error": "unsupported_grant_type"}, status_code=HTTPStatus.BAD_REQUEST) + return JSONResponse( + content={"error": "unsupported_grant_type"}, status_code=HTTPStatus.BAD_REQUEST + ) auth_data = smart_text(codecs.decode(smart_bytes(data), "base64")) if ":" not in auth_data: - return JSONResponse(content={"error": "invalid_request"}, status_code=HTTPStatus.BAD_REQUEST) + return JSONResponse( + content={"error": "invalid_request"}, status_code=HTTPStatus.BAD_REQUEST + ) user, password = auth_data.split(":", 1) auth_req = {"user": user, "password": password, "ip": request.client.host} else: -- GitLab From 1e5211a45071f6ffb7876d7608f81864c85d6320 Mon Sep 17 00:00:00 2001 From: Vladimir Komarov Date: Wed, 3 Feb 2021 20:23:05 +0700 Subject: [PATCH 18/22] vak-1501 fixed black's mistake. --- services/login/paths/token.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/services/login/paths/token.py b/services/login/paths/token.py index bbdd260317..45b2a8aa63 100644 --- a/services/login/paths/token.py +++ b/services/login/paths/token.py @@ -62,7 +62,9 @@ async def token( user, password = auth_data.split(":", 1) auth_req = {"user": user, "password": password, "ip": request.client.host} else: - return JSONResponse(content={"error": "unsupported_grant_type"}, status_code=HTTPStatus.BAD_REQUEST) + return JSONResponse( + content={"error": "unsupported_grant_type"}, status_code=HTTPStatus.BAD_REQUEST + ) # Authenticate if auth_req: user = authenticate(auth_req) -- GitLab From 72e759269bc83d260af76f58f3f06001d1c1b0a8 Mon Sep 17 00:00:00 2001 From: Vladimir Komarov Date: Thu, 4 Feb 2021 10:38:16 +0700 Subject: [PATCH 19/22] vak-1501 changed StatusResponse -> StatusResponseError. --- services/login/models/status.py | 2 +- services/login/paths/revoke.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/services/login/models/status.py b/services/login/models/status.py index e87dada9fc..aae231bdda 100644 --- a/services/login/models/status.py +++ b/services/login/models/status.py @@ -17,5 +17,5 @@ class StatusResponseOk(BaseModel): message: Optional[str] -class StatusResponse(BaseModel): +class StatusResponseError(BaseModel): error: Optional[str] = None diff --git a/services/login/paths/revoke.py b/services/login/paths/revoke.py index 1083104bc5..a8d1eb5804 100644 --- a/services/login/paths/revoke.py +++ b/services/login/paths/revoke.py @@ -11,7 +11,7 @@ from fastapi import APIRouter # NOC modules from ..auth import revoke_token, get_user_from_jwt from ..models.revoke import RevokeRequest -from ..models.status import StatusResponse, StatusResponseOk +from ..models.status import StatusResponseError, StatusResponseOk router = APIRouter() @@ -22,14 +22,14 @@ async def revoke(req: RevokeRequest): try: get_user_from_jwt(req.access_token, audience="auth") except ValueError: - return StatusResponse(error="unauthorized_client") + return StatusResponseError(error="unauthorized_client") revoke_token(req.access_token) if req.refresh_token: try: get_user_from_jwt(req.refresh_token, audience="auth") except ValueError: - return StatusResponse(error="invalid_request") + return StatusResponseError(error="invalid_request") revoke_token(req.refresh_token) if not req.access_token and not req.refresh_token: - return StatusResponse(error="invalid_request") + return StatusResponseError(error="invalid_request") return StatusResponseOk(status=True, message="Ok") -- GitLab From 8508e8d02faf2d44f84efd241c78cedc5494d922 Mon Sep 17 00:00:00 2001 From: Vladimir Komarov Date: Thu, 4 Feb 2021 14:49:28 +0700 Subject: [PATCH 20/22] vak-1501 repaired tests. --- services/login/paths/change_credentials.py | 8 ++++---- services/login/paths/login.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/services/login/paths/change_credentials.py b/services/login/paths/change_credentials.py index 620c7d000a..4e73f7141a 100644 --- a/services/login/paths/change_credentials.py +++ b/services/login/paths/change_credentials.py @@ -11,13 +11,13 @@ from fastapi import APIRouter # NOC modules from ..auth import change_credentials as _change_credentials from ..models.changecredentials import ChangeCredentialsRequest -from ..models.status import StatusResponse +from ..models.status import StatusResponseOk router = APIRouter() @router.put( - "/api/login/change_credentials", response_model=StatusResponse, tags=["login", "ext-ui"] + "/api/login/change_credentials", response_model=StatusResponseOk, tags=["login", "ext-ui"] ) async def change_credentials(req: ChangeCredentialsRequest): creds = { @@ -26,5 +26,5 @@ async def change_credentials(req: ChangeCredentialsRequest): "new_password": req.new_password, } if _change_credentials(creds): - return StatusResponse(status=True) - return StatusResponse(status=False, message="Failed to change credentials") + return StatusResponseOk(status=True) + return StatusResponseOk(status=False, message="Failed to change credentials") diff --git a/services/login/paths/login.py b/services/login/paths/login.py index 58ee90f996..63b530a663 100644 --- a/services/login/paths/login.py +++ b/services/login/paths/login.py @@ -11,13 +11,13 @@ from fastapi.responses import ORJSONResponse # NOC modules from ..models.login import LoginRequest -from ..models.status import StatusResponse +from ..models.status import StatusResponseOk from ..auth import authenticate, set_jwt_cookie router = APIRouter() -@router.post("/api/login/login", response_model=StatusResponse, tags=["login", "ext-ui"]) +@router.post("/api/login/login", response_model=StatusResponseOk, tags=["login", "ext-ui"]) async def login(request: Request, creds: LoginRequest): auth_req = {"user": creds.user, "password": creds.password, "ip": request.client.host} user = authenticate(auth_req) @@ -25,4 +25,4 @@ async def login(request: Request, creds: LoginRequest): response = ORJSONResponse({"status": True}) set_jwt_cookie(response, user) return response - return StatusResponse(status=False, message="Authentication failed") + return StatusResponseOk(status=False, message="Authentication failed") -- GitLab From 578490f6549d6accf75ebbc2435ba16cc2bafc21 Mon Sep 17 00:00:00 2001 From: Vladimir Komarov Date: Thu, 4 Feb 2021 15:03:19 +0700 Subject: [PATCH 21/22] vak-1501 renamed model from StatusResponseOk to StatusResponse. --- services/login/models/status.py | 2 +- services/login/paths/change_credentials.py | 8 ++++---- services/login/paths/login.py | 6 +++--- services/login/paths/revoke.py | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/services/login/models/status.py b/services/login/models/status.py index aae231bdda..1983cbdfd1 100644 --- a/services/login/models/status.py +++ b/services/login/models/status.py @@ -12,7 +12,7 @@ from typing import Optional from pydantic import BaseModel -class StatusResponseOk(BaseModel): +class StatusResponse(BaseModel): status: bool message: Optional[str] diff --git a/services/login/paths/change_credentials.py b/services/login/paths/change_credentials.py index 4e73f7141a..620c7d000a 100644 --- a/services/login/paths/change_credentials.py +++ b/services/login/paths/change_credentials.py @@ -11,13 +11,13 @@ from fastapi import APIRouter # NOC modules from ..auth import change_credentials as _change_credentials from ..models.changecredentials import ChangeCredentialsRequest -from ..models.status import StatusResponseOk +from ..models.status import StatusResponse router = APIRouter() @router.put( - "/api/login/change_credentials", response_model=StatusResponseOk, tags=["login", "ext-ui"] + "/api/login/change_credentials", response_model=StatusResponse, tags=["login", "ext-ui"] ) async def change_credentials(req: ChangeCredentialsRequest): creds = { @@ -26,5 +26,5 @@ async def change_credentials(req: ChangeCredentialsRequest): "new_password": req.new_password, } if _change_credentials(creds): - return StatusResponseOk(status=True) - return StatusResponseOk(status=False, message="Failed to change credentials") + return StatusResponse(status=True) + return StatusResponse(status=False, message="Failed to change credentials") diff --git a/services/login/paths/login.py b/services/login/paths/login.py index 63b530a663..58ee90f996 100644 --- a/services/login/paths/login.py +++ b/services/login/paths/login.py @@ -11,13 +11,13 @@ from fastapi.responses import ORJSONResponse # NOC modules from ..models.login import LoginRequest -from ..models.status import StatusResponseOk +from ..models.status import StatusResponse from ..auth import authenticate, set_jwt_cookie router = APIRouter() -@router.post("/api/login/login", response_model=StatusResponseOk, tags=["login", "ext-ui"]) +@router.post("/api/login/login", response_model=StatusResponse, tags=["login", "ext-ui"]) async def login(request: Request, creds: LoginRequest): auth_req = {"user": creds.user, "password": creds.password, "ip": request.client.host} user = authenticate(auth_req) @@ -25,4 +25,4 @@ async def login(request: Request, creds: LoginRequest): response = ORJSONResponse({"status": True}) set_jwt_cookie(response, user) return response - return StatusResponseOk(status=False, message="Authentication failed") + return StatusResponse(status=False, message="Authentication failed") diff --git a/services/login/paths/revoke.py b/services/login/paths/revoke.py index a8d1eb5804..7d95ea4767 100644 --- a/services/login/paths/revoke.py +++ b/services/login/paths/revoke.py @@ -11,7 +11,7 @@ from fastapi import APIRouter # NOC modules from ..auth import revoke_token, get_user_from_jwt from ..models.revoke import RevokeRequest -from ..models.status import StatusResponseError, StatusResponseOk +from ..models.status import StatusResponseError, StatusResponse router = APIRouter() @@ -32,4 +32,4 @@ async def revoke(req: RevokeRequest): revoke_token(req.refresh_token) if not req.access_token and not req.refresh_token: return StatusResponseError(error="invalid_request") - return StatusResponseOk(status=True, message="Ok") + return StatusResponse(status=True, message="Ok") -- GitLab From 377a0ebc082cc679769fb3c8f92bd0fa2ec6af57 Mon Sep 17 00:00:00 2001 From: Vladimir Komarov Date: Thu, 11 Feb 2021 14:43:27 +0700 Subject: [PATCH 22/22] vak-1501 added error_description into error messages. --- services/login/models/status.py | 1 + services/login/paths/revoke.py | 12 +++++++++--- services/login/paths/token.py | 29 +++++++++++++++++++++++------ 3 files changed, 33 insertions(+), 9 deletions(-) diff --git a/services/login/models/status.py b/services/login/models/status.py index 1983cbdfd1..540480f09b 100644 --- a/services/login/models/status.py +++ b/services/login/models/status.py @@ -19,3 +19,4 @@ class StatusResponse(BaseModel): class StatusResponseError(BaseModel): error: Optional[str] = None + error_description: Optional[str] = None diff --git a/services/login/paths/revoke.py b/services/login/paths/revoke.py index 7d95ea4767..8c47badf24 100644 --- a/services/login/paths/revoke.py +++ b/services/login/paths/revoke.py @@ -22,14 +22,20 @@ async def revoke(req: RevokeRequest): try: get_user_from_jwt(req.access_token, audience="auth") except ValueError: - return StatusResponseError(error="unauthorized_client") + return StatusResponseError( + error="unauthorized_client", error_description="Invalid access token" + ) revoke_token(req.access_token) if req.refresh_token: try: get_user_from_jwt(req.refresh_token, audience="auth") except ValueError: - return StatusResponseError(error="invalid_request") + return StatusResponseError( + error="invalid_request", error_description="Invalid refresh token" + ) revoke_token(req.refresh_token) if not req.access_token and not req.refresh_token: - return StatusResponseError(error="invalid_request") + return StatusResponseError( + error="invalid_request", error_description="Invalid refresh token" + ) return StatusResponse(status=True, message="Ok") diff --git a/services/login/paths/token.py b/services/login/paths/token.py index 45b2a8aa63..4d846adafb 100644 --- a/services/login/paths/token.py +++ b/services/login/paths/token.py @@ -34,13 +34,18 @@ async def token( # Refresh token if is_revoked(req.refresh_token): return JSONResponse( - content={"error": "invalid_grant"}, status_code=HTTPStatus.FORBIDDEN + content={"error": "invalid_grant", "error_description": "Token is expired"}, + status_code=HTTPStatus.FORBIDDEN, ) try: user = get_user_from_jwt(req.refresh_token, audience="refresh") except ValueError as e: return JSONResponse( - content={"error": "unauthorized_client (%s)" % e}, status_code=HTTPStatus.FORBIDDEN + content={ + "error": "unauthorized_client", + "error_description": "Access denied (%s)" % e, + }, + status_code=HTTPStatus.FORBIDDEN, ) revoke_token(req.refresh_token) return get_token_response(user) @@ -52,25 +57,37 @@ async def token( schema, data = authorization.split(" ", 1) if schema != "Basic": return JSONResponse( - content={"error": "unsupported_grant_type"}, status_code=HTTPStatus.BAD_REQUEST + content={ + "error": "unsupported_grant_type", + "error_description": "Basic authorization header required", + }, + status_code=HTTPStatus.BAD_REQUEST, ) auth_data = smart_text(codecs.decode(smart_bytes(data), "base64")) if ":" not in auth_data: return JSONResponse( - content={"error": "invalid_request"}, status_code=HTTPStatus.BAD_REQUEST + content={ + "error": "invalid_request", + "error_description": "Invalid basic auth header", + }, + status_code=HTTPStatus.BAD_REQUEST, ) user, password = auth_data.split(":", 1) auth_req = {"user": user, "password": password, "ip": request.client.host} else: return JSONResponse( - content={"error": "unsupported_grant_type"}, status_code=HTTPStatus.BAD_REQUEST + content={"error": "unsupported_grant_type", "error_description": "Invalid grant type"}, + status_code=HTTPStatus.BAD_REQUEST, ) # Authenticate if auth_req: user = authenticate(auth_req) if user: return get_token_response(user) - return JSONResponse(content={"error": "invalid_scope"}, status_code=HTTPStatus.BAD_REQUEST) + return JSONResponse( + content={"error": "invalid_scope", "error_description": "Access denied"}, + status_code=HTTPStatus.BAD_REQUEST, + ) def get_token_response(user: str) -> TokenResponse: -- GitLab