Commit dddd2290 authored by Dmitry Volodin's avatar Dmitry Volodin Committed by Andrey Vertiprahov
Browse files

Replace ujson with orjson

parent 2ed0743d
......@@ -15,7 +15,7 @@ import codecs
# Third-party modules
import uuid
import yaml
import ujson
import orjson
from fs import open_fs
# NOC modules
......@@ -490,7 +490,7 @@ class Command(BaseCommand):
"input": {},
"result": result,
}
data = bz2.compress(ujson.dumps(tc))
data = bz2.compress(orjson.dumps(tc))
#
rn = os.path.join(test_path, "%04d.%s.json.bz2" % (n, test["script"]))
self.print("[%04d] Writing %s" % (n, rn))
......@@ -642,7 +642,7 @@ class Command(BaseCommand):
def parse_json(j):
try:
return ujson.loads(j)
return orjson.loads(j)
except ValueError as e:
self.die("Failed to parse JSON: %s" % e)
......
......@@ -13,7 +13,7 @@ import time
from pymongo.errors import OperationFailure
# Third-party modules
import ujson
import orjson
# NOC modules
from noc.core.management.base import BaseCommand
......@@ -176,7 +176,7 @@ class Command(BaseCommand):
path = os.path.join(self.data_prefix, fn)
# Read data
with gzip.open(path, "rb") as f:
data = [ujson.loads(line) for line in f.read().splitlines() if line]
data = [orjson.loads(line) for line in f.read().splitlines() if line]
table = fn.split("-", 1)[0]
sharder = Sharder(table)
sharder.feed(data)
......
......@@ -10,7 +10,7 @@ import os
import argparse
# Third-party modules
import ujson
import orjson
# NOC modules
from noc.core.management.base import BaseCommand
......@@ -96,7 +96,7 @@ class Command(BaseCommand):
if not os.path.isfile(fp):
self.die("File not found: %s" % fp)
with open(fp) as f:
data = ujson.load(f)
data = orjson.loads(f.read())
try:
Collection.install(data)
if load:
......
# ----------------------------------------------------------------------
# ./noc crashinfo
# ----------------------------------------------------------------------
# Copyright (C) 2007-2019 The NOC Project
# Copyright (C) 2007-2020 The NOC Project
# See LICENSE for details
# ----------------------------------------------------------------------
......@@ -14,7 +14,7 @@ import stat
import re
# Third-party modules
import ujson
import orjson
# NOC modules
from noc.core.management.base import BaseCommand
......@@ -57,7 +57,7 @@ class Command(BaseCommand):
ts = os.stat(path)[stat.ST_MTIME]
t = datetime.datetime.fromtimestamp(ts)
with open(path) as f:
data = ujson.load(f)
data = orjson.loads(f.read())
service = data["process"]
if service.startswith("services/") and service.endswith("/service.py"):
service = service[9:-11]
......@@ -82,9 +82,16 @@ class Command(BaseCommand):
]
fs = "%s %36s %19s %-29s %-s\n"
self.stdout.write(fs % ("N", "UUID", "Time", "Service", "Exception"))
for l in sorted(fl, key=operator.itemgetter("time"), reverse=True):
for line in sorted(fl, key=operator.itemgetter("time"), reverse=True):
self.stdout.write(
fs % (l["status"], l["uuid"], l["time"].isoformat(), l["service"], l["exception"])
fs
% (
line["status"],
line["uuid"],
line["time"].isoformat(),
line["service"],
line["exception"],
)
)
def handle_view(self, view_uuids, *args, **options):
......@@ -97,7 +104,7 @@ class Command(BaseCommand):
else:
continue
with open(path) as f:
data = ujson.load(f)
data = orjson.loads(f.read())
self.stdout.write(data["traceback"])
self.stdout.write("\n\n")
......
......@@ -12,7 +12,7 @@ import itertools
import logging
# Third-party modules
import ujson
import orjson
# NOC modules
from noc.core.management.base import BaseCommand
......@@ -20,6 +20,7 @@ from noc.core.datastream.loader import loader
from noc.core.mongo.connection import connect
from noc.models import get_model
from noc.models import is_document
from noc.core.comp import smart_text
BATCH_SIZE = 20000
......@@ -96,7 +97,10 @@ class Command(BaseCommand):
model_id = self.MODELS.get(datastream)
if not model_id:
self.die("Unsupported datastream")
model = get_model(model_id)
if isinstance(model_id, tuple):
model = tuple(get_model(mid) for mid in model_id)
else:
model = get_model(model_id)
if not model:
self.die("Invalid model")
return model
......@@ -167,8 +171,8 @@ class Command(BaseCommand):
self.print(
"===[id: %s, change id: %s, time: %s]================" % (obj_id, change_id, gt)
)
d = ujson.loads(data)
self.print(ujson.dumps(d, indent=2))
d = orjson.loads(data)
self.print(smart_text(orjson.dumps(d, option=orjson.OPT_INDENT_2)))
if __name__ == "__main__":
......
# ----------------------------------------------------------------------
# Extract/Transfer/Load commands
# ----------------------------------------------------------------------
# Copyright (C) 2007-2019 The NOC Project
# Copyright (C) 2007-2020 The NOC Project
# See LICENSE for details
# ----------------------------------------------------------------------
......@@ -10,7 +10,7 @@ import argparse
# Third-party modules
import yaml
import ujson
import orjson
# NOC modules
from noc.core.mongo.connection import connect
......@@ -101,31 +101,31 @@ class Command(BaseCommand):
control_dict = {}
if options["control_dict"]:
try:
control_dict = ujson.loads(options["control_dict"])
control_dict = orjson.loads(options["control_dict"])
except ValueError as e:
self.die("Failed to parse JSON: %s in %s" % (e, options["control_dict"]))
except TypeError as e:
self.die("Failed to parse JSON: %s in %s" % (e, options["control_dict"]))
chain = remote_system.get_loader_chain()
for l in chain:
if diffs and l.name not in diffs:
for ldr in chain:
if diffs and ldr.name not in diffs:
continue
if summary:
i, u, d = l.check_diff_summary()
control_num = control_dict.get(l.name, options["control_default"])
self.stdout.write(self.SUMMARY_MASK % (l.name, i, u, d))
i, u, d = ldr.check_diff_summary()
control_num = control_dict.get(ldr.name, options["control_default"])
self.stdout.write(self.SUMMARY_MASK % (ldr.name, i, u, d))
if control_num:
if sum([i, u, d]) >= control_num:
self.stdout.write(
self.CONTROL_MESSAGE % (l.name, sum([i, u, d]), control_num)
self.CONTROL_MESSAGE % (ldr.name, sum([i, u, d]), control_num)
)
self.stderr.write(
self.CONTROL_MESSAGE % (l.name, sum([i, u, d]), control_num)
self.CONTROL_MESSAGE % (ldr.name, sum([i, u, d]), control_num)
)
n_errors = 1
break
else:
l.check_diff()
ldr.check_diff()
else:
n_errors = 0
return 1 if n_errors else 0
......
......@@ -15,7 +15,7 @@ import contextlib
import gzip
# Third-party modules
import ujson
import orjson
# NOC modules
from noc.core.management.base import BaseCommand
......@@ -166,7 +166,7 @@ class Command(BaseCommand):
# Serialize and write
path = kwargs.get("output") + mib.name + ".json.gz"
with self.open_output(path) as f:
f(smart_bytes(ujson.dumps(data)))
f(orjson.dumps(data))
def handle_make_cmib(self, mib_name, *args, **kwargs):
def has_worth_hint(syntax):
......
......@@ -14,7 +14,7 @@ import re
# Third-party modules
from fs import open_fs
import ujson
import orjson
import yaml
import codecs
import uuid
......@@ -225,7 +225,7 @@ class Command(BaseCommand):
def parse_json(j):
try:
return ujson.loads(j)
return orjson.loads(j)
except ValueError as e:
self.die("Failed to parse JSON: %s" % e)
......@@ -417,7 +417,7 @@ class VersionStub(object):
class JSONObject(object):
def __init__(self, path):
with open(path) as f:
data = ujson.load(f)
data = orjson.loads(f.read())
self.scheme = {"telnet": TELNET, "ssh": SSH, "http": HTTP, "https": HTTPS}.get(
data.get("scheme", "telnet"), TELNET
)
......
......@@ -14,7 +14,7 @@ import time
import argparse
# Third-party modules
import ujson
import orjson
# NOC modules
from noc.core.management.base import BaseCommand
......@@ -104,7 +104,7 @@ class Command(BaseCommand):
def update_mib(self, mib, data, version=None):
# Deserialize
d = ujson.loads(data)
d = orjson.loads(data)
# Update timestamp
mib.last_updated = self.decode_date(d["last_updated"])
# Update version
......@@ -118,7 +118,7 @@ class Command(BaseCommand):
def create_mib(self, data):
# Deserialize
d = ujson.loads(data)
d = orjson.loads(data)
# Create MIB
mib = MIB(
name=d["name"],
......
......@@ -10,11 +10,12 @@ import random
from collections import defaultdict
# Third-party modules
import ujson
import orjson
# NOC modules
from noc.config import config, CH_UNCLUSTERED, CH_REPLICATED, CH_SHARDED
from noc.core.service.pub import pub
from noc.core.comp import smart_text
class BaseSharder(object):
......@@ -37,7 +38,10 @@ class BaseSharder(object):
data = self.records[topic]
while data:
chunk, data = data[: self.chunk], data[self.chunk :]
yield topic, "%s\n%s" % (self.table, "\n".join(ujson.dumps(s) for s in chunk))
yield topic, "%s\n%s" % (
self.table,
smart_text(b"\n".join(orjson.dumps(s) for s in chunk)),
)
self.records = defaultdict(list)
def pub(self):
......
......@@ -19,7 +19,7 @@ import operator
from base64 import b85decode
# Third-party modules
import ujson
import orjson
import bson
from mongoengine.fields import ListField, EmbeddedDocumentField, BinaryField
from mongoengine.errors import NotUniqueError
......@@ -112,7 +112,7 @@ class Collection(object):
# Get state from database
cs = coll.find_one({"_id": self.name})
if cs:
return ujson.loads(zlib.decompress(smart_bytes(cs["state"])))
return orjson.loads(zlib.decompress(smart_bytes(cs["state"])))
# Fallback to legacy local
lpath = self.get_legacy_state_path()
state = {}
......@@ -133,7 +133,7 @@ class Collection(object):
coll = self.get_state_collection()
coll.update_one(
{"_id": self.name},
{"$set": {"state": bson.Binary(zlib.compress(smart_bytes(ujson.dumps(state))))}},
{"$set": {"state": bson.Binary(zlib.compress(orjson.dumps(state)))}},
upsert=True,
)
# Remove legacy state
......@@ -197,7 +197,7 @@ class Collection(object):
with open(fp) as f:
data = f.read()
try:
jdata = ujson.loads(data)
jdata = orjson.loads(data)
except ValueError as e:
raise ValueError("Error load %s: %s" % (fp, e))
if "uuid" not in jdata:
......
......@@ -6,10 +6,11 @@
# ----------------------------------------------------------------------
# Third-party modules
import ujson
import orjson
# NOC modules
from .base import BaseMarshaller
from noc.core.comp import smart_text
class JSONMarshaller(BaseMarshaller):
......@@ -24,4 +25,4 @@ class JSONMarshaller(BaseMarshaller):
r["children"] = children
return r
return ujson.dumps([get_node(x) for x in node.iter_nodes()])
return smart_text(orjson.dumps([get_node(x) for x in node.iter_nodes()]))
......@@ -13,7 +13,7 @@ import logging
from collections import defaultdict
# Third-party modules
import ujson
import orjson
import bson
import bson.errors
import pymongo
......@@ -23,7 +23,7 @@ from typing import Optional, Dict, Any, List, Union, Iterable, Tuple, Callable
# NOC modules
from noc.core.perf import metrics
from noc.core.mongo.connection import get_db
from noc.core.comp import smart_bytes
from noc.core.comp import smart_text
from noc.models import get_model
logger = logging.getLogger(__name__)
......@@ -118,7 +118,7 @@ class DataStream(object):
@staticmethod
def get_hash(data) -> str:
return hashlib.sha256(smart_bytes(ujson.dumps(data))).hexdigest()[: DataStream.HASH_LEN]
return hashlib.sha256(orjson.dumps(data)).hexdigest()[: DataStream.HASH_LEN]
@classmethod
def bulk_update(cls, objects: List[Union[id, str, bson.ObjectId]]) -> None:
......@@ -197,7 +197,13 @@ class DataStream(object):
metrics["ds_%s_changed" % m_name] += 1
change_id = bson.ObjectId()
data["change_id"] = str(change_id)
op = {"$set": {cls.F_CHANGEID: change_id, cls.F_HASH: hash, cls.F_DATA: ujson.dumps(data)}}
op = {
"$set": {
cls.F_CHANGEID: change_id,
cls.F_HASH: hash,
cls.F_DATA: smart_text(orjson.dumps(data)),
}
}
if meta:
op["$set"][cls.F_META] = meta
elif "$deleted" not in data:
......@@ -383,7 +389,7 @@ class DataStream(object):
return ""
if isinstance(s, datetime.datetime):
return s.isoformat()
return s.encode("utf-8")
return smart_text(s)
@classmethod
def _parse_filter(cls, expr):
......
......@@ -10,7 +10,7 @@ import asyncio
import logging
# Third-party modules
import ujson
import orjson
# NOC modules
from noc.core.http.client import fetch, ERR_READ_TIMEOUT, ERR_TIMEOUT
......@@ -84,7 +84,7 @@ class DataStreamClient(object):
raise NOCError(code=ERR_DS_BAD_CODE, msg="Invalid response code %s" % code)
# Parse response
try:
data = ujson.loads(data)
data = orjson.loads(data)
except ValueError as e:
logger.info("Cannot parse response: %s", e)
raise NOCError(code=ERR_DS_PARSE_ERROR, msg="Cannot parse response: %s" % e)
......
......@@ -8,7 +8,7 @@
# Python modules
import random
import time
import ujson
import orjson
import uuid
from urllib.parse import unquote
import asyncio
......@@ -21,6 +21,7 @@ from noc.config import config
from noc.core.http.client import fetch
from noc.core.perf import metrics
from noc.core.ioloop.timers import PeriodicCallback
from noc.core.comp import smart_text
from .base import DCSBase, ResolverBase
ConsulRepearableCodes = {500, 598, 599}
......@@ -411,7 +412,7 @@ class ConsulDCS(DCSBase):
if e["Key"] == manifest_path:
cas = e["ModifyIndex"]
# @todo: Handle errors
manifest = ujson.loads(e["Value"])
manifest = orjson.loads(e["Value"])
else:
if "Session" in e:
seen_sessions.add(e["Session"])
......@@ -445,7 +446,11 @@ class ConsulDCS(DCSBase):
try:
r = await self.consul.kv.put(
key=manifest_path,
value=ujson.dumps({"Limit": total_slots, "Holders": holders}, indent=2),
value=smart_text(
orjson.dumps(
{"Limit": total_slots, "Holders": holders}, option=orjson.OPT_INDENT_2
)
),
cas=cas,
)
except ConsulRepeatableErrors as e:
......
......@@ -17,7 +17,7 @@ import traceback
import uuid
# Third-party modules
import ujson
import orjson
# NOC modules
from noc.config import config
......@@ -311,7 +311,7 @@ def error_report(reverse=config.traceback.reverse, logger=logger):
"traceback": r,
}
try:
safe_rewrite(path, ujson.dumps(c))
safe_rewrite(path, orjson.dumps(c))
if CP_SET_UID:
os.chown(path, CP_SET_UID, -1)
logger.error("Writing CP report to %s", path)
......
......@@ -12,12 +12,11 @@ import itertools
import gzip
# Third-party modules
import ujson
import orjson
# NOC modules
from noc.config import config
from noc.core.fileutils import make_persistent
from noc.core.comp import smart_bytes
class Stream(object):
......@@ -50,7 +49,7 @@ class Stream(object):
)
self.out = gzip.open(self.out_path, "wb")
self.chunk_size = 0
self.out.write(smart_bytes(ujson.dumps(self.model.to_json(date=date, **kwargs))))
self.out.write(orjson.dumps(self.model.to_json(date=date, **kwargs)))
self.out.write(b"\n")
self.chunk_size += 1
if self.chunk_size == self.CHUNK_SIZE:
......
......@@ -9,7 +9,7 @@
from urllib.parse import quote as urllib_quote
# Third-party modules
import ujson
import orjson
# NOC modules
from noc.config import config
......@@ -45,7 +45,7 @@ class GoogleGeocoder(BaseGeocoder):
if code != 200:
raise GeoCoderError("%s: %s" % (code, response))
try:
r = ujson.loads(response)
r = orjson.loads(response)
except ValueError:
raise GeoCoderError("Cannot decode result")
if r["status"] != "OK":
......
......@@ -9,7 +9,7 @@
from urllib.parse import quote as urllib_quote
# Third-party modules
import ujson
import orjson
# NOC modules
from .base import BaseGeocoder, GeoCoderResult
......@@ -30,7 +30,7 @@ class OSMNominatimGeocoder(BaseGeocoder):
if code != 200:
raise GeoCoderError("%s: %s" % (code, response))
try:
r = ujson.loads(response)
r = orjson.loads(response)
except ValueError:
raise GeoCoderError("Cannot decode result")
return GeoCoderResult(exact=False, query=query, path=[], lon=r[0]["lon"], lat=r[0]["lat"])
......@@ -9,7 +9,7 @@
from urllib.parse import quote as urllib_quote
# Third-party modules
import ujson
import orjson
# NOC modules
from .base import BaseGeocoder, GeoCoderResult
......@@ -43,7 +43,7 @@ class YandexGeocoder(BaseGeocoder):
elif code != 200:
raise GeoCoderError("%s: %s" % (code, response))
try:
r = ujson.loads(response)
r = orjson.loads(response)
except ValueError:
raise GeoCoderError("Cannot decode result")
results = self.get_path(r, "response.GeoObjectCollection.featureMember") or []
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment