Verified Commit b04fb641 authored by Dmitry Volodin's avatar Dmitry Volodin
Browse files

black: gis, inv, ip, kb

parent caebb4d9
......@@ -10,6 +10,7 @@
from __future__ import absolute_import
from six.moves.urllib.parse import quote as urllib_quote
import requests
# NOC modules
from .base import Geocoder
......
......@@ -9,6 +9,7 @@
# Third-party modules
import pyproj
import geojson
# NOC modules
from noc.gis.models.layer import Layer
from noc.inv.models.object import Object
......@@ -71,10 +72,7 @@ class Map(object):
src_proj = self.get_proj(srid)
cx0, cy0 = pyproj.transform(src_proj, self.db_proj, x0, y0)
cx1, cy1 = pyproj.transform(src_proj, self.db_proj, x1, y1)
bbox = geojson.Polygon(
[[[cx0, cy0], [cx1, cy0], [cx1, cy1],
[cx0, cy1], [cx0, cy0]]]
)
bbox = geojson.Polygon([[[cx0, cy0], [cx1, cy0], [cx1, cy1], [cx0, cy1], [cx0, cy0]]])
return bbox
def get_layer_objects(self, layer, x0, y0, x1, y1, srid):
......@@ -89,23 +87,12 @@ class Map(object):
geojson.Feature(
id=str(d["_id"]),
geometry=self.transform(d["point"], self.db_proj, srid),
properties={
"object": str(d["_id"]),
"label": d.get("name", "")
}
properties={"object": str(d["_id"]), "label": d.get("name", "")},
)
for d in Object._get_collection().find(
{"layer": lr.id, "point": {"$geoWithin": {"$geometry": bbox}}},
{"_id": 1, "point": 1, "label": 1},
)
for d in Object._get_collection().find({
"layer": lr.id,
"point": {
"$geoWithin": {
"$geometry": bbox
}
}
}, {
"_id": 1,
"point": 1,
"label": 1
})
]
return geojson.FeatureCollection(features=features, crs=srid)
......@@ -116,7 +103,8 @@ class Map(object):
"""
if not hasattr(self, "_conduit_layers_ids"):
self._conduit_layers_ids = Layer.objects.filter(
code__in=self.CONDUITS_LAYERS).values_list("id")
code__in=self.CONDUITS_LAYERS
).values_list("id")
return self._conduit_layers_ids
def transform(self, data, src_srid, dst_srid):
......@@ -126,11 +114,9 @@ class Map(object):
return data
if data["type"] == "Point":
x, y = data["coordinates"]
data["coordinates"] = pyproj.transform(
src, dst, x, y)
data["coordinates"] = pyproj.transform(src, dst, x, y)
elif data["type"] == "LineString":
data["coordinates"] = [pyproj.transform(src, dst, x, y)
for x, y in data["coordinates"]]
data["coordinates"] = [pyproj.transform(src, dst, x, y) for x, y in data["coordinates"]]
return data
def get_connection_layer(self, layer, x0, y0, x1, y1, srid):
......@@ -141,31 +127,18 @@ class Map(object):
features = [
geojson.Feature(
id="-".join(str(c["object"]) for c in d["connection"]),
geometry=self.transform(d["line"], self.db_proj, srid)
geometry=self.transform(d["line"], self.db_proj, srid),
)
for d in ObjectConnection._get_collection().find(
{
"layer": layer.id,
"$or": [
{"line": {"$geoWithin": {"$geometry": bbox}}},
{"line": {"$geoIntersects": {"$geometry": bbox}}},
],
},
{"_id": 1, "connection": 1, "line": 1},
)
for d in ObjectConnection._get_collection().find({
"layer": layer.id,
"$or": [
{
"line": {
"$geoWithin": {
"$geometry": bbox
}
}
},
{
"line": {
"$geoIntersects": {
"$geometry": bbox
}
}
}
]
}, {
"_id": 1,
"connection": 1,
"line": 1
})
]
return geojson.FeatureCollection(features=features, crs=srid)
......@@ -180,9 +153,7 @@ class Map(object):
point = geojson.Point(coordinates=[point[0], point[1]])
if len(point) == 3:
point = self.transform(point, point[2], self.db_proj)
q = {
"point__near": point
}
q = {"point__near": point}
if isinstance(layers, list):
q["layer__in"] = layers
else:
......
......@@ -21,6 +21,6 @@ class Migration(BaseMigration):
"min_zoom": 0,
"max_zoom": 5,
"SW": [-180.0, -90.0],
"NE": [179.999999, 89.999999]
"NE": [179.999999, 89.999999],
}
)
......@@ -9,34 +9,51 @@
# Python modules
from __future__ import print_function
import json
# Third-party modules
import bson
from pymongo.errors import BulkWriteError
from pymongo import InsertOne
# NOC modules
from noc.core.migration.base import BaseMigration
class Migration(BaseMigration):
def migrate(self):
if self.db.execute("""
if (
self.db.execute(
"""
select count(*) from pg_class where relname='gis_geodata'
""")[0][0] == 0:
"""
)[0][0]
== 0
):
return # No PostGIS
c = self.mongo_db.noc.geodata
bulk = []
for layer, label, object, data in self.db.execute("""
for layer, label, object, data in self.db.execute(
"""
SELECT layer, label, object, ST_AsGeoJSON(data)
FROM gis_geodata
"""):
"""
):
data = json.loads(data)
bulk += [InsertOne({"layer": bson.ObjectId(layer), "object": bson.ObjectId(object),
"label": label, "data": data})]
bulk += [
InsertOne(
{
"layer": bson.ObjectId(layer),
"object": bson.ObjectId(object),
"label": label,
"data": data,
}
)
]
if bulk:
print("Commiting changes to database")
print ("Commiting changes to database")
try:
c.bulk_write(bulk)
print("Database has been synced")
print ("Database has been synced")
except BulkWriteError as e:
print(("Bulk write error: '%s'", e.details))
print("Stopping check")
print (("Bulk write error: '%s'", e.details))
print ("Stopping check")
......@@ -8,10 +8,12 @@
# Python modules
from __future__ import absolute_import
# Third-party modules
from mongoengine.document import Document
from mongoengine.fields import StringField, IntField, DictField, BooleanField
from mongoengine.signals import post_save
# NOC modules
from noc.lib.nosql import PlainReferenceField
from .street import Street
......@@ -23,7 +25,7 @@ class Address(Document):
"collection": "noc.addresses",
"strict": False,
"auto_create_index": False,
"indexes": ["building", "street"]
"indexes": ["building", "street"],
}
#
building = PlainReferenceField(Building)
......@@ -54,6 +56,7 @@ class Address(Document):
"""
Reset other primary addresses from building
"""
def q(x):
return x if x else ""
......@@ -64,37 +67,32 @@ class Address(Document):
if document.is_primary:
# Reset other primary addresses
Address._get_collection().update({
"building": document.building.id,
"id": {"$ne": document.id}
}, {
"$set": {
"is_primary": False
}
})
Address._get_collection().update(
{"building": document.building.id, "id": {"$ne": document.id}},
{"$set": {"is_primary": False}},
)
# Fill sort order
so = "|".join((str(x) if x else "") for x in [
document.street.name,
q(document.street.short_name),
nq(document.num),
q(document.num_letter),
nq(document.num2),
nq(document.build),
q(document.build_letter),
nq(document.struct),
q(document.struct_letter),
nq(document.struct2),
nq(document.estate),
q(document.estate_letter),
nq(document.estate2)
])
Building._get_collection().update({
"_id": document.building.id
}, {
"$set": {
"sort_order": so
}
})
so = "|".join(
(str(x) if x else "")
for x in [
document.street.name,
q(document.street.short_name),
nq(document.num),
q(document.num_letter),
nq(document.num2),
nq(document.build),
q(document.build_letter),
nq(document.struct),
q(document.struct_letter),
nq(document.struct2),
nq(document.estate),
q(document.estate_letter),
nq(document.estate2),
]
)
Building._get_collection().update(
{"_id": document.building.id}, {"$set": {"sort_order": so}}
)
def display_ru(self, levels=0, to_level=None, sep=", "):
"""
......
......@@ -14,11 +14,7 @@ from mongoengine.fields import StringField, BooleanField, IntField, GeoPointFiel
@six.python_2_unicode_compatible
class Area(Document):
meta = {
"strict": False,
"auto_create_index": False,
"collection": "noc.gis.areas"
}
meta = {"strict": False, "auto_create_index": False, "collection": "noc.gis.areas"}
name = StringField()
is_active = BooleanField(default=True)
......
......@@ -8,11 +8,19 @@
# Python modules
from __future__ import absolute_import
# Third-party modules
from mongoengine.document import Document
from mongoengine.fields import (StringField, IntField, BooleanField,
ListField, EmbeddedDocumentField,
DictField, DateTimeField)
from mongoengine.fields import (
StringField,
IntField,
BooleanField,
ListField,
EmbeddedDocumentField,
DictField,
DateTimeField,
)
# NOC modules
from noc.lib.nosql import PlainReferenceField
from noc.core.model.decorator import on_save
......@@ -22,16 +30,13 @@ from .division import Division
@on_save
@on_delete_check(check=[
("gis.Address", "building"),
("inv.CoveredBuilding", "building")
])
@on_delete_check(check=[("gis.Address", "building"), ("inv.CoveredBuilding", "building")])
class Building(Document):
meta = {
"collection": "noc.buildings",
"strict": False,
"auto_create_index": False,
"indexes": ["adm_division", "data", "sort_order"]
"indexes": ["adm_division", "data", "sort_order"],
}
# Administrative division
adm_division = PlainReferenceField(Division)
......@@ -42,9 +47,10 @@ class Building(Document):
("B", "BUILDING"),
("R", "READY"),
("E", "EVICTED"),
("D", "DEMOLISHED")
("D", "DEMOLISHED"),
],
default="R")
default="R",
)
# Total homes
homes = IntField()
# Maximal amount of floors
......@@ -79,9 +85,15 @@ class Building(Document):
# Fallback to first address found
return Address.objects.filter(building=self.id).first()
def fill_entrances(self, first_entrance=1, first_home=1,
n_entrances=1, first_floor=1, last_floor=1,
homes_per_entrance=1):
def fill_entrances(
self,
first_entrance=1,
first_home=1,
n_entrances=1,
first_floor=1,
last_floor=1,
homes_per_entrance=1,
):
e_home = first_home
for e in range(first_entrance, first_entrance + n_entrances):
self.entrances += [
......@@ -90,7 +102,7 @@ class Building(Document):
first_floor=str(first_floor),
last_floor=str(last_floor),
first_home=str(e_home),
last_home=str(e_home + homes_per_entrance - 1)
last_home=str(e_home + homes_per_entrance - 1),
)
]
e_home += homes_per_entrance
......
......@@ -8,17 +8,15 @@
# Python modules
from __future__ import absolute_import
# Third-party modules
from mongoengine.document import Document
# NOC models
from noc.lib.nosql import PlainReferenceField
from .normativedocument import NormativeDocument
class Change(Document):
meta = {
"collection": "noc.changes",
"strict": False,
"auto_create_index": False
}
meta = {"collection": "noc.changes", "strict": False, "auto_create_index": False}
document = PlainReferenceField(NormativeDocument)
......@@ -8,33 +8,37 @@
# Python modules
from __future__ import absolute_import
# Third-party modules
import six
from mongoengine.document import Document
from mongoengine.fields import (StringField, DictField, BooleanField,
DateTimeField, IntField, ListField)
from mongoengine.fields import (
StringField,
DictField,
BooleanField,
DateTimeField,
IntField,
ListField,
)
# NOC modules
from noc.lib.nosql import PlainReferenceField
from noc.core.model.decorator import on_delete_check
@on_delete_check(check=[
("gis.Street", "parent"),
("gis.Division", "parent"),
("gis.Building", "adm_division")
])
@on_delete_check(
check=[("gis.Street", "parent"), ("gis.Division", "parent"), ("gis.Building", "adm_division")]
)
@six.python_2_unicode_compatible
class Division(Document):
meta = {
"collection": "noc.divisions",
"strict": False,
"auto_create_index": False,
"indexes": ["parent", "data", "name"]
"indexes": ["parent", "data", "name"],
}
# Division type
type = StringField(default="A", choices=[
("A", "Administrative")
])
type = StringField(default="A", choices=[("A", "Administrative")])
#
parent = PlainReferenceField("self")
# Normalized name
......@@ -69,6 +73,7 @@ class Division(Document):
def get_buildings(self):
from .building import Building
return Building.objects.filter(adm_division=self.id).order_by("sort_order")
@classmethod
......@@ -76,6 +81,7 @@ class Division(Document):
"""
Update divisions levels
"""
def _update(root, level):
if root.level != level:
root.level = level
......
......@@ -14,10 +14,7 @@ from mongoengine.fields import StringField, IntField
@six.python_2_unicode_compatible
class Entrance(EmbeddedDocument):
meta = {
"strict": False,
"auto_create_index": False
}
meta = {"strict": False, "auto_create_index": False}
number = StringField()
# Floors
first_floor = IntField()
......
......@@ -13,10 +13,11 @@ import logging
import hashlib
import base64
import datetime
# Third-party modules
from mongoengine.document import Document
from mongoengine.fields import (StringField, FloatField, ListField,
DateTimeField)
from mongoengine.fields import StringField, FloatField, ListField, DateTimeField
# NOC modules
from noc.core.geocoding.base import GeoCoderError, GeoCoderResult
from noc.config import config
......@@ -28,12 +29,7 @@ logger = logging.getLogger(__name__)
class GeocoderCache(Document):
meta = {
"collection": "noc.geocodercache",
"indexes": [
{
"fields": ["expires"],
"expireAfterSeconds": 0
}
]
"indexes": [{"fields": ["expires"], "expireAfterSeconds": 0}],
}
# query hash
......@@ -64,7 +60,7 @@ class GeocoderCache(Document):
gcls = {
"yandex": "noc.core.geocoding.yandex.YandexGeocoder",
"google": "noc.core.geocoding.google.GoogleGeocoder"
"google": "noc.core.geocoding.google.GoogleGeocoder",
}
@classmethod
......@@ -119,7 +115,7 @@ class GeocoderCache(Document):
query=query,
path=r.get("path") or [],
lon=r.get("lon"),
lat=r.get("lat")
lat=r.get("lat"),
)
# Not found, resolve
r = None
......@@ -144,11 +140,7 @@ class GeocoderCache(Document):
r = None
except GeoCoderError as e:
error = str(e)
sq = {
"query": query,
"system": gsys,
"error": error
}
sq = {"query": query, "system": gsys, "error": error}
if not r and lr:
r = lr # Reuse first non-exact message
if r:
......@@ -160,7 +152,5 @@ class GeocoderCache(Document):
if not r or not r.exact:
sq["expires"] = datetime.datetime.now() + datetime.timedelta(seconds=cls.NEGATIVE_TTL)
# Write to database
c.update({
"_id": hash
}, {"$set": sq}, upsert=True)
c.update({"_id": hash}, {"$set": sq}, upsert=True)
return r
......@@ -10,11 +10,13 @@
import os
from threading import Lock
import operator
# Third-party modules
import six
from mongoengine.document import Document
from mongoengine.fields import StringField, UUIDField, IntField, BooleanField
import cachetools
# NOC modules
from noc.lib.prettyjson import to_json
from noc.lib.text import quote_safe_path
......@@ -23,16 +25,14 @@ from noc.core.model.decorator import on_delete_check
id_lock = Lock()
@on_delete_check(check=[
("inv.Object", "layer")
])
@on_delete_check(check=[("inv.Object", "layer")])
@six.python_2_unicode_compatible
class Layer(Document):
meta = {
"collection": "noc.layers",
"strict": False,
"auto_create_index": False,
"json_collection": "gis.layers"
"json_collection": "gis.layers",
}
name = StringField(unique=True)
code = StringField(unique=True)
......@@ -51,13 +51,27 @@ class Layer(Document):
# Point symbolizer
point_radius = IntField(default=5)
point_graphic = StringField(
choices=[(x, x) for x in ("circle", "triangle", "cross", "x", "square", "star", "diamond", "antenna", "flag")],
default="circle"
choices=[
(x, x)
for x in (
"circle",
"triangle",
"cross",
"x",
"square",
"star",
"diamond",
"antenna",
"flag",
)
],
default="circle",
)
# Line symbolizer
stroke_dashstyle = StringField(choices=[(x, x) for x in (
"solid", "dash", "dashdot", "longdash",
"longdashdot")], default="solid")
stroke_dashstyle = StringField(
choices=[(x, x) for x in ("solid", "dash", "dashdot", "longdash", "longdashdot")],
default="solid",
)