Commit 3b7a7f0e authored by Andrey Vertiprahov's avatar Andrey Vertiprahov
Browse files

Merge branch 'vak-1598' into 'master'

Extracting data for division gis model

See merge request noc/noc!5427
parents 861d6bf4 05d7515d
# ----------------------------------------------------------------------
# OKTMO Data Extractor
# ----------------------------------------------------------------------
# Copyright (C) 2021 The NOC Project
# See LICENSE for details
# ----------------------------------------------------------------------
# python modules
import requests
import os
import csv
from pathlib import Path
# NOC modules
from .base import BaseExtractor
from ..models.admdiv import AdmDiv
from noc.core.etl.remotesystem.base import BaseRemoteSystem
class AdmDivRemoteSystem(BaseRemoteSystem):
"""
base class
Configuration variables
*OKTMO_URL* - url of source oktmo data
*CACHE_PATH* - dir target download files
*REGION* - region code
"""
@AdmDivRemoteSystem.extractor
class AdmDivExtractor(BaseExtractor):
"""
Oktmo extractor.
"""
name = "admdiv"
model = AdmDiv
twice_code = []
def __init__(self, system, *args, **kwargs):
super(AdmDivExtractor, self).__init__(system)
self.oktmo_url = str(self.config.get("OKTMO_URL"))
self.cache_path = str(self.config.get("CACHE_PATH"))
self.region = str(self.config.get("REGION"))
self.check_path(self.cache_path)
self.csv_path = None
def check_path(self, path):
# check exists cache_path
dirpath = Path(path)
if not dirpath.exists() or not dirpath.is_dir():
os.makedirs(path)
def download(self):
r = requests.get(self.oktmo_url, stream=True)
self.csv_path = os.path.join(self.cache_path, "oktmo.csv")
if r.status_code == 200:
with open(self.csv_path, "wb") as f:
for chunk in r.iter_content(1024):
f.write(chunk)
def extract(self):
super(AdmDivExtractor, self).extract()
return
def check_twice_code(self, ter, kod1, kod2, kod3):
"""
Checking duble oktmo code
:param ter:
:param kod1:
:param kod2:
:param kod3:
:return:
"""
if kod3 == "000":
oktmo = f"{ter}{kod1}{kod2}{kod3}"
if oktmo in self.twice_code:
return False
else:
self.twice_code.append(oktmo)
return True
def parent_level(self, ter, kod1, kod2, kod3):
"""
Creating parent code
:param ter:
:param kod1:
:param kod2:
:param kod3:
:return:
"""
if self.region != "0" and kod1[1:3] == "00" and kod2 == "000" and kod3 == "000":
return f"{ter}000000000"
elif kod1[1:3] != "00" and kod2 == "000" and kod3 == "000":
return f"{ter}{kod1[0]}00000000"
elif kod2 != "000" and kod3 == "000":
return f"{ter}{kod1}000000"
elif kod2 == "000" and kod3 != "000":
return f"{ter}{kod1}000000"
elif kod2 != "000" and kod3 != "000":
return f"{ter}{kod1}{kod2}000"
else:
return ""
def iter_data(self):
self.download()
with open(self.csv_path, encoding="cp1251") as f:
reader = csv.reader(f, delimiter=";", quotechar='"')
for row in reader:
ter = row[0]
kod1 = row[1]
kod2 = row[2]
kod3 = row[3]
name = " ".join(row[6].split()[1:])
short_name = row[6].split()[0]
name = " ".join(row[6].split()[1:])
short_name = row[6].split()[0]
oktmo = f"{ter}{kod1}{kod2}{kod3}"
if ter == "04" and self.check_twice_code(ter, kod1, kod2, kod3):
parent = self.parent_level(ter, kod1, kod2, kod3)
parent = "" if parent == oktmo else parent
yield f"{ter}{kod1}{kod2}{kod3}", parent, name, short_name
else:
continue
# ----------------------------------------------------------------------
# AddressModel
# ----------------------------------------------------------------------
# Copyright (C) 2021 The NOC Project
# See LICENSE for details
# ----------------------------------------------------------------------
# Python modules
from typing import Optional
# NOC modules
from .base import BaseModel
from .typing import Reference
from .building import Building
from .street import Street
class Address(BaseModel):
id: str
building: Reference["Building"]
street: Reference["Street"]
num: int
num_letter: Optional[str]
# ----------------------------------------------------------------------
# BuildingModel
# ----------------------------------------------------------------------
# Copyright (C) 2021 The NOC Project
# See LICENSE for details
# ----------------------------------------------------------------------
# Python modules
from typing import Optional
from datetime import date
# NOC modules
from .base import BaseModel
from .typing import Reference
from .division import Division
class Building(BaseModel):
id: str
oktmo: str
adm_division: Reference["Division"]
postal_code: Optional[str]
start_date: Optional[date]
end_date: Optional[date]
# ----------------------------------------------------------------------
# DivisionModel
# ----------------------------------------------------------------------
# Copyright (C) 2021 The NOC Project
# See LICENSE for details
# ----------------------------------------------------------------------
# Python modules
from typing import Optional
# NOC modules
from .base import BaseModel
class Division(BaseModel):
id: str
name: str
start_date: Optional[str]
# ----------------------------------------------------------------------
# StreetModel
# ----------------------------------------------------------------------
# Copyright (C) 2021 The NOC Project
# See LICENSE for details
# ----------------------------------------------------------------------
# Python modules
from typing import Optional
from datetime import date
# NOC modules
from .base import BaseModel
from .typing import Reference
from .division import Division
class Street(BaseModel):
id: str
parent: Reference["Division"]
oktmo: str
name: str
short_name: str
start_date: Optional[date]
end_date: Optional[date]
# ---------------------------------------------------------------------
# Address object
# ---------------------------------------------------------------------
# Copyright (C) 2007-2019 The NOC Project
# Copyright (C) 2007-2021 The NOC Project
# See LICENSE for details
# ---------------------------------------------------------------------
# Third-party modules
from mongoengine.document import Document
from mongoengine.fields import StringField, IntField, DictField, BooleanField
from mongoengine.fields import (
StringField,
IntField,
DictField,
BooleanField,
ReferenceField,
LongField,
)
from mongoengine.signals import post_save
# NOC modules
from noc.core.mongo.fields import PlainReferenceField
from .street import Street
from .building import Building
from noc.main.models.remotesystem import RemoteSystem
from noc.core.bi.decorator import bi_sync
@bi_sync
class Address(Document):
meta = {
"collection": "noc.addresses",
......@@ -46,6 +56,12 @@ class Address(Document):
data = DictField()
is_primary = BooleanField(default=True)
# Reference to remote system object has been imported from
remote_system = ReferenceField(RemoteSystem)
# Object id in remote system
remote_id = StringField()
# Object id in BI
bi_id = LongField(unique=True)
@classmethod
def update_primary(cls, sender, document, **kwargs):
......
# ---------------------------------------------------------------------
# Building object
# ---------------------------------------------------------------------
# Copyright (C) 2007-2019 The NOC Project
# Copyright (C) 2007-2021 The NOC Project
# See LICENSE for details
# ---------------------------------------------------------------------
......@@ -15,6 +15,8 @@ from mongoengine.fields import (
EmbeddedDocumentField,
DictField,
DateTimeField,
LongField,
ReferenceField,
)
# NOC modules
......@@ -23,8 +25,11 @@ from noc.core.model.decorator import on_save
from noc.core.model.decorator import on_delete_check
from .entrance import Entrance
from .division import Division
from noc.main.models.remotesystem import RemoteSystem
from noc.core.bi.decorator import bi_sync
@bi_sync
@on_save
@on_delete_check(check=[("gis.Address", "building"), ("inv.CoveredBuilding", "building")])
class Building(Document):
......@@ -69,6 +74,12 @@ class Building(Document):
# Internal field for sorting
# Filled by primary address trigger
sort_order = StringField()
# Reference to remote system object has been imported from
remote_system = ReferenceField(RemoteSystem)
# Object id in remote system
remote_id = StringField()
# Object id in BI
bi_id = LongField(unique=True)
@property
def primary_address(self):
......
# ---------------------------------------------------------------------
# Division object
# ---------------------------------------------------------------------
# Copyright (C) 2007-2020 The NOC Project
# Copyright (C) 2007-2021 The NOC Project
# See LICENSE for details
# ---------------------------------------------------------------------
......@@ -14,6 +14,8 @@ from mongoengine.fields import (
DateTimeField,
IntField,
ListField,
LongField,
ReferenceField,
)
# NOC modules
......@@ -21,8 +23,11 @@ from noc.main.models.label import Label
from noc.core.mongo.fields import PlainReferenceField
from noc.core.comp import smart_text
from noc.core.model.decorator import on_delete_check
from noc.main.models.remotesystem import RemoteSystem
from noc.core.bi.decorator import bi_sync
@bi_sync
@Label.model
@on_delete_check(
check=[("gis.Street", "parent"), ("gis.Division", "parent"), ("gis.Building", "adm_division")]
......@@ -52,6 +57,12 @@ class Division(Document):
#
start_date = DateTimeField()
end_date = DateTimeField()
# Reference to remote system object has been imported from
remote_system = ReferenceField(RemoteSystem)
# Object id in remote system
remote_id = StringField()
# Object id in BI
bi_id = LongField(unique=True)
# Labels
labels = ListField(StringField())
effective_labels = ListField(StringField())
......
# ---------------------------------------------------------------------
# Street object
# ---------------------------------------------------------------------
# Copyright (C) 2007-2020 The NOC Project
# Copyright (C) 2007-2021 The NOC Project
# See LICENSE for details
# ---------------------------------------------------------------------
# Third-party modules
from mongoengine.document import Document
from mongoengine.fields import StringField, DictField, BooleanField, DateTimeField
from mongoengine.fields import (
StringField,
DictField,
BooleanField,
DateTimeField,
ReferenceField,
LongField,
)
# NOC modules
from noc.core.mongo.fields import PlainReferenceField
from noc.core.model.decorator import on_delete_check
from noc.core.comp import smart_text
from .division import Division
from noc.main.models.remotesystem import RemoteSystem
from noc.core.bi.decorator import bi_sync
@bi_sync
@on_delete_check(check=[("gis.Address", "street")])
class Street(Document):
meta = {
......@@ -35,6 +45,12 @@ class Street(Document):
# Additional data
# Depends on importer
data = DictField()
# Reference to remote system object has been imported from
remote_system = ReferenceField(RemoteSystem)
# Object id in remote system
remote_id = StringField()
# Object id in BI
bi_id = LongField(unique=True)
#
start_date = DateTimeField()
end_date = DateTimeField()
......
//---------------------------------------------------------------------
// gis.building application
//---------------------------------------------------------------------
// Copyright (C) 2007-2014 The NOC Project
// Copyright (C) 2007-2021 The NOC Project
// See LICENSE for details
//---------------------------------------------------------------------
console.debug("Defining NOC.gis.building.Application");
......@@ -39,6 +39,18 @@ Ext.define("NOC.gis.building.Application", {
dataIndex: "homes",
width: 75,
align: "right"
},
{
text: __("Remote System"),
dataIndex: "remote_system",
flex: 1,
sort: false
},
{
text: __("Remote Id"),
dataIndex: "remote_id",
flex: 1,
sort: false
}
],
fields: [
......
//---------------------------------------------------------------------
// gis.building Model
//---------------------------------------------------------------------
// Copyright (C) 2007-2014 The NOC Project
// Copyright (C) 2007-2021 The NOC Project
// See LICENSE for details
//---------------------------------------------------------------------
console.debug("Defining NOC.gis.building.Model");
......@@ -76,6 +76,14 @@ Ext.define("NOC.gis.building.Model", {
name: "full_path",
type: "string",
persist: false
},
{
name: "remote_system",
type: "string"
},
{
name: "remote_id",
type: "string"
}
]
});
//---------------------------------------------------------------------
// gis.division application
//---------------------------------------------------------------------
// Copyright (C) 2007-2014 The NOC Project
// Copyright (C) 2007-2021 The NOC Project
// See LICENSE for details
//---------------------------------------------------------------------
console.debug("Defining NOC.gis.division.Application");
......@@ -49,6 +49,18 @@ Ext.define("NOC.gis.division.Application", {
dataIndex: "full_parent",
flex: 1,
sort: false
},
{
text: __("Remote System"),
dataIndex: "remote_system",
flex: 1,
sort: false
},
{
text: __("Remote Id"),
dataIndex: "remote_id",
flex: 1,
sort: false
}
],
fields: [
......
//---------------------------------------------------------------------
// gis.division Model
//---------------------------------------------------------------------
// Copyright (C) 2007-2014 The NOC Project
// Copyright (C) 2007-2021 The NOC Project
// See LICENSE for details
//---------------------------------------------------------------------
console.debug("Defining NOC.gis.division.Model");
......@@ -66,6 +66,14 @@ Ext.define("NOC.gis.division.Model", {
{
name: "name",
type: "string"
},
{
name: "remote_system",
type: "string"
},
{
name: "remote_id",
type: "string"
}
]
});
//---------------------------------------------------------------------
// gis.street application
//---------------------------------------------------------------------
// Copyright (C) 2007-2014 The NOC Project
// Copyright (C) 2007-2021 The NOC Project
// See LICENSE for details
//---------------------------------------------------------------------
console.debug("Defining NOC.gis.street.Application");
......@@ -42,6 +42,18 @@ Ext.define("NOC.gis.street.Application", {
dataIndex: "full_parent",
flex: 1,
sort: false
},
{
text: __("Remote System"),
dataIndex: "remote_system",
flex: 1,
sort: false
},
{
text: __("Remote Id"),
dataIndex: "remote_id",
flex: 1,
sort: false
}
],
fields: [
......
//---------------------------------------------------------------------
// gis.division Model
//---------------------------------------------------------------------
// Copyright (C) 2007-2014 The NOC Project
// Copyright (C) 2007-2021 The NOC Project
// See LICENSE for details
//---------------------------------------------------------------------
console.debug("Defining NOC.gis.street.Model");
......@@ -57,6 +57,14 @@ Ext.define("NOC.gis.street.Model", {
{
name: "name",
type: "string"
},
{
name: "remote_system",
type: "string"
},
{
name: "remote_id",
type: "string"
}
]
});
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment