Commit d0f2fef9 authored by Andrey Vertiprahov's avatar Andrey Vertiprahov
Browse files

Merge branch 'vak-1598-building' into 'master'

#1598 Extracting data for Building gis model

See merge request noc/noc!5444
parents f6e67761 99d89ff3
# ----------------------------------------------------------------------
# FIAS Building Extractor
# ----------------------------------------------------------------------
# Copyright (C) 2021 The NOC Project
# See LICENSE for details
# ----------------------------------------------------------------------
# python modules
import dbf
import requests
import os
from datetime import datetime
from pathlib import Path
from zipfile import ZipFile, is_zipfile
# NOC modules
from .base import BaseExtractor
from ..models.building import Building
from noc.core.etl.remotesystem.base import BaseRemoteSystem
class BuildingRemoteSystem(BaseRemoteSystem):
"""
base class
Configuration variables
*FIAS_URL* - url of source FIAS data
*CACHE_PATH* - dir target download files
*REGION* - region code
"""
@BuildingRemoteSystem.extractor
class BuildingExtractor(BaseExtractor):
"""
Building extractor.
"""
name = "building"
model = Building
def __init__(self, system, *args, **kwargs):
super(BuildingExtractor, self).__init__(system)
self.fias_url = str(self.config.get("FIAS_URL"))
self.cache_path = str(self.config.get("CACHE_PATH"))
self.region = str(self.config.get("REGION"))
self.check_path(self.cache_path)
self.zip_path = os.path.join(self.cache_path, "fias_dbf.zip")
self.dbf_file_house = f"HOUSE{self.region}.DBF"
self.dbf_file_address = f"ADDROB{self.region}.DBF"
self.dbf_path_house = os.path.join(self.cache_path, self.dbf_file_house)
self.dbf_path_address = os.path.join(self.cache_path, self.dbf_file_address)
self.now = datetime.now().date()
def check_path(self, path):
# check exists cache_path
dirpath = Path(path)
if not dirpath.exists() or not dirpath.is_dir():
os.makedirs(path)
def download(self):
if (
not os.path.isfile(self.zip_path)
or datetime.fromtimestamp(os.path.getctime(self.zip_path)).date()
!= datetime.now().date()
):
r = requests.get(self.fias_url, stream=True)
if r.status_code == 200:
with open(self.zip_path, "wb") as f:
for chunk in r.iter_content(1024):
f.write(chunk)
if is_zipfile(self.zip_path):
with ZipFile(self.zip_path, "r") as f:
f.extract(self.dbf_file_house, self.cache_path)
f.extract(self.dbf_file_address, self.cache_path)
else:
raise Exception("zipfile not found!")
def extract(self):
super().extract()
return
def get_oktmo_data(self):
oktmo_data = {}
with dbf.Table(filename=self.dbf_path_address, codepage="cp866") as table:
for r in table:
if r.AOLEVEL in [7, 4, 35] and r.NEXTID == " " * 36 and r.OKTMO != " " * 11:
oktmo = r.OKTMO.rstrip().zfill(11)
oktmo_data[r.AOGUID] = oktmo
return oktmo_data
def iter_data(self):
self.download()
oktmo_data = self.get_oktmo_data()
with dbf.Table(filename=self.dbf_path_house, codepage="cp866") as table:
for r in table:
oktmo = oktmo_data.get(r.AOGUID)
if r.ENDDATE >= self.now and oktmo:
yield r.HOUSEGUID, oktmo, r.POSTALCODE, r.STARTDATE, r.ENDDATE
......@@ -12,13 +12,15 @@ from datetime import date
# NOC modules
from .base import BaseModel
from .typing import Reference
from .division import Division
from .admdiv import AdmDiv
class Building(BaseModel):
id: str
oktmo: str
adm_division: Reference["Division"]
adm_division: Reference["AdmDiv"]
postal_code: Optional[str]
start_date: Optional[date]
end_date: Optional[date]
_csv_fields = ["id", "oktmo", "adm_division", "postal_code", "start_date", "end_date"]
......@@ -20,6 +20,7 @@ class BaseRemoteSystem(object):
extractors_order = [
"label",
"admdiv",
"building",
"networksegmentprofile",
"networksegment",
"object",
......
......@@ -86,6 +86,7 @@ class RemoteSystem(Document):
enable_admdiv = BooleanField()
enable_administrativedomain = BooleanField()
enable_authprofile = BooleanField()
enable_building = BooleanField()
enable_container = BooleanField()
enable_link = BooleanField()
enable_managedobject = BooleanField()
......
......@@ -98,3 +98,5 @@ pytest-dependency==0.5.1; extra == "testing"
Cython==0.29.21; extra == "cython"
# Kafka
aiokafka==0.6.0; extra == "sender-kafka"
# etl/extractor
dbf==0.99.1; extra == "testing"
......@@ -73,6 +73,11 @@ Ext.define("NOC.main.remotesystem.Application", {
xtype: "checkbox",
boxLabel: __("Auth Profile")
},
{
name: "enable_building",
xtype: "checkbox",
boxLabel: __("Building")
},
{
name: "enable_container",
xtype: "checkbox",
......
......@@ -44,6 +44,10 @@ Ext.define("NOC.main.remotesystem.Model", {
name: "enable_authprofile",
type: "boolean"
},
{
name: "enable_building",
type: "boolean"
},
{
name: "enable_container",
type: "boolean"
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment