#1598 Extracting data for Building gis model

core/etl/extractor/building.py

+# ----------------------------------------------------------------------
+# FIAS Building Extractor
+# ----------------------------------------------------------------------
+# Copyright (C) 2021 The NOC Project
+# See LICENSE for details
+# ----------------------------------------------------------------------
+
+# python modules
+import dbf
+import requests
+import os
+from datetime import datetime
+from pathlib import Path
+from zipfile import ZipFile, is_zipfile
+
+# NOC modules
+from .base import BaseExtractor
+from ..models.building import Building
+from noc.core.etl.remotesystem.base import BaseRemoteSystem
+
+
+class BuildingRemoteSystem(BaseRemoteSystem):
+    """
+    base class
+
+    Configuration variables
+    *FIAS_URL* - url of source FIAS data
+    *CACHE_PATH* - dir target download files
+    *REGION* - region code
+    """
+
+
+@BuildingRemoteSystem.extractor
+class BuildingExtractor(BaseExtractor):
+    """
+    Building extractor.
+    """
+
+    name = "building"
+    model = Building
+
+    def __init__(self, system, *args, **kwargs):
+        super(BuildingExtractor, self).__init__(system)
+        self.fias_url = str(self.config.get("FIAS_URL"))
+        self.cache_path = str(self.config.get("CACHE_PATH"))
+        self.region = str(self.config.get("REGION"))
+        self.check_path(self.cache_path)
+        self.zip_path = os.path.join(self.cache_path, "fias_dbf.zip")
+        self.dbf_file_house = f"HOUSE{self.region}.DBF"
+        self.dbf_file_address = f"ADDROB{self.region}.DBF"
+        self.dbf_path_house = os.path.join(self.cache_path, self.dbf_file_house)
+        self.dbf_path_address = os.path.join(self.cache_path, self.dbf_file_address)
+        self.now = datetime.now().date()
+
+    def check_path(self, path):
+        # check exists cache_path
+        dirpath = Path(path)
+        if not dirpath.exists() or not dirpath.is_dir():
+            os.makedirs(path)
+
+    def download(self):
+        if (
+            not os.path.isfile(self.zip_path)
+            or datetime.fromtimestamp(os.path.getctime(self.zip_path)).date()
+            != datetime.now().date()
+        ):
+            r = requests.get(self.fias_url, stream=True)
+            if r.status_code == 200:
+                with open(self.zip_path, "wb") as f:
+                    for chunk in r.iter_content(1024):
+                        f.write(chunk)
+            if is_zipfile(self.zip_path):
+                with ZipFile(self.zip_path, "r") as f:
+                    f.extract(self.dbf_file_house, self.cache_path)
+                    f.extract(self.dbf_file_address, self.cache_path)
+            else:
+                raise Exception("zipfile not found!")
+
+    def extract(self):
+        super().extract()
+        return
+
+    def get_oktmo_data(self):
+        oktmo_data = {}
+        with dbf.Table(filename=self.dbf_path_address, codepage="cp866") as table:
+            for r in table:
+                if r.AOLEVEL in [7, 4, 35] and r.NEXTID == " " * 36 and r.OKTMO != " " * 11:
+                    oktmo = r.OKTMO.rstrip().zfill(11)
+                    oktmo_data[r.AOGUID] = oktmo
+
+        return oktmo_data
+
+    def iter_data(self):
+        self.download()
+        oktmo_data = self.get_oktmo_data()
+        with dbf.Table(filename=self.dbf_path_house, codepage="cp866") as table:
+            for r in table:
+                oktmo = oktmo_data.get(r.AOGUID)
+                if r.ENDDATE >= self.now and oktmo:
+                    yield r.HOUSEGUID, oktmo, r.POSTALCODE, r.STARTDATE, r.ENDDATE

core/etl/models/building.py

@@ -12,13 +12,15 @@ from datetime import date
 # NOC modules
 from .base import BaseModel
 from .typing import Reference
-from .division import Division
+from .admdiv import AdmDiv
 
 
 class Building(BaseModel):
     id: str
     oktmo: str
-    adm_division: Reference["Division"]
+    adm_division: Reference["AdmDiv"]
     postal_code: Optional[str]
     start_date: Optional[date]
     end_date: Optional[date]
+
+    _csv_fields = ["id", "oktmo", "adm_division", "postal_code", "start_date", "end_date"]

core/etl/remotesystem/base.py

@@ -20,6 +20,7 @@ class BaseRemoteSystem(object):
     extractors_order = [
         "label",
         "admdiv",
+        "building",
         "networksegmentprofile",
         "networksegment",
         "object",

main/models/remotesystem.py

@@ -86,6 +86,7 @@ class RemoteSystem(Document):
     enable_admdiv = BooleanField()
     enable_administrativedomain = BooleanField()
     enable_authprofile = BooleanField()
+    enable_building = BooleanField()
     enable_container = BooleanField()
     enable_link = BooleanField()
     enable_managedobject = BooleanField()

requirements.txt

@@ -98,3 +98,5 @@ pytest-dependency==0.5.1; extra == "testing"
 Cython==0.29.21; extra == "cython"
 # Kafka
 aiokafka==0.6.0; extra == "sender-kafka"
+# etl/extractor
+dbf==0.99.1; extra == "testing"

ui/web/main/remotesystem/Application.js

@@ -73,6 +73,11 @@ Ext.define("NOC.main.remotesystem.Application", {
                             xtype: "checkbox",
                             boxLabel: __("Auth Profile")
                         },
+                        {
+                            name: "enable_building",
+                            xtype: "checkbox",
+                            boxLabel: __("Building")
+                        },
                         {
                             name: "enable_container",
                             xtype: "checkbox",

ui/web/main/remotesystem/Model.js

@@ -44,6 +44,10 @@ Ext.define("NOC.main.remotesystem.Model", {
             name: "enable_authprofile",
             type: "boolean"
         },
+        {
+            name: "enable_building",
+            type: "boolean"
+        },
         {
             name: "enable_container",
             type: "boolean"