Extracting data for division gis model

core/etl/extractor/admdiv.py

+# ----------------------------------------------------------------------
+# OKTMO Data Extractor
+# ----------------------------------------------------------------------
+# Copyright (C) 2021 The NOC Project
+# See LICENSE for details
+# ----------------------------------------------------------------------
+
+# python modules
+import requests
+import os
+import csv
+from pathlib import Path
+
+# NOC modules
+from .base import BaseExtractor
+from ..models.admdiv import AdmDiv
+from noc.core.etl.remotesystem.base import BaseRemoteSystem
+
+
+class AdmDivRemoteSystem(BaseRemoteSystem):
+    """
+    base class
+
+    Configuration variables
+    *OKTMO_URL* - url of source oktmo data
+    *CACHE_PATH* - dir target download files
+    *REGION* - region code
+    """
+
+
+@AdmDivRemoteSystem.extractor
+class AdmDivExtractor(BaseExtractor):
+    """
+    Oktmo extractor.
+    """
+
+    name = "admdiv"
+    model = AdmDiv
+    twice_code = []
+
+    def __init__(self, system, *args, **kwargs):
+        super(AdmDivExtractor, self).__init__(system)
+        self.oktmo_url = str(self.config.get("OKTMO_URL"))
+        self.cache_path = str(self.config.get("CACHE_PATH"))
+        self.region = str(self.config.get("REGION"))
+        self.check_path(self.cache_path)
+        self.csv_path = None
+
+    def check_path(self, path):
+        # check exists cache_path
+        dirpath = Path(path)
+        if not dirpath.exists() or not dirpath.is_dir():
+            os.makedirs(path)
+
+    def download(self):
+        r = requests.get(self.oktmo_url, stream=True)
+        self.csv_path = os.path.join(self.cache_path, "oktmo.csv")
+        if r.status_code == 200:
+            with open(self.csv_path, "wb") as f:
+                for chunk in r.iter_content(1024):
+                    f.write(chunk)
+
+    def extract(self):
+        super(AdmDivExtractor, self).extract()
+        return
+
+    def check_twice_code(self, ter, kod1, kod2, kod3):
+        """
+        Checking duble oktmo code
+
+        :param ter:
+        :param kod1:
+        :param kod2:
+        :param kod3:
+        :return:
+        """
+        if kod3 == "000":
+            oktmo = f"{ter}{kod1}{kod2}{kod3}"
+            if oktmo in self.twice_code:
+                return False
+            else:
+                self.twice_code.append(oktmo)
+        return True
+
+    def parent_level(self, ter, kod1, kod2, kod3):
+        """
+        Creating parent code
+
+        :param ter:
+        :param kod1:
+        :param kod2:
+        :param kod3:
+        :return:
+        """
+        if self.region != "0" and kod1[1:3] == "00" and kod2 == "000" and kod3 == "000":
+            return f"{ter}000000000"
+        elif kod1[1:3] != "00" and kod2 == "000" and kod3 == "000":
+            return f"{ter}{kod1[0]}00000000"
+        elif kod2 != "000" and kod3 == "000":
+            return f"{ter}{kod1}000000"
+        elif kod2 == "000" and kod3 != "000":
+            return f"{ter}{kod1}000000"
+        elif kod2 != "000" and kod3 != "000":
+            return f"{ter}{kod1}{kod2}000"
+        else:
+            return ""
+
+    def iter_data(self):
+        self.download()
+        with open(self.csv_path, encoding="cp1251") as f:
+            reader = csv.reader(f, delimiter=";", quotechar='"')
+            for row in reader:
+                ter = row[0]
+                kod1 = row[1]
+                kod2 = row[2]
+                kod3 = row[3]
+                name = " ".join(row[6].split()[1:])
+                short_name = row[6].split()[0]
+                name = " ".join(row[6].split()[1:])
+                short_name = row[6].split()[0]
+                oktmo = f"{ter}{kod1}{kod2}{kod3}"
+                if ter == "04" and self.check_twice_code(ter, kod1, kod2, kod3):
+                    parent = self.parent_level(ter, kod1, kod2, kod3)
+                    parent = "" if parent == oktmo else parent
+                    yield f"{ter}{kod1}{kod2}{kod3}", parent, name, short_name
+                else:
+                    continue

core/etl/models/address.py

+# ----------------------------------------------------------------------
+# AddressModel
+# ----------------------------------------------------------------------
+# Copyright (C) 2021 The NOC Project
+# See LICENSE for details
+# ----------------------------------------------------------------------
+
+# Python modules
+from typing import Optional
+
+# NOC modules
+from .base import BaseModel
+from .typing import Reference
+from .building import Building
+from .street import Street
+
+
+class Address(BaseModel):
+    id: str
+    building: Reference["Building"]
+    street: Reference["Street"]
+    num: int
+    num_letter: Optional[str]

core/etl/models/building.py

+# ----------------------------------------------------------------------
+# BuildingModel
+# ----------------------------------------------------------------------
+# Copyright (C) 2021 The NOC Project
+# See LICENSE for details
+# ----------------------------------------------------------------------
+
+# Python modules
+from typing import Optional
+from datetime import date
+
+# NOC modules
+from .base import BaseModel
+from .typing import Reference
+from .division import Division
+
+
+class Building(BaseModel):
+    id: str
+    oktmo: str
+    adm_division: Reference["Division"]
+    postal_code: Optional[str]
+    start_date: Optional[date]
+    end_date: Optional[date]

core/etl/models/division.py

+# ----------------------------------------------------------------------
+# DivisionModel
+# ----------------------------------------------------------------------
+# Copyright (C) 2021 The NOC Project
+# See LICENSE for details
+# ----------------------------------------------------------------------
+
+# Python modules
+from typing import Optional
+
+# NOC modules
+from .base import BaseModel
+
+
+class Division(BaseModel):
+    id: str
+    name: str
+    start_date: Optional[str]

core/etl/models/street.py

+# ----------------------------------------------------------------------
+# StreetModel
+# ----------------------------------------------------------------------
+# Copyright (C) 2021 The NOC Project
+# See LICENSE for details
+# ----------------------------------------------------------------------
+
+# Python modules
+from typing import Optional
+from datetime import date
+
+# NOC modules
+from .base import BaseModel
+from .typing import Reference
+from .division import Division
+
+
+class Street(BaseModel):
+    id: str
+    parent: Reference["Division"]
+    oktmo: str
+    name: str
+    short_name: str
+    start_date: Optional[date]
+    end_date: Optional[date]

gis/models/address.py

 # ---------------------------------------------------------------------
 # Address object
 # ---------------------------------------------------------------------
-# Copyright (C) 2007-2019 The NOC Project
+# Copyright (C) 2007-2021 The NOC Project
 # See LICENSE for details
 # ---------------------------------------------------------------------
 
 # Third-party modules
 from mongoengine.document import Document
-from mongoengine.fields import StringField, IntField, DictField, BooleanField
+from mongoengine.fields import (
+    StringField,
+    IntField,
+    DictField,
+    BooleanField,
+    ReferenceField,
+    LongField,
+)
 from mongoengine.signals import post_save
 
 # NOC modules
 from noc.core.mongo.fields import PlainReferenceField
 from .street import Street
 from .building import Building
+from noc.main.models.remotesystem import RemoteSystem
+from noc.core.bi.decorator import bi_sync
 
 
+@bi_sync
 class Address(Document):
     meta = {
         "collection": "noc.addresses",
@@ -46,6 +56,12 @@ class Address(Document):
     data = DictField()
 
     is_primary = BooleanField(default=True)
+    # Reference to remote system object has been imported from
+    remote_system = ReferenceField(RemoteSystem)
+    # Object id in remote system
+    remote_id = StringField()
+    # Object id in BI
+    bi_id = LongField(unique=True)
 
     @classmethod
     def update_primary(cls, sender, document, **kwargs):

gis/models/building.py

 # ---------------------------------------------------------------------
 # Building object
 # ---------------------------------------------------------------------
-# Copyright (C) 2007-2019 The NOC Project
+# Copyright (C) 2007-2021 The NOC Project
 # See LICENSE for details
 # ---------------------------------------------------------------------
 
@@ -15,6 +15,8 @@ from mongoengine.fields import (
     EmbeddedDocumentField,
     DictField,
     DateTimeField,
+    LongField,
+    ReferenceField,
 )
 
 # NOC modules
@@ -23,8 +25,11 @@ from noc.core.model.decorator import on_save
 from noc.core.model.decorator import on_delete_check
 from .entrance import Entrance
 from .division import Division
+from noc.main.models.remotesystem import RemoteSystem
+from noc.core.bi.decorator import bi_sync
 
 
+@bi_sync
 @on_save
 @on_delete_check(check=[("gis.Address", "building"), ("inv.CoveredBuilding", "building")])
 class Building(Document):
@@ -69,6 +74,12 @@ class Building(Document):
     # Internal field for sorting
     # Filled by primary address trigger
     sort_order = StringField()
+    # Reference to remote system object has been imported from
+    remote_system = ReferenceField(RemoteSystem)
+    # Object id in remote system
+    remote_id = StringField()
+    # Object id in BI
+    bi_id = LongField(unique=True)
 
     @property
     def primary_address(self):

gis/models/division.py

 # ---------------------------------------------------------------------
 # Division object
 # ---------------------------------------------------------------------
-# Copyright (C) 2007-2020 The NOC Project
+# Copyright (C) 2007-2021 The NOC Project
 # See LICENSE for details
 # ---------------------------------------------------------------------
 
@@ -14,6 +14,8 @@ from mongoengine.fields import (
     DateTimeField,
     IntField,
     ListField,
+    LongField,
+    ReferenceField,
 )
 
 # NOC modules
@@ -21,8 +23,11 @@ from noc.main.models.label import Label
 from noc.core.mongo.fields import PlainReferenceField
 from noc.core.comp import smart_text
 from noc.core.model.decorator import on_delete_check
+from noc.main.models.remotesystem import RemoteSystem
+from noc.core.bi.decorator import bi_sync
 
 
+@bi_sync
 @Label.model
 @on_delete_check(
     check=[("gis.Street", "parent"), ("gis.Division", "parent"), ("gis.Building", "adm_division")]
@@ -52,6 +57,12 @@ class Division(Document):
     #
     start_date = DateTimeField()
     end_date = DateTimeField()
+    # Reference to remote system object has been imported from
+    remote_system = ReferenceField(RemoteSystem)
+    # Object id in remote system
+    remote_id = StringField()
+    # Object id in BI
+    bi_id = LongField(unique=True)
     # Labels
     labels = ListField(StringField())
     effective_labels = ListField(StringField())

gis/models/street.py

 # ---------------------------------------------------------------------
 # Street object
 # ---------------------------------------------------------------------
-# Copyright (C) 2007-2020 The NOC Project
+# Copyright (C) 2007-2021 The NOC Project
 # See LICENSE for details
 # ---------------------------------------------------------------------
 
 # Third-party modules
 from mongoengine.document import Document
-from mongoengine.fields import StringField, DictField, BooleanField, DateTimeField
+from mongoengine.fields import (
+    StringField,
+    DictField,
+    BooleanField,
+    DateTimeField,
+    ReferenceField,
+    LongField,
+)
 
 # NOC modules
 from noc.core.mongo.fields import PlainReferenceField
 from noc.core.model.decorator import on_delete_check
 from noc.core.comp import smart_text
 from .division import Division
+from noc.main.models.remotesystem import RemoteSystem
+from noc.core.bi.decorator import bi_sync
 
 
+@bi_sync
 @on_delete_check(check=[("gis.Address", "street")])
 class Street(Document):
     meta = {
@@ -35,6 +45,12 @@ class Street(Document):
     # Additional data
     # Depends on importer
     data = DictField()
+    # Reference to remote system object has been imported from
+    remote_system = ReferenceField(RemoteSystem)
+    # Object id in remote system
+    remote_id = StringField()
+    # Object id in BI
+    bi_id = LongField(unique=True)
     #
     start_date = DateTimeField()
     end_date = DateTimeField()

ui/web/gis/building/Application.js

 //---------------------------------------------------------------------
 // gis.building application
 //---------------------------------------------------------------------
-// Copyright (C) 2007-2014 The NOC Project
+// Copyright (C) 2007-2021 The NOC Project
 // See LICENSE for details
 //---------------------------------------------------------------------
 console.debug("Defining NOC.gis.building.Application");
@@ -39,6 +39,18 @@ Ext.define("NOC.gis.building.Application", {
                     dataIndex: "homes",
                     width: 75,
                     align: "right"
+                },
+                {
+                    text: __("Remote System"),
+                    dataIndex: "remote_system",
+                    flex: 1,
+                    sort: false
+                },
+                {
+                    text: __("Remote Id"),
+                    dataIndex: "remote_id",
+                    flex: 1,
+                    sort: false
                 }
             ],
             fields: [

ui/web/gis/building/Model.js

 //---------------------------------------------------------------------
 // gis.building Model
 //---------------------------------------------------------------------
-// Copyright (C) 2007-2014 The NOC Project
+// Copyright (C) 2007-2021 The NOC Project
 // See LICENSE for details
 //---------------------------------------------------------------------
 console.debug("Defining NOC.gis.building.Model");
@@ -76,6 +76,14 @@ Ext.define("NOC.gis.building.Model", {
             name: "full_path",
             type: "string",
             persist: false
+        },
+        {
+            name: "remote_system",
+            type: "string"
+        },
+        {
+            name: "remote_id",
+            type: "string"
         }
     ]
 });

ui/web/gis/division/Application.js

 //---------------------------------------------------------------------
 // gis.division application
 //---------------------------------------------------------------------
-// Copyright (C) 2007-2014 The NOC Project
+// Copyright (C) 2007-2021 The NOC Project
 // See LICENSE for details
 //---------------------------------------------------------------------
 console.debug("Defining NOC.gis.division.Application");
@@ -49,6 +49,18 @@ Ext.define("NOC.gis.division.Application", {
                     dataIndex: "full_parent",
                     flex: 1,
                     sort: false
+                },
+                {
+                    text: __("Remote System"),
+                    dataIndex: "remote_system",
+                    flex: 1,
+                    sort: false
+                },
+                {
+                    text: __("Remote Id"),
+                    dataIndex: "remote_id",
+                    flex: 1,
+                    sort: false
                 }
             ],
             fields: [

ui/web/gis/division/Model.js

 //---------------------------------------------------------------------
 // gis.division Model
 //---------------------------------------------------------------------
-// Copyright (C) 2007-2014 The NOC Project
+// Copyright (C) 2007-2021 The NOC Project
 // See LICENSE for details
 //---------------------------------------------------------------------
 console.debug("Defining NOC.gis.division.Model");
@@ -66,6 +66,14 @@ Ext.define("NOC.gis.division.Model", {
         {
             name: "name",
             type: "string"
+        },
+        {
+            name: "remote_system",
+            type: "string"
+        },
+        {
+            name: "remote_id",
+            type: "string"
         }
     ]
 });

ui/web/gis/street/Application.js

 //---------------------------------------------------------------------
 // gis.street application
 //---------------------------------------------------------------------
-// Copyright (C) 2007-2014 The NOC Project
+// Copyright (C) 2007-2021 The NOC Project
 // See LICENSE for details
 //---------------------------------------------------------------------
 console.debug("Defining NOC.gis.street.Application");
@@ -42,6 +42,18 @@ Ext.define("NOC.gis.street.Application", {
                     dataIndex: "full_parent",
                     flex: 1,
                     sort: false
+                },
+                {
+                    text: __("Remote System"),
+                    dataIndex: "remote_system",
+                    flex: 1,
+                    sort: false
+                },
+                {
+                    text: __("Remote Id"),
+                    dataIndex: "remote_id",
+                    flex: 1,
+                    sort: false
                 }
             ],
             fields: [

ui/web/gis/street/Model.js

 //---------------------------------------------------------------------
 // gis.division Model
 //---------------------------------------------------------------------
-// Copyright (C) 2007-2014 The NOC Project
+// Copyright (C) 2007-2021 The NOC Project
 // See LICENSE for details
 //---------------------------------------------------------------------
 console.debug("Defining NOC.gis.street.Model");
@@ -57,6 +57,14 @@ Ext.define("NOC.gis.street.Model", {
         {
             name: "name",
             type: "string"
+        },
+        {
+            name: "remote_system",
+            type: "string"
+        },
+        {
+            name: "remote_id",
+            type: "string"
         }
     ]
 });