Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in / Register
Toggle navigation
Menu
Open sidebar
arthur-zzz
noc
Commits
99d89ff3
Commit
99d89ff3
authored
Jun 19, 2021
by
Vladimir Komarov
Committed by
Andrey Vertiprahov
Jun 19, 2021
Browse files
#1598 Extracting data for Building gis model
parent
56eae3b7
Changes
7
Hide whitespace changes
Inline
Side-by-side
core/etl/extractor/building.py
0 → 100644
View file @
99d89ff3
# ----------------------------------------------------------------------
# FIAS Building Extractor
# ----------------------------------------------------------------------
# Copyright (C) 2021 The NOC Project
# See LICENSE for details
# ----------------------------------------------------------------------
# python modules
import
dbf
import
requests
import
os
from
datetime
import
datetime
from
pathlib
import
Path
from
zipfile
import
ZipFile
,
is_zipfile
# NOC modules
from
.base
import
BaseExtractor
from
..models.building
import
Building
from
noc.core.etl.remotesystem.base
import
BaseRemoteSystem
class
BuildingRemoteSystem
(
BaseRemoteSystem
):
"""
base class
Configuration variables
*FIAS_URL* - url of source FIAS data
*CACHE_PATH* - dir target download files
*REGION* - region code
"""
@
BuildingRemoteSystem
.
extractor
class
BuildingExtractor
(
BaseExtractor
):
"""
Building extractor.
"""
name
=
"building"
model
=
Building
def
__init__
(
self
,
system
,
*
args
,
**
kwargs
):
super
(
BuildingExtractor
,
self
).
__init__
(
system
)
self
.
fias_url
=
str
(
self
.
config
.
get
(
"FIAS_URL"
))
self
.
cache_path
=
str
(
self
.
config
.
get
(
"CACHE_PATH"
))
self
.
region
=
str
(
self
.
config
.
get
(
"REGION"
))
self
.
check_path
(
self
.
cache_path
)
self
.
zip_path
=
os
.
path
.
join
(
self
.
cache_path
,
"fias_dbf.zip"
)
self
.
dbf_file_house
=
f
"HOUSE
{
self
.
region
}
.DBF"
self
.
dbf_file_address
=
f
"ADDROB
{
self
.
region
}
.DBF"
self
.
dbf_path_house
=
os
.
path
.
join
(
self
.
cache_path
,
self
.
dbf_file_house
)
self
.
dbf_path_address
=
os
.
path
.
join
(
self
.
cache_path
,
self
.
dbf_file_address
)
self
.
now
=
datetime
.
now
().
date
()
def
check_path
(
self
,
path
):
# check exists cache_path
dirpath
=
Path
(
path
)
if
not
dirpath
.
exists
()
or
not
dirpath
.
is_dir
():
os
.
makedirs
(
path
)
def
download
(
self
):
if
(
not
os
.
path
.
isfile
(
self
.
zip_path
)
or
datetime
.
fromtimestamp
(
os
.
path
.
getctime
(
self
.
zip_path
)).
date
()
!=
datetime
.
now
().
date
()
):
r
=
requests
.
get
(
self
.
fias_url
,
stream
=
True
)
if
r
.
status_code
==
200
:
with
open
(
self
.
zip_path
,
"wb"
)
as
f
:
for
chunk
in
r
.
iter_content
(
1024
):
f
.
write
(
chunk
)
if
is_zipfile
(
self
.
zip_path
):
with
ZipFile
(
self
.
zip_path
,
"r"
)
as
f
:
f
.
extract
(
self
.
dbf_file_house
,
self
.
cache_path
)
f
.
extract
(
self
.
dbf_file_address
,
self
.
cache_path
)
else
:
raise
Exception
(
"zipfile not found!"
)
def
extract
(
self
):
super
().
extract
()
return
def
get_oktmo_data
(
self
):
oktmo_data
=
{}
with
dbf
.
Table
(
filename
=
self
.
dbf_path_address
,
codepage
=
"cp866"
)
as
table
:
for
r
in
table
:
if
r
.
AOLEVEL
in
[
7
,
4
,
35
]
and
r
.
NEXTID
==
" "
*
36
and
r
.
OKTMO
!=
" "
*
11
:
oktmo
=
r
.
OKTMO
.
rstrip
().
zfill
(
11
)
oktmo_data
[
r
.
AOGUID
]
=
oktmo
return
oktmo_data
def
iter_data
(
self
):
self
.
download
()
oktmo_data
=
self
.
get_oktmo_data
()
with
dbf
.
Table
(
filename
=
self
.
dbf_path_house
,
codepage
=
"cp866"
)
as
table
:
for
r
in
table
:
oktmo
=
oktmo_data
.
get
(
r
.
AOGUID
)
if
r
.
ENDDATE
>=
self
.
now
and
oktmo
:
yield
r
.
HOUSEGUID
,
oktmo
,
r
.
POSTALCODE
,
r
.
STARTDATE
,
r
.
ENDDATE
core/etl/models/building.py
View file @
99d89ff3
...
...
@@ -12,13 +12,15 @@ from datetime import date
# NOC modules
from
.base
import
BaseModel
from
.typing
import
Reference
from
.
division
import
Division
from
.
admdiv
import
AdmDiv
class
Building
(
BaseModel
):
id
:
str
oktmo
:
str
adm_division
:
Reference
[
"Div
ision
"
]
adm_division
:
Reference
[
"
Adm
Div"
]
postal_code
:
Optional
[
str
]
start_date
:
Optional
[
date
]
end_date
:
Optional
[
date
]
_csv_fields
=
[
"id"
,
"oktmo"
,
"adm_division"
,
"postal_code"
,
"start_date"
,
"end_date"
]
core/etl/remotesystem/base.py
View file @
99d89ff3
...
...
@@ -20,6 +20,7 @@ class BaseRemoteSystem(object):
extractors_order
=
[
"label"
,
"admdiv"
,
"building"
,
"networksegmentprofile"
,
"networksegment"
,
"object"
,
...
...
main/models/remotesystem.py
View file @
99d89ff3
...
...
@@ -86,6 +86,7 @@ class RemoteSystem(Document):
enable_admdiv
=
BooleanField
()
enable_administrativedomain
=
BooleanField
()
enable_authprofile
=
BooleanField
()
enable_building
=
BooleanField
()
enable_container
=
BooleanField
()
enable_link
=
BooleanField
()
enable_managedobject
=
BooleanField
()
...
...
requirements.txt
View file @
99d89ff3
...
...
@@ -98,3 +98,5 @@ pytest-dependency==0.5.1; extra == "testing"
Cython
==0.29.21; extra == "cython"
# Kafka
aiokafka
==0.6.0; extra == "sender-kafka"
# etl/extractor
dbf
==0.99.1; extra == "testing"
ui/web/main/remotesystem/Application.js
View file @
99d89ff3
...
...
@@ -73,6 +73,11 @@ Ext.define("NOC.main.remotesystem.Application", {
xtype
:
"
checkbox
"
,
boxLabel
:
__
(
"
Auth Profile
"
)
},
{
name
:
"
enable_building
"
,
xtype
:
"
checkbox
"
,
boxLabel
:
__
(
"
Building
"
)
},
{
name
:
"
enable_container
"
,
xtype
:
"
checkbox
"
,
...
...
ui/web/main/remotesystem/Model.js
View file @
99d89ff3
...
...
@@ -44,6 +44,10 @@ Ext.define("NOC.main.remotesystem.Model", {
name
:
"
enable_authprofile
"
,
type
:
"
boolean
"
},
{
name
:
"
enable_building
"
,
type
:
"
boolean
"
},
{
name
:
"
enable_container
"
,
type
:
"
boolean
"
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment