Skip to content
GitLab
Menu
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in / Register
Toggle navigation
Menu
Open sidebar
arthur-zzz
noc
Commits
cc2cad7a
Commit
cc2cad7a
authored
Jun 19, 2021
by
Vladimir Komarov
Committed by
Andrey Vertiprahov
Jun 19, 2021
Browse files
#1598 Extracting data for street gis model
parent
56eae3b7
Changes
7
Hide whitespace changes
Inline
Side-by-side
core/etl/extractor/street.py
0 → 100644
View file @
cc2cad7a
# ----------------------------------------------------------------------
# FIAS Street Extractor
# ----------------------------------------------------------------------
# Copyright (C) 2021 The NOC Project
# See LICENSE for details
# ----------------------------------------------------------------------
# python modules
import
dbf
import
requests
import
os
from
datetime
import
datetime
from
pathlib
import
Path
from
zipfile
import
ZipFile
,
is_zipfile
# NOC modules
from
.base
import
BaseExtractor
from
..models.street
import
Street
from
noc.core.etl.remotesystem.base
import
BaseRemoteSystem
class
StreetRemoteSystem
(
BaseRemoteSystem
):
"""
base class
Configuration variables
*FIAS_URL* - url of source FIAS data
*CACHE_PATH* - dir target download files
*REGION* - region code
"""
@
StreetRemoteSystem
.
extractor
class
StreetExtractor
(
BaseExtractor
):
"""
Street extractor.
"""
name
=
"street"
model
=
Street
def
__init__
(
self
,
system
,
*
args
,
**
kwargs
):
super
(
StreetExtractor
,
self
).
__init__
(
system
)
self
.
fias_url
=
str
(
self
.
config
.
get
(
"FIAS_URL"
))
self
.
cache_path
=
str
(
self
.
config
.
get
(
"CACHE_PATH"
))
self
.
region
=
str
(
self
.
config
.
get
(
"REGION"
))
self
.
check_path
(
self
.
cache_path
)
self
.
zip_path
=
os
.
path
.
join
(
self
.
cache_path
,
"fias_dbf.zip"
)
self
.
dbf_file
=
f
"ADDROB
{
self
.
region
}
.DBF"
self
.
dbf_path
=
os
.
path
.
join
(
self
.
cache_path
,
self
.
dbf_file
)
def
check_path
(
self
,
path
):
# check exists cache_path
dirpath
=
Path
(
path
)
if
not
dirpath
.
exists
()
or
not
dirpath
.
is_dir
():
os
.
makedirs
(
path
)
def
download
(
self
):
if
(
not
os
.
path
.
isfile
(
self
.
zip_path
)
or
datetime
.
fromtimestamp
(
os
.
path
.
getctime
(
self
.
zip_path
)).
date
()
!=
datetime
.
now
().
date
()
):
r
=
requests
.
get
(
self
.
fias_url
,
stream
=
True
)
if
r
.
status_code
==
200
:
with
open
(
self
.
zip_path
,
"wb"
)
as
f
:
for
chunk
in
r
.
iter_content
(
1024
):
f
.
write
(
chunk
)
if
is_zipfile
(
self
.
zip_path
):
with
ZipFile
(
self
.
zip_path
,
"r"
)
as
f
:
f
.
extract
(
self
.
dbf_file
,
self
.
cache_path
)
else
:
raise
Exception
(
"zipfile not found!"
)
def
extract
(
self
):
super
().
extract
()
return
def
get_tables
(
self
):
field_specs
=
(
"OKTMO C(11); AOLEVEL N(3,0); SHORTNAME C(10); FORMALNAME C(120);"
"AOGUID C(36); PARENTGUID C(36); STARTDATE D; ENDDATE D"
)
tmp
=
dbf
.
Table
(
filename
=
"tmp"
,
codepage
=
"cp866"
,
field_specs
=
field_specs
,
on_disk
=
False
)
cities
=
dbf
.
Table
(
filename
=
"tmp"
,
codepage
=
"cp866"
,
field_specs
=
field_specs
,
on_disk
=
False
)
streets
=
dbf
.
Table
(
filename
=
"tmp"
,
codepage
=
"cp866"
,
field_specs
=
field_specs
,
on_disk
=
False
)
with
dbf
.
Table
(
filename
=
self
.
dbf_path
,
codepage
=
"cp866"
)
as
table
:
with
tmp
:
for
row
in
table
:
if
(
row
.
AOLEVEL
in
[
7
,
4
,
35
]
and
row
.
NEXTID
==
" "
*
36
and
row
.
OKTMO
!=
" "
*
11
):
OKTMO
=
row
.
OKTMO
.
rstrip
().
zfill
(
11
)
tmp
.
append
(
(
OKTMO
,
row
.
AOLEVEL
,
row
.
SHORTNAME
,
row
.
FORMALNAME
,
row
.
AOGUID
,
row
.
PARENTGUID
,
row
.
STARTDATE
,
row
.
ENDDATE
,
)
)
cities
.
open
(
dbf
.
READ_WRITE
)
streets
.
open
(
dbf
.
READ_WRITE
)
with
tmp
:
for
record
in
tmp
:
if
record
.
AOLEVEL
in
[
7
]:
streets
.
append
(
record
)
else
:
cities
.
append
(
record
)
return
cities
,
streets
def
get_parent
(
self
,
table
,
id
):
table
.
top
()
for
r
in
table
:
if
id
==
r
.
AOGUID
:
return
r
return
None
def
iter_data
(
self
):
self
.
download
()
cities
,
streets
=
self
.
get_tables
()
for
r
in
streets
:
parent
=
self
.
get_parent
(
cities
,
r
.
PARENTGUID
)
if
parent
:
yield
r
.
AOGUID
,
parent
.
OKTMO
,
r
.
FORMALNAME
.
rstrip
(),
r
.
SHORTNAME
.
rstrip
(),
r
.
STARTDATE
,
r
.
ENDDATE
cities
.
close
()
streets
.
close
()
core/etl/models/street.py
View file @
cc2cad7a
...
...
@@ -12,14 +12,15 @@ from datetime import date
# NOC modules
from
.base
import
BaseModel
from
.typing
import
Reference
from
.
division
import
Division
from
.
admdiv
import
AdmDiv
class
Street
(
BaseModel
):
id
:
str
parent
:
Reference
[
"Division"
]
oktmo
:
str
parent
:
Reference
[
"AdmDiv"
]
name
:
str
short_name
:
str
start_date
:
Optional
[
date
]
end_date
:
Optional
[
date
]
_csv_fields
=
[
"id"
,
"parent"
,
"name"
,
"short_name"
,
"start_date"
,
"end_date"
]
core/etl/remotesystem/base.py
View file @
cc2cad7a
...
...
@@ -36,6 +36,7 @@ class BaseRemoteSystem(object):
"subscriber"
,
"serviceprofile"
,
"service"
,
"street"
,
]
def
__init__
(
self
,
remote_system
):
...
...
main/models/remotesystem.py
View file @
cc2cad7a
...
...
@@ -95,6 +95,7 @@ class RemoteSystem(Document):
enable_object
=
BooleanField
()
enable_service
=
BooleanField
()
enable_serviceprofile
=
BooleanField
()
enable_street
=
BooleanField
()
enable_subscriber
=
BooleanField
()
enable_subscriberprofile
=
BooleanField
()
enable_resourcegroup
=
BooleanField
()
...
...
requirements.txt
View file @
cc2cad7a
...
...
@@ -98,3 +98,5 @@ pytest-dependency==0.5.1; extra == "testing"
Cython
==0.29.21; extra == "cython"
# Kafka
aiokafka
==0.6.0; extra == "sender-kafka"
# etl/extractor
dbf
==0.99.1; extra == "testing"
ui/web/main/remotesystem/Application.js
View file @
cc2cad7a
...
...
@@ -118,6 +118,11 @@ Ext.define("NOC.main.remotesystem.Application", {
xtype
:
"
checkbox
"
,
boxLabel
:
__
(
"
Service Profile
"
)
},
{
name
:
"
enable_street
"
,
xtype
:
"
checkbox
"
,
boxLabel
:
__
(
"
Street
"
)
},
{
name
:
"
enable_subscriber
"
,
xtype
:
"
checkbox
"
,
...
...
ui/web/main/remotesystem/Model.js
View file @
cc2cad7a
...
...
@@ -103,6 +103,10 @@ Ext.define("NOC.main.remotesystem.Model", {
{
name
:
"
enable_label
"
,
type
:
"
boolean
"
},
{
name
:
"
enable_street
"
,
type
:
"
boolean
"
}
]
});
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment