Skip to content
GitLab
Commit b6c3f6ca authored by Andrey Vertiprahov's avatar Andrey Vertiprahov
Browse files

Merge branch 'noc_dnspython' into 'master' 

noc/noc#1535 Change DIG - > DNSPython

See merge request noc/noc!6220
parents efc60016 2c04ee6d
Hide whitespace changes
Inline Side-by-side
commands/dnszone.py
View file @ b6c3f6ca
# ---------------------------------------------------------------------
# Import DNS Zone
# ---------------------------------------------------------------------
# Copyright (C) 2007-2021 The NOC Project
# Copyright (C) 2007-2022 The NOC Project
# See LICENSE for details
# ---------------------------------------------------------------------
# Python modules
import dns
import argparse
import re
import subprocess
from itertools import zip_longest
# NOC modules
......@@ -20,11 +20,10 @@ from noc.dns.models.dnszoneprofile import DNSZoneProfile
from noc.ip.models.vrf import VRF
from noc.ip.models.addressprofile import AddressProfile
from noc.ip.models.address import Address
from noc.core.validators import is_int
from noc.core.validators import is_int, is_ipv4, is_ipv6
from noc.core.dns.rr import RR
from noc.config import config
from noc.core.text import split_alnum
from noc.core.comp import smart_text, smart_bytes
from noc.core.comp import smart_text
class Command(BaseCommand):
......@@ -72,9 +71,6 @@ class Command(BaseCommand):
dest="transfer_zone",
action="store",
help="DNS Zone name to transfer",
),
import_parser.add_argument(
"--source", dest="source", action="store", help="Source address to issue zone transfer"
)
self.print((import_parser.print_help()))
......@@ -127,7 +123,6 @@ class Command(BaseCommand):
dry_run=False,
zone_profile=None,
address_profile=None,
source=None,
transfer_zone=None,
nameserver=None,
):
......@@ -152,8 +147,6 @@ class Command(BaseCommand):
address_profile=ap,
)
if axfr:
if not source:
self.die("--source is not set")
if not transfer_zone:
self.die("--transfer_zone is not set")
if not nameserver:
......@@ -167,20 +160,22 @@ class Command(BaseCommand):
address_profile=ap,
transfer_zone=transfer_zone,
nameserver=nameserver,
source_address=source,
)
def load_axfr(self, nameserver, transfer_zone, source_address):
opts = []
opts += ["-b", source_address]
pipe = subprocess.Popen(
[config.path.dig] + opts + ["axfr", "@%s" % nameserver, transfer_zone],
shell=False,
stdout=subprocess.PIPE,
).stdout
data = pipe.read()
pipe.close()
return smart_text(smart_bytes(data)).split("\n")
def load_axfr(self, ip, transfer_zone):
try:
_zone = dns.zone.from_xfr(
dns.query.xfr(str(ip).rstrip("."), transfer_zone, lifetime=5.0)
)
data = "\n".join(
_zone[z_node].to_text(z_node)
for z_node in _zone.nodes.keys()
if "@" not in _zone[z_node].to_text(z_node)
)
except dns.exception.DNSException as e:
self.print("ERROR:", e)
return
return data
def dns_zone(self, zone, zone_profile, dry_run=False, clean=False):
z = DNSZone.get_by_name(zone)
......@@ -215,14 +210,13 @@ class Command(BaseCommand):
address_profile=None,
transfer_zone=None,
nameserver=None,
source_address=None,
dry_run=False,
force=False,
clean=False,
):
self.print("Loading zone file '%s'" % path)
self.print("Parsing zone file using BIND parser")
if path:
self.print("Loading zone file '%s'" % path)
self.print("Parsing zone file using BIND parser")
with open(path) as f:
rrs = self.iter_bind_zone_rr(f)
try:
......@@ -291,62 +285,55 @@ class Command(BaseCommand):
if not dry_run:
zrr.save()
if axfr:
data = self.load_axfr(nameserver, transfer_zone, source_address)
self.print("Loading zone: %s by AXFR from server: %s" % (transfer_zone, nameserver))
if not is_ipv4(nameserver) and not is_ipv6(nameserver):
try:
answer = dns.resolver.resolve(qname=nameserver, rdtype="A", lifetime=5.0)
ip = answer[0].address
except dns.exception.DNSException as e:
self.print(f"Resolv Error: {e}")
return
else:
ip = nameserver
print(ip, transfer_zone)
data = self.load_axfr(ip, transfer_zone)
if data is None:
self.print("No result")
return
zone = self.from_idna(transfer_zone)
z = self.dns_zone(zone, zone_profile, dry_run, clean)
# Populate zone
vrf = VRF.get_global()
zz = zone + "."
lz = len(zz)
if z.is_forward:
zp = None
elif z.is_reverse_ipv4:
# Calculate prefix for reverse zone
zp = ".".join(reversed(zone[:-13].split("."))) + "."
elif z.is_reverse_ipv6:
raise CommandError("IPv6 reverse import is not implemented")
else:
if not z.is_forward and not z.is_reverse_ipv4 and not z.is_reverse_ipv6:
raise CommandError("Unknown zone type")
for row in data:
row = row.strip()
if row == "" or row.startswith(";"):
continue
row = row.split()
if len(row) != 5 or row[2] != "IN" or row[3] not in ("A", "AAAA", "PTR"):
for row in data.splitlines():
row = row.strip().split()
if len(row) != 5 or row[3] not in ("A", "AAAA", "PTR"):
continue
if row[3] in ("A", "AAAA"):
name = row[0]
if name.endswith(zz):
name = name[:-lz]
if name.endswith("."):
name = name[:-1]
self.create_address(
zone,
vrf,
row[4],
"%s.%s" % (name, zone) if name else zone,
address_profile,
dry_run=dry_run,
force=force,
)
if row[3] == "PTR":
name = row[4]
if name.endswith(zz):
name = name[:-lz]
if name.endswith("."):
name = name[:-1]
# @todo: IPv6
if "." in row[0]:
address = ".".join(reversed(row[0].split(".")[:-3]))
else:
address = zp + name
host = dns.name.from_text(f"{row[0]}.{zone}.")
ip = dns.reversename.to_address(host)
fqdn = row[4]
if fqdn.endswith("."):
fqdn = fqdn[:-1]
self.create_address(
zone, vrf, address, fqdn, address_profile, dry_run=dry_run, force=force
)
elif row[3] in ("A", "AAAA"):
fqdn = row[0]
if fqdn.endswith(zz):
fqdn = fqdn[:-lz]
if fqdn.endswith("."):
fqdn = fqdn[:-1]
ip = row[4]
else:
continue
self.create_address(
zone,
vrf,
ip,
fqdn,
address_profile,
dry_run=dry_run,
force=force,
)
def create_address(self, zone, vrf, address, fqdn, address_profile, dry_run=False, force=False):
"""
......
requirements.txt
View file @ b6c3f6ca
......@@ -34,6 +34,7 @@ cachetools==4.2.4
crontab==0.22.9
csiphash==0.0.5
demjson3==3.0.5
dnspython==2.2.1
fs==2.4.13
geojson==2.5.0
geopy==2.2.0
......
services/web/apps/ip/ipam/views.py
View file @ b6c3f6ca
......@@ -324,7 +324,7 @@ class IPAMApplication(ExtApplication):
[
{"address": z[0], "isFree": True}
for z in spot
if str(z[0]) not in allocated_addresses
if str(z[0]) not in allocated_addresses and z[0] is not None
]
if spot
else []
......
services/web/apps/ip/tools/views.py
View file @ b6c3f6ca
# ---------------------------------------------------------------------
# Tools
# ---------------------------------------------------------------------
# Copyright (C) 2007-2020 The NOC Project
# Copyright (C) 2007-2022 The NOC Project
# See LICENSE for details
# ---------------------------------------------------------------------
# Python Modules
import csv
import subprocess
import dns
import orjson
from io import StringIO
# Third-party modules
from django import forms
from django.http import HttpResponse
# NOC Modules
from noc.lib.app.application import Application, HasPerm, view
from noc.core.ip import IP, IPv4
from noc.core.ip import IP, IPv4, IPv6
from noc.core.validators import is_ipv4, is_ipv6
from noc.ip.models.address import Address
from noc.ip.models.prefix import Prefix
from noc.ip.models.vrf import VRF
from noc.core.forms import NOCForm
from noc.config import config
# from noc.core.comp import smart_text
from noc.ip.models.addressprofile import AddressProfile
from noc.core.translation import ugettext as _
......@@ -100,12 +105,6 @@ class ToolsAppplication(Application):
help_text=_("Name server IP address. NS must have zone transfer enabled for NOC host"),
)
zone = forms.CharField(label=_("Zone"), help_text=_("DNS Zone name to transfer"))
source_address = forms.GenericIPAddressField(
label=_("Source Address"),
required=False,
protocol="IPv4",
help_text=_("Source address to issue zone transfer"),
)
@view(
url=r"^(?P<vrf_id>\d+)/(?P<afi>[46])/(?P<prefix>\S+)/upload_axfr/$",
......@@ -122,63 +121,101 @@ class ToolsAppplication(Application):
:return:
"""
def upload_axfr(data):
def upload_axfr(data, zone):
p = IP.prefix(prefix.prefix)
count = 0
for row in data:
row = row.strip()
if row == "" or row.startswith(";"):
continue
row = row.split()
if len(row) != 5 or row[2] != "IN" or row[3] != "PTR":
create = 0
change = 0
zz = zone + "."
lz = len(zz)
ap = AddressProfile.objects.filter(name="default").first()
for row in data.splitlines():
row = row.strip().split()
if len(row) != 5 or row[3] not in ("A", "AAAA", "PTR"):
continue
if row[3] == "PTR":
# @todo: IPv6
x = row[0].split(".")
ip = "%s.%s.%s.%s" % (x[3], x[2], x[1], x[0])
host = dns.name.from_text(f"{row[0]}.{zone}.")
ip = dns.reversename.to_address(host)
fqdn = row[4]
if fqdn.endswith("."):
fqdn = fqdn[:-1]
elif row[3] in ("A", "AAAA"):
fqdn = row[0]
if fqdn.endswith(zz):
fqdn = fqdn[:-lz]
if fqdn.endswith("."):
fqdn = fqdn[:-1]
ip = row[4]
# Leave only addresses residing into "prefix"
# To prevent uploading to not-owned blocks
if not p.contains(IPv4(ip)):
if (
is_ipv4(ip)
and not p.contains(IPv4(ip))
or is_ipv6(ip)
and not p.contains(IPv6(ip))
):
continue
a, changed = Address.objects.get_or_create(vrf=vrf, afi=afi, address=ip)
if a.fqdn != fqdn:
a.fqdn = fqdn
changed = True
if changed:
a = Address.objects.filter(vrf=vrf, afi=afi, address=ip).first()
if a:
if a.fqdn != fqdn:
a.fqdn = fqdn
a.name = fqdn
a.save()
change += 1
else:
# Not found
a = Address(
vrf=vrf,
afi=afi,
address=ip,
profile=ap,
fqdn=fqdn,
name=fqdn,
description="Imported from %s zone" % zone,
)
a.save()
count += 1
return count
create += 1
return create, change
vrf = self.get_object_or_404(VRF, id=int(vrf_id))
prefix = self.get_object_or_404(Prefix, vrf=vrf, afi=afi, prefix=prefix)
if not prefix.can_change(request.user):
return self.response_forbidden(_("Permission denined"))
if request.POST:
form = self.AXFRForm(request.POST)
if form.is_valid():
opts = []
if form.cleaned_data["source_address"]:
opts += ["-b", form.cleaned_data["source_address"]]
pipe = subprocess.Popen(
[config.path.dig]
+ opts
+ ["axfr", "@%s" % form.cleaned_data["ns"], form.cleaned_data["zone"]],
shell=False,
stdout=subprocess.PIPE,
).stdout
data = pipe.read()
pipe.close()
count = upload_axfr(data.split("\n"))
self.message_user(
request,
_("%(count)s IP addresses uploaded via zone transfer") % {"count": count},
)
return self.response_redirect("ip:ipam:vrf_index", vrf.id, afi, prefix.prefix)
body = orjson.loads(request.body)
if not is_ipv4(body["ns"]) and not is_ipv6(body["ns"]):
try:
answer = dns.resolver.resolve(qname=body["ns"], rdtype="A", lifetime=5.0)
ip = answer[0].address
except dns.exception.DNSException as e:
self.error(f"Resolv Error: {e}")
return HttpResponse(e, status=500)
else:
form = self.AXFRForm()
return self.render(
request, "index.html", vrf=vrf, afi=afi, prefix=prefix, upload_ips_axfr_form=form
)
ip = body["ns"]
try:
_zone = dns.zone.from_xfr(
dns.query.xfr(
ip,
body["zone"],
lifetime=5.0,
)
)
data = "\n".join(
_zone[z_node].to_text(z_node)
for z_node in _zone.nodes.keys()
if "@" not in _zone[z_node].to_text(z_node)
)
except dns.exception.DNSException as e:
self.error(f"DNS Error: {e}")
return HttpResponse(e, status=400)
except Exception as e:
self.error(f"Other Error: {e}")
return HttpResponse(e, status=500)
if data:
create, change = upload_axfr(data, body["zone"])
return HttpResponse(
_(
"Created: %(create)s and Changed: %(change)s IP addresses uploaded via zone transfer."
)
% {"create": create, "change": change}
)
return HttpResponse("No DNS Zone", status=404)
ui/web/ip/ipam/Application.js
View file @ b6c3f6ca
......@@ -20,6 +20,7 @@ Ext.define("NOC.ip.ipam.Application", {
"NOC.ip.ipam.view.forms.prefix.PrefixDeletePanel",
"NOC.ip.ipam.view.forms.prefix.RebasePanel",
"NOC.ip.ipam.view.forms.prefix.AddressPanel",
"NOC.ip.ipam.view.forms.tools.ToolsForm",
"NOC.ip.ipam.ApplicationModel",
"NOC.ip.ipam.ApplicationController"
],
......@@ -58,6 +59,7 @@ Ext.define("NOC.ip.ipam.Application", {
ipIPAMVRFListOpen: "openVRFList",
ipIPAMAddressFormEdit: "onAddressFormEdit",
ipIPAMAddressFormNew: "onAddressFormNew",
ipIPAMToolsFormOpen: "onToolsFormOpen",
}
},
{
......@@ -88,6 +90,13 @@ Ext.define("NOC.ip.ipam.Application", {
listeners: {
ipIPAMRebaseCloseForm: "onRebaseFormClose"
}
},
{
itemId: "ipam-tools-form",
xtype: "ip.ipam.form.tools",
listeners: {
ipIPAMToolsFormClose: "onToolsFormClose"
}
}
]
});
\ No newline at end of file
});
ui/web/ip/ipam/ApplicationController.js
View file @ b6c3f6ca
......@@ -74,6 +74,16 @@ Ext.define("NOC.ip.ipam.ApplicationController", {
onVRFFormClose: function() {
this.openVRFList();
},
onToolsFormOpen: function(parentForm, param) {
var form = this.getView().down("[itemId=ipam-tools-form]");
form.getViewModel().set("prefix", param.prefix);
this.getViewModel().set("activeItem", "ipam-tools-form");
},
onToolsFormClose: function(form) {
var prefix = form.getViewModel().get("prefix");
this.openPrefixContents("contents/" + prefix.id + "/");
},
onPrefixContentsOpen: function(grid, params) {
var url = "contents/" + params.id + "/";
if(params.hasOwnProperty("afi")) {
......@@ -140,7 +150,7 @@ Ext.define("NOC.ip.ipam.ApplicationController", {
scope: this,
success: function(response) {
var value = Ext.decode(response.responseText);
if(value.hasOwnProperty("state")){
if(value.hasOwnProperty("state")) {
value.state = {
value: value.state,
label: value.state__label,
......@@ -175,4 +185,4 @@ Ext.define("NOC.ip.ipam.ApplicationController", {
this.setQuery();
}
}
});
\ No newline at end of file
});
ui/web/ip/ipam/view/forms/prefix/Prefix.js
View file @ b6c3f6ca
......@@ -86,11 +86,11 @@ Ext.define("NOC.ip.ipam.view.forms.prefix.Prefix", {
disabled: "{!prefix.permissions.change}"
}
},
// {
// text: __("Tools"),
// tooltip: __("Open tools"),
// glyph: NOC.glyph.edit,
// handler: "onTools",
// }
{
text: __("Tools"),
tooltip: __("Open tools"),
glyph: NOC.glyph.edit,
handler: "onTools",
}
]
});
ui/web/ip/ipam/view/forms/prefix/PrefixController.js
View file @ b6c3f6ca
......@@ -13,9 +13,10 @@ Ext.define("NOC.ip.ipam.view.forms.prefix.PrefixController", {
onClose: function() {
this.fireViewEvent("ipIPAMPrefixListClose");
},
// onTools: function() {
// console.warn("not implemented");
// }
onTools: function() {
var prefix = this.getViewModel().get("prefix");
this.fireViewEvent("ipIPAMToolsFormOpen", {prefix});
},
onAddAddress: function() {
this.fireViewEvent("ipIPAMAddressFormNew", {address: "create_new"});
},
......
ui/web/ip/ipam/view/forms/tools/ToolsController.js 0 → 100644
View file @ b6c3f6ca
//---------------------------------------------------------------------
// ip.ipam.tools form controller
//---------------------------------------------------------------------
// Copyright (C) 2007-2022 The NOC Project
// See LICENSE for details
//---------------------------------------------------------------------
console.debug("Defining NOC.ip.ipam.view.forms.tools.ToolsController");
Ext.define("NOC.ip.ipam.view.forms.tools.ToolsController", {
extend: "Ext.app.ViewController",
alias: "controller.ip.ipam.form.tools",
//
onClose: function() {
this.fireViewEvent("ipIPAMToolsFormClose");
},
//
onDownload: function() {
var prefix = this.getViewModel().get("prefix");
Ext.Ajax.request({
url: "/ip/tools/" + prefix.vrf + "/" + prefix.afi + "/" + prefix.name + "/download_ip/",
method: "POST",
success: function(response) {
var blob = new Blob([response.responseText], {type: "text/plain;charset=utf-8"});
saveAs(blob, "ips.csv");
},
failure: function(r) {
var msg = r.responseText || r.statusText;
NOC.error(msg);
}
});
},
//
onStartZoneTransfer: function() {
var model = this.getViewModel(),
prefix = model.get("prefix"),
data = {
ns: model.get("ns"),
zone: model.get("zone"),
};
Ext.apply(data);
Ext.Ajax.request({
url: "/ip/tools/" + prefix.vrf + "/" + prefix.afi + "/" + prefix.name + "/upload_axfr/",
method: "POST",
jsonData: data,
success: function(r) {
var msg = r.responseText || r.statusText;
NOC.info(msg);
},
failure: function(r) {
var msg = r.responseText || r.statusText;
NOC.error(msg);
}
});
}
});
ui/web/ip/ipam/view/forms/tools/ToolsForm.js 0 → 100644
View file @ b6c3f6ca
//---------------------------------------------------------------------
// ip.ipam.tools form
//---------------------------------------------------------------------
// Copyright (C) 2007-2022 The NOC Project
// See LICENSE for details
//---------------------------------------------------------------------
console.debug("Defining NOC.ip.ipam.view.forms.tools.ToolsForm");
Ext.define("NOC.ip.ipam.view.forms.tools.ToolsForm", {
extend: "Ext.form.Panel",
alias: "widget.ip.ipam.form.tools",
controller: "ip.ipam.form.tools",
viewModel: "ip.ipam.form.tools",
requires: [
"NOC.core.ComboBox",
"NOC.ip.ipam.view.forms.tools.ToolsController",
"NOC.ip.ipam.view.forms.tools.ToolsModel",
"NOC.dns.dnszone.LookupField",