Commit 3db5729e authored by Dmitry Volodin's avatar Dmitry Volodin Committed by Andrey Vertiprahov
Browse files

Bye-bye HG

parent fa8231c7
......@@ -14,7 +14,6 @@ COPY . /opt/noc
RUN apk add --update --no-cache \
ca-certificates \
libpq libssh2 \
mercurial \
py-cffi \
py-numpy \
py-pip \
......
......@@ -15,7 +15,6 @@ COPY requirements/test.txt /opt/noc/requirements/test.txt
RUN apk add --update --no-cache \
ca-certificates \
libpq libssh2 \
mercurial \
py-cffi \
py-numpy \
py-pip \
......
## migrations dns:032 and sa:076 looks for this file and fails on no mercurial in image
scripts/migrate-repo
.git/
.docker/
# -*- coding: utf-8 -*-
# ---------------------------------------------------------------------
# Configuration Management Object
# ---------------------------------------------------------------------
# Copyright (C) 2007-2018 The NOC Project
# See LICENSE for details
# ---------------------------------------------------------------------
# Python modules
from __future__ import absolute_import
import os
import datetime
# Django modules
from django.db import models
from django.db.models import Q
# NOC modules
from noc.config import config
from noc.core.fileutils import rewrite_when_differ, read_file, in_dir
from noc.cm.vcs import vcs_registry
from noc.lib.validators import is_int
from noc.main.models.notificationgroup import NotificationGroup
from .objectnotify import ObjectNotify
class Object(models.Model):
class Meta(object):
abstract = True
repo_path = models.CharField("Repo Path", max_length=128, unique=True)
#
last_modified = models.DateTimeField("Last Modified", blank=True, null=True)
#
push_every = models.PositiveIntegerField("Push Every (secs)", default=86400,
blank=True, null=True)
next_push = models.DateTimeField("Next Push", blank=True, null=True)
last_push = models.DateTimeField("Last Push", blank=True, null=True)
#
pull_every = models.PositiveIntegerField("Pull Every (secs)", default=86400,
blank=True, null=True)
next_pull = models.DateTimeField("Next Pull", blank=True, null=True)
last_pull = models.DateTimeField("Last Pull", blank=True,
null=True) # Updated by write() method
def __unicode__(self):
return u"%s/%s" % (self.repo_name, self.repo_path)
@property
def vcs(self):
return vcs_registry.get(self.repo)
def save(self, *args, **kwargs):
if self.repo_path and not in_dir(self.path, self.repo):
raise Exception("Attempting to write outside of repo")
mv = None
if self._get_pk_val():
old = self.__class__.objects.get(pk=self._get_pk_val())
if old.repo_path != self.repo_path and old.repo_path != ".":
mv = (old.repo_path, self.repo_path)
models.Model.save(self, *args, **kwargs)
vcs = self.vcs
if mv is not None and vcs.in_repo(mv[0]):
vcs.mv(mv[0], mv[1])
@property
def repo(self):
return os.path.join(config.get("cm", "repo"), self.repo_name)
@property
def path(self):
return os.path.join(self.repo, self.repo_path)
@property
def in_repo(self):
"""
Check object is in repository
:return: True if object is present, False otherwise
"""
return self.vcs.in_repo(self.repo_path)
def status(self):
return {True: "Ready", False: "Waiting"}[self.in_repo]
def write(self, data):
"""
Write data to repository and commit
:param data:
:return:
"""
path = self.path
if not in_dir(path, self.repo):
raise Exception("Attempting to write outside of repo")
is_new = not os.path.exists(path)
now = datetime.datetime.now()
if rewrite_when_differ(self.path, data):
vcs = self.vcs
if is_new:
vcs.add(self.repo_path)
vcs.commit(self.repo_path)
self.last_modified = now
self.on_object_changed()
self.last_pull = now
self.save()
@property
def data(self):
"""
Return object's content or None, if not present
:return: Object content
"""
return read_file(self.path)
def delete(self):
if os.path.exists(self.repo_path):
self.vcs.rm(self.path)
super(Object, self).delete()
@property
def revisions(self):
"""
Get list of revisions
:return:
"""
return self.vcs.log(self.repo_path)
# Finds revision of the object and returns Revision
def find_revision(self, rev):
assert is_int(rev)
for r in self.revisions:
if r.revision == rev:
return r
raise Exception("Not found")
# Return object's current revision
@property
def current_revision(self):
"""
Get object's current revision
:return:
"""
return self.vcs.get_current_revision(self.repo_path)
def diff(self, rev1, rev2):
return self.vcs.diff(self.repo_path, rev1, rev2)
def get_revision(self, rev):
return self.vcs.get_revision(self.repo_path, rev)
def annotate(self):
return self.vcs.annotate(self.repo_path)
@classmethod
def get_object_class(self, repo):
if repo == "prefix-list":
from .prefixlist import PrefixList
return PrefixList
elif repo == "rpsl":
from .rpsl import RPSL
return RPSL
else:
raise Exception("Invalid repo '%s'" % repo)
@property
def module_name(self):
"""
object._meta.model_name
:return:
"""
return self._meta.module_name
@property
def verbose_name_plural(self):
"""
Shortcut to object._meta.verbose_name_plural
"""
return self._meta.verbose_name_plural
@property
def verbose_name(self):
return self._meta.verbose_name
def get_notification_groups(self, immediately=False, delayed=False):
q = Q(type=self.repo_name)
if immediately:
q &= Q(notify_immediately=True)
if delayed:
q &= Q(notify_delayed=True)
return set(
[n.notification_group for n in ObjectNotify.objects.filter(q)])
def notification_diff(self, old_data, new_data):
return self.diff(old_data, new_data)
def on_object_changed(self):
notification_groups = self.get_notification_groups(immediately=True)
if not notification_groups:
return
revs = self.revisions
now = datetime.datetime.now()
if len(revs) == 1:
# @todo: replace with template
subject = "NOC: Object '%s' was created" % str(self)
message = "The object %s was created at %s\n" % (str(self), now)
message += "Object value follows:\n---------------------------\n%s\n-----------------------\n" % self.data
link = None
else:
diff = self.notification_diff(revs[1], revs[0])
if not diff:
# No significant difference to notify
return
subject = "NOC: Object changed '%s'" % str(self)
message = "The object %s was changed at %s\n" % (str(self), now)
message += "Object changes follows:\n---------------------------\n%s\n-----------------------\n" % diff
link = None
NotificationGroup.group_notify(groups=notification_groups,
subject=subject, body=message,
link=link)
def push(self):
pass
def pull(self):
pass
@classmethod
def global_push(self):
"""
Push all objects of the given type
:return:
"""
pass
@classmethod
def global_pull(self):
"""
Pull all objects of the given type
:return:
"""
pass
def has_access(self, user):
"""
Chech user has permission to access an object
:param user:
:return:
"""
if user.is_superuser:
return True
return False
# -*- coding: utf-8 -*-
# ---------------------------------------------------------------------
# Prefix-List Object
# ---------------------------------------------------------------------
# Copyright (C) 2007-2018 The NOC Project
# See LICENSE for details
# ---------------------------------------------------------------------
# Python modules
from __future__ import absolute_import
import logging
import datetime
import os
# NOC modules
from .object import Object
logger = logging.getLogger(__name__)
class PrefixList(Object):
class Meta(object):
app_label = "cm"
db_table = "cm_prefixlist"
verbose_name = "Prefix List"
verbose_name_plural = "Prefix Lists"
repo_name = "prefix-list"
@classmethod
def build_prefix_lists(cls):
from noc.peer.models.peeringpoint import PeeringPoint
from noc.peer.models.whoiscache import WhoisCache
for pp in PeeringPoint.objects.all():
profile = pp.profile
for name, filter_exp in pp.generated_prefix_lists:
prefixes = WhoisCache.resolve_as_set_prefixes_maxlen(filter_exp)
pl = profile.generate_prefix_list(name, prefixes)
yield (pp, name, pl, prefixes)
@classmethod
def global_pull(cls):
from noc.peer.models.prefixlistcache import PrefixListCache, PrefixListCachePrefix
objects = {}
for o in PrefixList.objects.all():
objects[o.repo_path] = o
c_objects = set() # peering_point/name
logger.debug("PrefixList.global_pull(): building prefix lists")
for peering_point, pl_name, pl, prefixes in cls.build_prefix_lists():
logger.debug(
"PrefixList.global_pull(): writing %s/%s (%d lines)" % (
peering_point.hostname, pl_name, len(pl.split("\n"))
)
)
path = os.path.join(peering_point.hostname, pl_name)
if path in objects:
o = objects[path]
del objects[path]
else:
o = PrefixList(repo_path=path)
o.save()
o.write(pl)
# Populate cache
cname = "%s/%s" % (peering_point.hostname, pl_name)
try:
c = PrefixListCache.objects.get(peering_point=peering_point.id,
name=pl_name)
if c.cmp_prefixes(prefixes):
logger.debug("Updating cache for %s" % cname)
c.changed = datetime.datetime.now()
c.prefixes = [PrefixListCachePrefix(prefix=prefix,
min=min, max=max)
for prefix, min, max in prefixes]
c.save()
except PrefixListCache.DoesNotExist:
logger.debug("Writing cache for %s" % cname)
PrefixListCache(peering_point=peering_point.id,
name=pl_name,
prefixes=[PrefixListCachePrefix(prefix=prefix,
min=min,
max=max)
for prefix, min, max in
prefixes]).save()
c_objects.add("%s/%s" % (peering_point.hostname, pl_name))
# Remove deleted prefix lists
for o in objects.values():
o.delete()
# Remove unused cache entries
for o in PrefixListCache.objects.all():
n = "%s/%s" % (o.peering_point.hostname, o.name)
if n not in c_objects:
o.delete()
# -*- coding: utf-8 -*-
# ---------------------------------------------------------------------
# RPSL Object
# ---------------------------------------------------------------------
# Copyright (C) 2007-2015 The NOC Project
# See LICENSE for details
# ---------------------------------------------------------------------
# Python modules
from __future__ import absolute_import
import os
import logging
# NOC modules
from .object import Object
logger = logging.getLogger(__name__)
class RPSL(Object):
class Meta(object):
app_label = "cm"
db_table = "cm_rpsl"
verbose_name = "RPSL Object"
verbose_name_plural = "RPSL Objects"
repo_name = "rpsl"
@classmethod
def global_pull(cls):
def global_pull_class(name, c, name_fun):
objects = {}
for o in RPSL.objects.filter(
repo_path__startswith=name + os.sep):
objects[o.repo_path] = o
for a in c.objects.all():
if not a.rpsl:
continue
path = os.path.join(name, name_fun(a))
if path in objects:
o = objects[path]
del objects[path]
else:
o = RPSL(repo_path=path)
o.save()
o.write(a.rpsl)
for o in objects.values():
o.delete()
from noc.peer.models.asn import AS
from noc.peer.models.asset import ASSet
from noc.peer.models.peeringpoint import PeeringPoint
from noc.dns.models.dnszone import DNSZone
logger.debug("RPSL.global_pull(): building RPSL")
global_pull_class("inet-rtr", PeeringPoint,
lambda a: a.hostname)
global_pull_class("as", AS, lambda a: "AS%d" % a.asn)
global_pull_class("as-set", ASSet, lambda a: a.name)
global_pull_class("domain", DNSZone, lambda a: a.name)
@classmethod
def global_push(cls):
pass
# -*- coding: utf-8 -*-
# ---------------------------------------------------------------------
# Repo management application
# ---------------------------------------------------------------------
# Copyright (C) 2007-2018 The NOC Project
# See LICENSE for details
# ---------------------------------------------------------------------
# Python modules
import difflib
# Third-party modules
from django.shortcuts import get_object_or_404
from django.utils.html import escape
from pygments.lexers import DiffLexer
from pygments import highlight
# NOc modules
from noc.lib.app.modelapplication import ModelApplication
from noc.lib.app.access import HasPerm
from noc.lib.app.site import URL
from noc.cm.models.object import Object
from noc.lib.highlight import NOCHtmlFormatter
class RepoApplication(ModelApplication):
"""
#
# Repository management application
#
"""
repo = None
def render_content(self, object, content):
"""
##
## Format content for output
##
:param object:
:param content:
:return:
"""
return "<pre>%s</pre>" % escape(content).replace("\n", "<br/>")
def view_change(self, request, object_id, revision=None, format="html"):
if format is None:
format = "html"
# Check format
if format not in ("html", "text"):
return self.response_not_found("Invalid format '%s'" % format)
# Get object
o = get_object_or_404(Object.get_object_class(self.repo), id=int(object_id))
# Check permissions
if not o.has_access(request.user):
return self.response_forbidden("Access Denied")
# Check object in repo
if not o.in_repo:
return self.render(request, "view.html", {"o": o, "r": [], "content": "Object not ready"})
# Find revision
if revision is not None:
try:
r = o.find_revision(revision)
except Exception:
return self.response_not_found("Revision %s is not found" % revision)
else:
r = o.current_revision
# Get content
content = o.get_revision(r)
# Render content
if format == "html":
content = self.render_content(o, content)
return self.render(request, "view.html", {"o": o, "r": r, "content": content})
else:
return self.render_plain_text(content)
view_change.url = [
URL(r"^(?P<object_id>\d+)/$", name="view"),
URL(r"^(?P<object_id>\d+)/(?P<revision>\d+)/$", name="view_revision"),
]
view_change.access = HasPerm("view")
def view_text(self, request, object_id, revision=None):
return self.view_change(request, object_id, revision, format="text")
view_text.url = [
URL(r"^(?P<object_id>\d+)/text/$", name="view_text"),
URL(r"^(?P<object_id>\d+)/(?P<revision>\d+)/text/$", name="view_text_revision"),
]
view_text.access = HasPerm("view")
def view_diff(self, request, object_id, mode="u", r1=None, r2=None):
"""
##
## Render diff form
##
:param request:
:param object_id:
:param mode:
:param r1:
:param r2:
:return:
"""
o = get_object_or_404(Object.get_object_class(self.repo), id=int(object_id))
if not o.has_access(request.user):
return self.response_forbidden("Access denied")
if request.POST:
r1 = request.POST.get("r1", r1)
r2 = request.POST.get("r2", r2)
if r1 and r2:
rev1 = o.find_revision(r1)
rev2 = o.find_revision(r2)
if mode == "2":
d1 = o.get_revision(rev1)
d2 = o.get_revision(rev2)
d = difflib.HtmlDiff()
diff = d.make_table(d1.splitlines(), d2.splitlines())
diff = diff.replace("rules=\"groups\"", "rules=\"none\"", 1) # Use no colgroup rules
else:
diff = o.diff(rev1, rev2)
diff = unicode(diff, "utf8")
diff = highlight(diff, DiffLexer(), NOCHtmlFormatter()) # Highlight diff
return self.render(request, "diff.html", {"o": o, "diff": diff, "r1": r1, "r2": r2, "mode": mode})
else:
return self.response_redirect_to_object(o)
view_diff.url = [
URL(r"^(?P<object_id>\d+)/diff/$", name="diff"),
URL(r"^(?P<object_id>\d+)/diff/(?P<mode>[u2])/(?P<r1>\d+)/(?P<r2>\d+)/$", name="diff_rev")
]
view_diff.access = HasPerm("view")
def view_annotate(self, request, object_id):
"""
:param request:
:param object_id:
:return:
"""
o = get_object_or_404(Object.get_object_class(self.repo), id=int(object_id))
if not o.has_access(request.user):
return self.response_forbidden("Access denied")
# Build annotate styles
last_revision = None
n = -1
annotate = []
for r, t in o.annotate():
if r.revision != last_revision:
n += 1
last_revision = r.revision
annotate += [("row%d" % (n % 2), r, t)]
return self.render(request, "view.html", {"o": o, "annotate": annotate, "r": o.current_revision})
view_annotate.url = r"^(?P<object_id>\d+)/annotate/$"
view_annotate.url_name = "annotate"
view_annotate.access = HasPerm("view")
# -*- coding: utf-8 -*-
# ----------------------------------------------------------------------
# Version Control System support
# ----------------------------------------------------------------------
# Copyright (C) 2007-2017 The NOC Project
# See LICENSE for details
# ----------------------------------------------------------------------
# Python modules
import os
import subprocess
# NOC modules
from noc.lib.registry import Registry
from noc.core.fileutils import copy_file
from noc.config import config
class VCSRegistry(Registry):
"""
Registry for VCS
"""
name = "VCSRegistry"
subdir = "vcs"
classname = "VCS"
def get(self, repo):
return self[config.cm.vcs_type](repo)