Verified Commit 36a3ab15 authored by Aleksey Shirokih's avatar Aleksey Shirokih
Browse files

Merge branch 'master' of code.getnoc.com:noc/noc

parents 03b26995 be7bd794
......@@ -6,6 +6,10 @@ services:
command: --wiredTigerCacheSizeGB 1.5 --bind_ip_all --replSet noc --port 27017
volumes:
- ./files/mongo-init.js:/docker-entrypoint-initdb.d/01mongo-init.js
- type: tmpfs
target: /data/db
tmpfs:
size: "512M"
environment:
MONGO_INITDB_DATABASE: noc
MONGO_INITDB_ROOT_USERNAME: noc
......@@ -36,6 +40,11 @@ services:
POSTGRES_USER: noc
POSTGRES_DB: noc
POSTGRES_PASSWORD: noc
volumes:
- type: tmpfs
target: /data
tmpfs:
size: "512M"
command: -c fsync=off
healthcheck:
test: ["CMD", "su", "-", "postgres", "-c", "psql -U noc -l | grep noc"]
......
......@@ -7,12 +7,24 @@ stages:
- Build additional release containers
- Upload docs
check labels:
stage: lint
image: registry.getnoc.com/infrastructure/noc-lint:master
script:
- set -x
- FLIST=$(git --no-pager diff --name-only $CI_COMMIT_SHA $(git merge-base origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME $CI_COMMIT_SHA))
- python ./scripts/check-labels.py $FLIST
only:
- merge_requests
tags:
- docker
flake8:
stage: lint
image: registry.getnoc.com/infrastructure/noc-lint:master
script:
- set -x
- FLIST=$(git --no-pager diff --name-only $(git branch -q -r --contains) $(git merge-base origin/HEAD HEAD) |egrep ".py$" | grep -v ".docker/"|| true)
- FLIST=$(git --no-pager diff --name-only $CI_COMMIT_SHA $(git merge-base origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME $CI_COMMIT_SHA) | egrep ".py$" | grep -v ".docker/"|| true)
- >
if [ ! -z "$FLIST" ];
then
......@@ -21,28 +33,30 @@ flake8:
echo "No files to lint"
true;
fi
except:
- master
only:
- merge_requests
tags:
- docker
yapf:
black:
stage: lint
image: registry.getnoc.com/infrastructure/noc-lint:master
image: registry.getnoc.com/infrastructure/black:master
script:
- set -x
- FLIST=$(git --no-pager diff --name-only $(git branch -q -r --contains) $(git merge-base origin/HEAD HEAD) |egrep ".py$" || true)
- FLIST=$(git --no-pager diff --name-only $CI_COMMIT_SHA $(git merge-base origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME $CI_COMMIT_SHA) | egrep ".py$" | grep -v ".docker/"|| true)
- >
if [ ! -z "$FLIST" ];
then
yapf -d -vv $FLIST;
black --check $FLIST;
else
echo "No files to lint"
true;
fi
only:
- merge_requests
tags:
- docker
allow_failure: true
allow_failure: false
# waiting for https://github.com/rubik/xenon/issues/21
xenon:
......@@ -50,7 +64,7 @@ xenon:
image: registry.getnoc.com/infrastructure/noc-lint:master
script:
- set -x
- FLIST=$(git --no-pager diff --name-only $(git branch -q -r --contains) $(git merge-base origin/HEAD HEAD) |egrep ".py$" || true)
- FLIST=$(git --no-pager diff --name-only $CI_COMMIT_SHA $(git merge-base origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME $CI_COMMIT_SHA) | egrep ".py$" | grep -v ".docker/"|| true)
- x=0
- >
if [ ! -z "$FLIST" ];
......@@ -61,8 +75,8 @@ xenon:
done;
fi
- if [ "$x" -gt 0 ] ; then exit 1; fi
except:
- master
only:
- merge_requests
tags:
- docker
allow_failure: true
......@@ -72,7 +86,7 @@ pylint:
image: registry.getnoc.com/infrastructure/noc-lint:master
script:
- set -x
- FLIST=$(git --no-pager diff --name-only $(git branch -q -r --contains) $(git merge-base origin/HEAD HEAD) |egrep ".py$" || true)
- FLIST=$(git --no-pager diff --name-only $CI_COMMIT_SHA $(git merge-base origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME $CI_COMMIT_SHA) | egrep ".py$" | grep -v ".docker/"|| true)
- >
if [ ! -z "$FLIST" ];
then
......@@ -81,8 +95,8 @@ pylint:
echo "No files to lint"
true;
fi
except:
- master
only:
- merge_requests
tags:
- docker
......@@ -90,10 +104,10 @@ futurize:
stage: lint
image: registry.getnoc.com/infrastructure/noc-lint:master
script:
- FLIST=$(git --no-pager diff --name-only $(git branch -q -r --contains) $(git merge-base origin/HEAD HEAD) |egrep ".py$" | grep -v ".docker/" || true)
- FLIST=$(git --no-pager diff --name-only $CI_COMMIT_SHA $(git merge-base origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME $CI_COMMIT_SHA) | egrep ".py$" | grep -v ".docker/"|| true)
- /bin/future.sh "$FLIST"
except:
- master
only:
- merge_requests
tags:
- docker
......@@ -101,10 +115,10 @@ futurize stage2:
stage: lint
image: registry.getnoc.com/infrastructure/noc-lint:master
script:
- FLIST=$(git --no-pager diff --name-only $(git branch -q -r --contains) $(git merge-base origin/HEAD HEAD) |egrep ".py$" | grep -v ".docker/" || true)
- FLIST=$(git --no-pager diff --name-only $CI_COMMIT_SHA $(git merge-base origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME $CI_COMMIT_SHA) | egrep ".py$" | grep -v ".docker/"|| true)
- /bin/future2.sh "$FLIST"
except:
- master
only:
- merge_requests
tags:
- docker
allow_failure: true
......@@ -134,6 +148,11 @@ Test migrate:
-f .docker/docker-compose.yml
-p "${DOCKER_COMPOSE_PROJECT}-migrate"
down -v
only:
refs:
- merge_requests
- master
- /^release-\d+\.\d+/
tags:
- shell
allow_failure: false
......@@ -162,6 +181,11 @@ Test code:
-f .docker/docker-compose.yml
-p "${DOCKER_COMPOSE_PROJECT}-tests"
down -v
only:
refs:
- merge_requests
- master
- /^release-\d+\.\d+/
tags:
- shell
allow_failure: false
......@@ -214,6 +238,25 @@ Build docs:
changes:
- docs/**
Build Master Docs:
stage: Build docs
environment: docs
image: registry.getnoc.com/infrastructure/docs:master
script:
- cd docs/
- sphinx-build -a -E -b html src/en ../build/docs/en
- python ./scripts/build-go.py ../build/docs/en/objects.inv
tags:
- docker
artifacts:
paths:
- build/docs/en
expire_in: 1 day
only:
refs:
- master
- /^release-\d+\.\d+/
Build release image:
stage: Build release container
before_script:
......
......@@ -43,9 +43,9 @@ class Permission(NOCModel):
implied = CharField(
"Implied", max_length=256, null=True, blank=True)
users = ManyToManyField(
User, related_name="noc_user_permissions")
User, related_name="permissions")
groups = ManyToManyField(
Group, related_name="noc_group_permissions")
Group, related_name="permissions")
_id_cache = cachetools.TTLCache(maxsize=100, ttl=60)
_name_cache = cachetools.TTLCache(maxsize=100, ttl=60)
......@@ -92,8 +92,7 @@ class Permission(NOCModel):
"""
Return a set of user permissions
"""
return set(user.noc_user_permissions.values_list("name",
flat=True))
return set(user.permissions.values_list("name", flat=True))
@classmethod
def set_user_permissions(cls, user, perms):
......@@ -127,8 +126,7 @@ class Permission(NOCModel):
"""
Get set of group permissions
"""
return set(group.noc_group_permissions.values_list("name",
flat=True))
return set(group.permissions.values_list("name", flat=True))
@classmethod
def set_group_permissions(cls, group, perms):
......@@ -167,13 +165,13 @@ class Permission(NOCModel):
"name", flat=True))
perms = set()
# User permissions
for p in user.noc_user_permissions.all():
for p in user.permissions.all():
perms.add(p.name)
if p.implied:
perms.update(p.implied.split(","))
# Group permissions
for g in user.groups.all():
for p in g.noc_group_permissions.all():
for p in g.permissions.all():
perms.add(p.name)
if p.implied:
perms.update(p.implied.split(","))
......
......@@ -2,14 +2,16 @@
# ----------------------------------------------------------------------
# MAC Model
# ----------------------------------------------------------------------
# Copyright (C) 2007-2017 The NOC Project
# Copyright (C) 2007-2019 The NOC Project
# See LICENSE for details
# ----------------------------------------------------------------------
# Python modules
from collections import defaultdict
import datetime
# Third-party modules
import six
from six.moves import zip
# NOC modules
from noc.config import config
from noc.core.clickhouse.model import Model
......
{% extends "template.html" %}
{% load cmtags %}
{% block extrahead %}
<link rel="stylesheet" type="text/css" href="/ui/web/css/diff.css" />
<link rel="stylesheet" type="text/css" href="/ui/web/css/pygments.css" />
<link rel="stylesheet" type="text/css" href="/ui/web/css/highlight.css" />
{% endblock %}
{% block breadcrumbs %}{{block.super}}<li>Diff</li>{%endblock%}
{% block content %}
<h1>Diff: {{o.handler_class_name}}::{{o.repo_path}}</h1>
<DIV CLASS="module">
<TABLE SUMMARY="Object info" WIDTH="100%">
<CAPTION>Object info</CAPTION>
<TR><TD WIDTH="100px"><b>ID</b></TD><TD>{{o.id}}</TD></TR>
<TR><TD><b>Repo Path</b></TD><TD>{{o.repo_path}}</TD></TR>
<TR><TD><b>From Revision</b></TD><TD>{{r1}}</TD></TR>
<TR><TD><b>To Revision</b></TD><TD>{{r2}}</TD></TR>
<TR><TD><b>Categories</b></TD><TD>{% for c in o.categories.all %} {{c}} {% endfor %}</TD></TR>
<TR><TD><b>Preview Mode</b></TD><TD>
<form>
<select onchange="location = this.options[this.selectedIndex].value;">
<option value="{% cm_url diff_rev o.id 'u' r1 r2 %}" {%ifequal mode "u"%} selected{% endifequal %}>Unified Diff</option>
<option value="{% cm_url diff_rev o.id '2' r1 r2 %}" {%ifequal mode "2"%} selected{% endifequal %}>Side-by-side</option>
</select>
</form>
</TD></TR>
</TABLE>
</DIV>
<DIV CLASS="module">
<TABLE SUMMARY="Diff" WIDTH="100%">
<CAPTION>Diff</CAPTION>
<TR><TD>{{diff|safe}}</TD></TR>
</TABLE>
{%ifequal mode "2"%}
<B>Legend:</B><BR/>
<SPAN class="diff_add">&nbsp;&nbsp;</SPAN> Added<br/>
<SPAN class="diff_sub">&nbsp;&nbsp;</SPAN> Removed<br/>
<SPAN class="diff_chg">&nbsp;&nbsp;</SPAN> Modified
{% endifequal %}
</DIV>
<DIV CLASS="module">
<FORM METHOD="POST" ACTION="{% cm_url diff o.id %}">{% csrf_token %}
<TABLE SUMMARY="Revisions" WIDTH="100%">
<CAPTION>Revisions</CAPTION>
<INPUT TYPE="submit" VALUE="Diff">
{% for rev in o.revisions %}
<TR>
<TD><A HREF="{% cm_url view_revision o.id rev.revision %}">{{rev.revision}}</A></TD>
<TD><A HREF="{% cm_url view_revision o.id rev.revision %}">{{rev.date}}</a></TD>
<TD><INPUT TYPE="radio" NAME="r1" ID="r1{{rev.revision}}" VALUE="{{rev.revision}}" {% ifequal rev.revision r1 %}checked{%endifequal%}/></TD>
<TD><INPUT TYPE="radio" NAME="r2" ID="r2{{rev.revision}}" VALUE="{{rev.revision}}" {% ifequal rev.revision r2 %}checked{%endifequal%}/></TD>
</TR>
{% endfor %}
</TABLE>
<INPUT TYPE="submit" VALUE="Diff">
</FORM>
</DIV>
{%endblock%}
{% extends "template.html" %}
{% load cmtags %}
{% block extrahead %}
<link rel="stylesheet" type="text/css" href="/media/admin/css/changelists.css" />
<link rel="stylesheet" type="text/css" href="/ui/web/css/pygments.css" />
<link rel="stylesheet" type="text/css" href="/ui/web/css/highlight.css" />
{% endblock %}
{% block breadcrumbs %}{{block.super}}<li>{{o.repo_path}}</li>{% endblock %}
<{% block content %}
<ul class="object-tools">
{% if annotate %}
<li><a href="{% cm_url view o.id %}">View</a></li>
{% endif %}
{% if content %}
<li><a href="{% cm_url annotate o.id %}">Annotate</a></li>
{% endif %}
<li><a href="{% if r.revision %}{% cm_url view_text_revision o.id r.revision %}{% else %}{% cm_url view_text o.id %}{% endif %}">Plain Text</a></li>
</ul>
<h1>{{o.verbose_name}}: {{o.repo_path}}</h1>
<DIV CLASS="module">
<TABLE SUMMARY="Object info" WIDTH="100%">
<CAPTION>Object info</CAPTION>
<TR><TD WIDTH="100px"><b>ID</b></TD><TD>{{o.id}}</TD></TR>
<TR><TD><b>Repo Path</b></TD><TD>{{o.repo_path}}</TD></TR>
<TR><TD><b>Revision</b></TD><TD>{{r.revision}}</TD></TR>
<TR><TD><b>Modified</b></TD><TD>{{r.date}}</TD></TR>
<TR><TD><b>Categories</b></TD><TD>{% for c in o.categories.all %} {{c}} {% endfor %}</TD></TR>
</TABLE>
</DIV>
{% if annotate %}
<style>
.a-rev {
width: 30px;
margin: 0 4pt 0 0;
padding: 1px 3px 1px 0;
text-align: right;
-webkit-user-select: none;
border-right: 1px solid #d7d7d7;
border-bottom: 1px solid #998;
background: #eee;
color: #999;
}
.a-date {
width: 135px;
margin: 0 4pt 0 0;
padding: 1px 3px 1px 0;
text-align: right;
-webkit-user-select: none;
border-right: 1px solid #d7d7d7;
border-bottom: 1px solid #998;
background: #eee;
color: #999;
}
.a-text {
white-space: pre;
font-family: monospace;
}
</style>
{% endif %}
<DIV CLASS="module">
<TABLE SUMMARY="Content" WIDTH="100%">
<CAPTION>Content</CAPTION>
<TR><TD>{% if content %}{{content|safe}}{% endif %}{% if annotate %}
<table border="1" width="100%">
<thead>
<tr><th>Rev</th><th>Date</th><th>Text</th></tr>
</thead>
{% for s,ar,l in annotate %}
<tr class="{{s}}">
<td class="a-rev"><a href="{% cm_url view_revision o.id ar.revision %}">{{ar.revision}}</a></td>
<td class="a-date"><a href="{% cm_url view_revision o.id ar.revision %}">{{ar.date}}</a></td>
<td class="a-text">{{l}}</td>
</tr>
{% endfor %}
</table>
{%endif%}</TD></TR>
</TABLE>
</DIV>
<DIV CLASS="module">
<FORM METHOD="POST" ACTION="{% cm_url diff o.id %}">{% csrf_token %}
<TABLE SUMMARY="Revisions" WIDTH="100%">
<CAPTION>Revisions</CAPTION>
<INPUT TYPE="submit" VALUE="Diff">
<TBODY>
{% for rev in o.revisions %}
<TR class={% ifequal r.revision rev.revision %}"selected"{% else %}"{%cycle 'row1' 'row2'%}"{% endifequal %}>
<TD WIDTH="50px"><A HREF="{% cm_url view_revision o.id rev.revision %}">{{rev.revision}}</A></TD>
<TD WIDTH="200px"><A HREF="{% cm_url view_revision o.id rev.revision %}">{{rev.date}}</A></TD>
<TD WIDTH="50px"><INPUT TYPE="radio" NAME="r1" ID="r1{{rev.revision}}" VALUE="{{rev.revision}}"/></TD>
<TD><INPUT TYPE="radio" NAME="r2" ID="r2{{rev.revision}}" VALUE="{{rev.revision}}"/></TD>
</TR>
{% endfor %}
</TBODY>
</TABLE>
<INPUT TYPE="submit" VALUE="Diff">
</FORM>
</DIV>
{%endblock%}
# -*- coding: utf-8 -*-
# ---------------------------------------------------------------------
# Template tags
# ---------------------------------------------------------------------
# Copyright (C) 2007-2009 The NOC Project
# See LICENSE for details
# ---------------------------------------------------------------------
# Third-party modules
from django.template.defaulttags import URLNode, register
from django.template import TemplateSyntaxError
class CMURLNode(URLNode):
"""
Act as django's standard {%url%} node.
Add <module>:<app>: to view name when missed
"""
def render(self, context):
if ":" not in self.view_name:
# Append <module>:<app>: to view name when missed
app = context["app"]
self.view_name = "%s:%s:%s" % (app.module, app.app, self.view_name)
return super(CMURLNode, self).render(context)
def cm_url(parser, token):
bits = token.split_contents()
if len(bits) < 2:
raise TemplateSyntaxError("'%s' takes at least one argument"
" (path to a view)" % bits[0])
viewname = bits[1]
args = []
kwargs = {}
asvar = None
if len(bits) > 2:
bits = iter(bits[2:])
for bit in bits:
if bit == 'as':
asvar = next(bits.next)
break
else:
for arg in bit.split(","):
if '=' in arg:
k, v = arg.split('=', 1)
k = k.strip()
kwargs[k] = parser.compile_filter(v)
elif arg:
args.append(parser.compile_filter(arg))
return CMURLNode(viewname, args, kwargs, asvar)
cm_url = register.tag(cm_url)
......@@ -8,16 +8,14 @@
# Python modules
from __future__ import absolute_import
import logging
# Third-party modules
from six.moves import zip
import pylibmc
import pylibmc.pools
# NOC modules
from noc.config import config
from noc.core.perf import metrics
# NOC modules
from .base import BaseCache
logger = logging.getLogger(__name__)
......
......@@ -13,6 +13,8 @@ import itertools
from functools import partial
import socket
import struct
# Third-party modules
from six.moves import zip
# NOC modules
from noc.config import config
......
......@@ -13,6 +13,8 @@ import itertools
import types
import re
from collections import defaultdict
# Third-party modules
from six.moves import zip
# NOC modules
from noc.core.vlan import has_vlan, optimize_filter
from .transformer import PredicateTransformer
......
......@@ -10,6 +10,7 @@
import csv
# Third-party modules
import six
from six.moves import zip
# Django modules
from django.db import models
......
# -*- coding: utf-8 -*-
# ----------------------------------------------------------------------
# Django template context
# Deprecation Warning classes
# ----------------------------------------------------------------------
# Copyright (C) 2007-2018 The NOC Project
# Copyright (C) 2007-2019 The NOC Project
# See LICENSE for details
# ----------------------------------------------------------------------
"""
On every new NOC release ensure:
# Third-party modules
import ujson
# NOC modules
from noc.core.cache.base import cache
* All features with RemovedInNOC<current>Warning is really removed from code
* Remove unnecessary deprecation warning
* Mark Next Release's warnings as DeprecationWarning
* Add Next-after-next Release's warnings as PendingDeperecationWarning
"""
CACHE_VERSION = 1
class RemovedInNOC1904Warning(DeprecationWarning):
"""
Features to be removed in NOC 19.4
"""
def messages(request):
if "noc_user" in request.COOKIES:
session_id = request.COOKIES["noc_user"].rsplit("|", 1)[-1]
key = "msg-%s" % session_id
msg = cache.get(key, version=CACHE_VERSION)
if msg:
messages = ujson.loads(msg)
cache.delete(key, version=CACHE_VERSION)
return {
"messages": messages
}
return {}
class RemovedInNOC1905Warning(PendingDeprecationWarning):
"""
Features to be removed in NOC 19.5
"""
......@@ -47,8 +47,7 @@ class Stream(object):
if not self.out:
self.out_path = os.path.join(
self.prefix,
"%s-%06d.tsv.gz.tmp" % (self.fs,
self.chunk_count.next())
"%s-%06d.tsv.gz.tmp" % (self.fs, next(self.chunk_count))
)
meta_path = self.out_path[:-11] + ".meta"
with open(meta_path, "w") as f:
......
......@@ -2,7 +2,7 @@
# ----------------------------------------------------------------------
# GridVCS
# ----------------------------------------------------------------------
# Copyright (C) 2007-2018 The NOC Project
# Copyright (C) 2007-2019 The NOC Project
# See LICENSE for details
# ----------------------------------------------------------------------
......@@ -17,8 +17,9 @@ import pymongo
import gridfs
import gridfs.errors
import bsdiff4
from bson import ObjectId
# NOC modules
from noc.lib.nosql import get_db, ObjectId
from noc.lib.nosql import get_db
from .revision import Revision
......
......@@ -2,19 +2,26 @@
# ----------------------------------------------------------------------
# ExtFormat middleware
# ----------------------------------------------------------------------
# Copyright (C) 2007-2018 The NOC Project
# Copyright (C) 2007-2019 The NOC Project
# See LICENSE for details
# ----------------------------------------------------------------------
class ExtFormatMiddleware(object):