Verified Commit ebbf12f2 authored by Aleksey Shirokih's avatar Aleksey Shirokih
Browse files

Merge branch 'master' of code.getnoc.com:noc/noc

parents 36a3ab15 b678b235
[flake8]
exclude=.git,var,share
# E203 whitespace before ':'
# Conflicts with black formatting
# E402 module level import not at top of file
# Imports may be at the end of module to resolve circular references
# E501 line too long
......@@ -8,9 +10,8 @@ exclude=.git,var,share
# Needs global refactoring, postponed for a while
# F405 'xxx' may be undefined, or defined from star imports: xxx
# Needs global refactoring, postponed for a while
# W504 Line break occurred after a binary operator
# Changed PEP8 recommendation. Breaks already passed tests.
# Postponed for a while
# W503 line break before binary operator
# Not PEP8 compliant
# W605 invalid escape sequence 'x'
# As of Python 3.6, a backslash-character pair that is
# not a valid escape sequence now generates a DeprecationWarning.
......@@ -18,4 +19,4 @@ exclude=.git,var,share
# that will not be for several Python releases.
# Not an actual error for Python2 and breaks already passed tests.
# Postponed for a while
ignore=E402,E501,F403,F405,W504,W605
ignore=E203,E402,E501,F403,F405,W503,W605
......@@ -11,7 +11,6 @@ check labels:
stage: lint
image: registry.getnoc.com/infrastructure/noc-lint:master
script:
- set -x
- FLIST=$(git --no-pager diff --name-only $CI_COMMIT_SHA $(git merge-base origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME $CI_COMMIT_SHA))
- python ./scripts/check-labels.py $FLIST
only:
......@@ -100,6 +99,26 @@ pylint:
tags:
- docker
bandit:
stage: lint
image: registry.getnoc.com/infrastructure/noc-lint:master
script:
- set -x
- FLIST=$(git --no-pager diff --name-only $CI_COMMIT_SHA $(git merge-base origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME $CI_COMMIT_SHA) | egrep ".py$" | grep -v ".docker/"|| true)
- >
if [ ! -z "$FLIST" ];
then
bandit "$FLIST";
else
echo "No files to lint"
true;
fi
only:
- merge_requests
tags:
- docker
allow_failure: true
futurize:
stage: lint
image: registry.getnoc.com/infrastructure/noc-lint:master
......@@ -233,7 +252,7 @@ Build docs:
artifacts:
paths:
- build/docs/en
expire_in: 1 day
expire_in: 2 hours
only:
changes:
- docs/**
......@@ -251,7 +270,7 @@ Build Master Docs:
artifacts:
paths:
- build/docs/en
expire_in: 1 day
expire_in: 2 hours
only:
refs:
- master
......
......@@ -8,8 +8,10 @@
# Python import datetime
import datetime
# Third-party modules
from django.db import models
# NOC modules
from noc.core.migration.base import BaseMigration
......@@ -25,38 +27,36 @@ class Migration(BaseMigration):
return
# User model
self.db.create_table(
"auth_user", (
"auth_user",
(
("id", models.AutoField(verbose_name="ID", primary_key=True, auto_created=True)),
("username", models.CharField(max_length=75, unique=True)),
("username", models.CharField(max_length=75, unique=True)),
("first_name", models.CharField(max_length=75, blank=True)),
("last_name", models.CharField( max_length=75, blank=True)),
("last_name", models.CharField(max_length=75, blank=True)),
("email", models.EmailField(blank=True)),
("password", models.CharField(max_length=128)),
("is_active", models.BooleanField(default=True)),
("is_superuser", models.BooleanField(default=False)),
("date_joined", models.DateTimeField(default=datetime.datetime.now))
))
("date_joined", models.DateTimeField(default=datetime.datetime.now)),
),
)
# Group model
self.db.create_table(
"auth_group", (
"auth_group",
(
("id", models.AutoField(verbose_name="ID", primary_key=True, auto_created=True)),
("name", models.CharField(max_length=80, unique=True))
)
("name", models.CharField(max_length=80, unique=True)),
),
)
#
# Adding ManyToManyField "User.groups"
User = self.db.mock_model(
model_name="User",
db_table="auth_user"
)
Group = self.db.mock_model(
model_name="Group",
db_table="auth_group"
)
User = self.db.mock_model(model_name="User", db_table="auth_user")
Group = self.db.mock_model(model_name="Group", db_table="auth_group")
self.db.create_table(
"auth_user_groups", (
"auth_user_groups",
(
("id", models.AutoField(verbose_name="ID", primary_key=True, auto_created=True)),
("user", models.ForeignKey(User, null=False, on_delete=models.CASCADE)),
("group", models.ForeignKey(Group, null=False, on_delete=models.CASCADE))
)
("group", models.ForeignKey(Group, null=False, on_delete=models.CASCADE)),
),
)
......@@ -18,9 +18,14 @@ class Migration(BaseMigration):
self.db.execute(
"INSERT INTO auth_user"
"(username, first_name, last_name, email, password, is_active, is_superuser, date_joined) "
"VALUES(%s, %s, %s, %s, %s, %s, %s, 'now')", [
"admin", "NOC", "Admin", "test@example.com",
"VALUES(%s, %s, %s, %s, %s, %s, %s, 'now')",
[
"admin",
"NOC",
"Admin",
"test@example.com",
"sha1$235c1$e8e4d9aaa945e1fae62a965ee87fbf7b4a185e3f",
True, True
]
True,
True,
],
)
......@@ -8,6 +8,7 @@
# Third-party modules
from django.db import models
# NOC modules
from noc.core.migration.base import BaseMigration
......@@ -19,38 +20,32 @@ class Migration(BaseMigration):
return
# Adding model "Permission"
self.db.create_table(
"main_permission", (
"main_permission",
(
("id", models.AutoField(verbose_name="ID", primary_key=True, auto_created=True)),
("name", models.CharField("Name", max_length=128, unique=True)),
)
)
Permission = self.db.mock_model(
model_name="Permission",
db_table="main_permission"
),
)
Permission = self.db.mock_model(model_name="Permission", db_table="main_permission")
# Adding ManyToManyField "Permission.groups"
Group = self.db.mock_model(
model_name="Group",
db_table="auth_group"
)
Group = self.db.mock_model(model_name="Group", db_table="auth_group")
self.db.create_table(
"main_permission_groups", (
"main_permission_groups",
(
("id", models.AutoField(verbose_name="ID", primary_key=True, auto_created=True)),
("permission", models.ForeignKey(Permission, null=False, on_delete=models.CASCADE)),
("group", models.ForeignKey(Group, null=False, on_delete=models.CASCADE))
)
("group", models.ForeignKey(Group, null=False, on_delete=models.CASCADE)),
),
)
# Adding ManyToManyField "Permission.users"
User = self.db.mock_model(
model_name="User",
db_table="auth_user"
)
User = self.db.mock_model(model_name="User", db_table="auth_user")
self.db.create_table(
"main_permission_users", (
"main_permission_users",
(
("id", models.AutoField(verbose_name="ID", primary_key=True, auto_created=True)),
("permission", models.ForeignKey(Permission, null=False, on_delete=models.CASCADE)),
("user", models.ForeignKey(User, null=False, on_delete=models.CASCADE))
)
("user", models.ForeignKey(User, null=False, on_delete=models.CASCADE)),
),
)
......@@ -8,48 +8,56 @@
# Third-party modules
from django.db import models
# NOC modules
from noc.core.migration.base import BaseMigration
from noc.settings import LANGUAGE_CODE
class Migration(BaseMigration):
depends_on = [
("main", "0056_userprofile_heatmap")
]
depends_on = [("main", "0056_userprofile_heatmap")]
def migrate(self):
# Add fields to user
self.db.add_column(
"auth_user",
"preferred_language",
models.CharField("Preferred Language", max_length=16, null=True, blank=True, default=LANGUAGE_CODE)
models.CharField(
"Preferred Language", max_length=16, null=True, blank=True, default=LANGUAGE_CODE
),
)
self.db.add_column("auth_user", "heatmap_lon", models.FloatField("Longitude", blank=True, null=True))
self.db.add_column("auth_user", "heatmap_lat", models.FloatField("Latitude", blank=True, null=True))
self.db.add_column("auth_user", "heatmap_zoom", models.IntegerField("Zoom", blank=True, null=True))
# UserContacts
User = self.db.mock_model(
model_name="User",
db_table="auth_user"
self.db.add_column(
"auth_user", "heatmap_lon", models.FloatField("Longitude", blank=True, null=True)
)
self.db.add_column(
"auth_user", "heatmap_lat", models.FloatField("Latitude", blank=True, null=True)
)
TimePattern = self.db.mock_model(
model_name="TimePattern",
db_table="main_timepattern"
self.db.add_column(
"auth_user", "heatmap_zoom", models.IntegerField("Zoom", blank=True, null=True)
)
# UserContacts
User = self.db.mock_model(model_name="User", db_table="auth_user")
TimePattern = self.db.mock_model(model_name="TimePattern", db_table="main_timepattern")
self.db.create_table(
"aaa_usercontact", (
"aaa_usercontact",
(
("id", models.AutoField(primary_key=True)),
("user", models.ForeignKey(User, verbose_name="User", on_delete=models.CASCADE)),
("notification_method", models.CharField("Method", max_length=16)),
("params", models.CharField("Params", max_length=256)),
("time_pattern", models.ForeignKey(TimePattern, verbose_name="Time Pattern", on_delete=models.CASCADE))
)
(
"time_pattern",
models.ForeignKey(
TimePattern, verbose_name="Time Pattern", on_delete=models.CASCADE
),
),
),
)
# Creating unique_together for [user_profile, time_pattern, notification_method, params] on UserProfileContact.
self.db.create_index(
"aaa_usercontact", ["user_id", "time_pattern_id", "notification_method", "params"],
unique=True
"aaa_usercontact",
["user_id", "time_pattern_id", "notification_method", "params"],
unique=True,
)
# Update users
p_map = {} # profile_id -> user_id
......@@ -62,7 +70,7 @@ class Migration(BaseMigration):
"UPDATE auth_user "
"SET preferred_language=%s, heatmap_lon=%s, heatmap_lat=%s, heatmap_zoom=%s "
"WHERE id = %s",
[preferred_language, heatmap_lon, heatmap_lat, heatmap_zoom, user_id]
[preferred_language, heatmap_lon, heatmap_lat, heatmap_zoom, user_id],
)
p_map[p_id] = user_id
# Move contacts
......@@ -74,7 +82,7 @@ class Migration(BaseMigration):
self.db.execute(
"INSERT INTO aaa_usercontact(user_id, notification_method, params, time_pattern_id) "
"VALUES(%s, %s, %s, %s)",
[p_map[p_id], notification_method, params, time_pattern_id]
[p_map[p_id], notification_method, params, time_pattern_id],
)
# Delete profiles and contacts
self.db.delete_table("main_userprofilecontact")
......
......@@ -8,11 +8,18 @@
# Python modules
import datetime
# Third-party modules
import six
from mongoengine.document import Document, EmbeddedDocument
from mongoengine.fields import (StringField, BooleanField, DateTimeField, ListField,
EmbeddedDocumentField)
from mongoengine.fields import (
StringField,
BooleanField,
DateTimeField,
ListField,
EmbeddedDocumentField,
)
# NOC modules
from noc.core.acl import match
......@@ -42,11 +49,7 @@ class APIAccessACL(EmbeddedDocument):
@six.python_2_unicode_compatible
class APIKey(Document):
meta = {
"collection": "apikeys",
"strict": False,
"auto_create_index": False
}
meta = {"collection": "apikeys", "strict": False, "auto_create_index": False}
name = StringField(unique=True)
is_active = BooleanField()
......
......@@ -9,10 +9,12 @@
# Python modules
from threading import Lock
import operator
# Third-party modules
import cachetools
import six
from django.db import models
# NOC modules
from noc.core.model.base import NOCModel
from noc.core.model.decorator import on_delete_check
......@@ -20,9 +22,7 @@ from noc.core.model.decorator import on_delete_check
id_lock = Lock()
@on_delete_check(check=[
("sa.GroupAccess", "group")
])
@on_delete_check(check=[("sa.GroupAccess", "group")])
@six.python_2_unicode_compatible
class Group(NOCModel):
class Meta(object):
......
......@@ -10,10 +10,12 @@
from __future__ import print_function
from threading import Lock
import operator
# Third-party modules
import six
from django.db.models import Model, CharField, ManyToManyField
from django.db.models import CharField, ManyToManyField
import cachetools
# NOC modules
from noc.core.model.base import NOCModel
from noc.aaa.models.user import User
......@@ -31,6 +33,7 @@ class Permission(NOCModel):
Populated by manage.py sync-perm
@todo: Check name format
"""
class Meta(object):
verbose_name = "Permission"
verbose_name_plural = "Permissions"
......@@ -40,12 +43,9 @@ class Permission(NOCModel):
# module:app:permission
name = CharField("Name", max_length=128, unique=True)
# comma-separated list of permissions
implied = CharField(
"Implied", max_length=256, null=True, blank=True)
users = ManyToManyField(
User, related_name="permissions")
groups = ManyToManyField(
Group, related_name="permissions")
implied = CharField("Implied", max_length=256, null=True, blank=True)
users = ManyToManyField(User, related_name="permissions")
groups = ManyToManyField(Group, related_name="permissions")
_id_cache = cachetools.TTLCache(maxsize=100, ttl=60)
_name_cache = cachetools.TTLCache(maxsize=100, ttl=60)
......@@ -73,8 +73,7 @@ class Permission(NOCModel):
def get_implied_permissions(self):
if not self.implied:
return []
return [Permission.objects.get(name=p.strip())
for p in self.implied.split(",")]
return [Permission.objects.get(name=p.strip()) for p in self.implied.split(",")]
@classmethod
def has_perm(cls, user, perm):
......@@ -106,8 +105,7 @@ class Permission(NOCModel):
"""
# Add implied permissions
perms = set(perms) # Copy
for p in cls.objects.filter(
name__in=list(perms), implied__isnull=False):
for p in cls.objects.filter(name__in=list(perms), implied__isnull=False):
perms.update([x.strip() for x in p.implied.split(",")])
#
current = cls.get_user_permissions(user)
......@@ -140,8 +138,7 @@ class Permission(NOCModel):
"""
# Add implied permissions
perms = set(perms) # Copy
for p in cls.objects.filter(
name__in=list(perms), implied__isnull=False):
for p in cls.objects.filter(name__in=list(perms), implied__isnull=False):
perms.update([x.strip() for x in p.implied.split(",")])
#
current = cls.get_group_permissions(group)
......@@ -153,16 +150,14 @@ class Permission(NOCModel):
Permission.objects.get(name=p).groups.remove(group)
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_effective_perm_cache"),
lock=lambda _: perm_lock)
@cachetools.cachedmethod(operator.attrgetter("_effective_perm_cache"), lock=lambda _: perm_lock)
def get_effective_permissions(cls, user):
"""
Returns a set of effective user permissions,
counting group and implied ones
"""
if user.is_superuser:
return set(Permission.objects.values_list(
"name", flat=True))
return set(Permission.objects.values_list("name", flat=True))
perms = set()
# User permissions
for p in user.permissions.all():
......@@ -198,14 +193,15 @@ class Permission(NOCModel):
new_perms = set()
implied_permissions = {}
diverged_permissions = {} # new -> old
for app in site.apps.values():
for app in six.itervalues(site.apps):
new_perms = new_perms.union(app.get_permissions())
for p in app.implied_permissions:
ips = sorted([normalize(app, pp)
for pp in app.implied_permissions[p]])
ips = sorted([normalize(app, pp) for pp in app.implied_permissions[p]])
implied_permissions[normalize(app, p)] = ips
for p in app.diverged_permissions:
diverged_permissions[normalize(app, p)] = normalize(app, app.diverged_permissions[p])
diverged_permissions[normalize(app, p)] = normalize(
app, app.diverged_permissions[p]
)
# Check all implied permissions are present
for p in implied_permissions:
if p not in new_perms:
......@@ -221,19 +217,19 @@ class Permission(NOCModel):
# @todo: add implied permissions
p = Permission(name=name, implied=get_implied(name))
p.save()
print("+ %s" % name)
print ("+ %s" % name)
created_perms[name] = p
# Check implied permissions match
for name in old_perms.intersection(new_perms):
implied = get_implied(name)
p = Permission.objects.get(name=name)
if p.implied != implied:
print("~ %s" % name)
print ("~ %s" % name)
p.implied = implied
p.save()
# Deleted permissions
for name in old_perms - new_perms:
print("- %s" % name)
print ("- %s" % name)
Permission.objects.get(name=name).delete()
# Diverge created permissions
for name in created_perms:
......@@ -244,7 +240,7 @@ class Permission(NOCModel):
op = Permission.get_by_name(op_name)
if not op:
continue
print(": %s -> (%s, %s)" % (op_name, op_name, name))
print (": %s -> (%s, %s)" % (op_name, op_name, name))
# Migrate users
dp = created_perms[name]
for u in op.users.all():
......
......@@ -11,12 +11,14 @@ from __future__ import absolute_import
import datetime
from threading import Lock
import operator
# Third-party modules
import cachetools
import six
from django.db import models
from django.core import validators
from django.contrib.auth.hashers import check_password, make_password
# NOC modules
from noc.core.model.base import NOCModel
from noc.core.model.decorator import on_delete_check
......@@ -27,19 +29,21 @@ from .group import Group
id_lock = Lock()
@on_delete_check(check=[
("kb.KBEntryPreviewLog", "user"),
("aaa.UserContact", "user"),
("sa.UserAccess", "user"),
("kb.KBUserBookmark", "user"),
("main.Checkpoint", "user"),
("main.NotificationGroupUser", "user"),
("main.ReportSubscription", "run_as"),
("ip.PrefixBookmark", "user"),
("kb.KBEntryHistory", "user"),
("ip.PrefixAccess", "user"),
("main.Favorites", "user")
])
@on_delete_check(
check=[
("kb.KBEntryPreviewLog", "user"),
("aaa.UserContact", "user"),
("sa.UserAccess", "user"),
("kb.KBUserBookmark", "user"),
("main.Checkpoint", "user"),
("main.NotificationGroupUser", "user"),
("main.ReportSubscription", "run_as"),
("ip.PrefixBookmark", "user"),
("kb.KBEntryHistory", "user"),
("ip.PrefixAccess", "user"),
("main.Favorites", "user"),
]
)
@six.python_2_unicode_compatible
class User(NOCModel):
class Meta(object):
......@@ -53,46 +57,50 @@ class User(NOCModel):
username = models.CharField(
max_length=75,
unique=True,
help_text=_("Required. 30 characters or fewer. Letters, digits and "
"@/./+/-/_ only."),
help_text=_("Required. 30 characters or fewer. Letters, digits and " "@/./+/-/_ only."),
validators=[
validators.RegexValidator(r"^[\w.@+-]+$", _("Enter a valid username."), "invalid")
])
],
)
first_name = models.CharField(max_length=75, blank=True)
last_name = models.CharField(max_length=75, blank=True)
email = models.EmailField(blank=True)
password = models.CharField(max_length=128)
is_active = models.BooleanField(
default=True,
help_text=_("Designates whether this user should be treated as "
"active. Unselect this instead of deleting accounts."))
help_text=_(
"Designates whether this user should be treated as "
"active. Unselect this instead of deleting accounts."
),
)
is_superuser = models.BooleanField(
default=False,
help_text=_(
"Designates that this user has all permissions without "
"explicitly assigning them."))
"Designates that this user has all permissions without " "explicitly assigning them."
),
)
date_joined = models.DateTimeField(default=datetime.datetime.now)
groups = models.ManyToManyField(
Group,
blank=True,
help_text=_("The groups this user belongs to. A user will "
"get all permissions granted to each of "
"his/her group."),
related_name="user_set", related_query_name="user"
help_text=_(
"The groups this user belongs to. A user will "
"get all permissions granted to each of "
"his/her group."
),
related_name="user_set",
related_query_name="user",
)
# Custom profile