Commit 1f3113f2 authored by EKbfh's avatar EKbfh 🐼
Browse files

Merge branch 'py3' into 'master'

Python3

See merge request !48
parents c3d54953 e7bac1e0
Pipeline #23004 passed with stages
in 7 minutes and 38 seconds
# -*- coding: utf-8 -*-
# -----------------------------------------------------------------------
# Ansible dynamic inventory
# -----------------------------------------------------------------------
# Copyright (C) 2015 The NOC Project
# See LICENSE for details
# -----------------------------------------------------------------------
from __future__ import print_function
import Queue
import argparse
# Python modules
import os
import subprocess
import sys
import threading
# Tower modules
os.chdir(
os.path.join(os.path.dirname(sys.argv[0]), "..")
)
from tower.models.environment import Environment # noqa
from tower.models.node import Node # noqa
from tower.models.crashinfo import Crashinfo # noqa
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
"--env",
action="store", dest="env",
help="Use environment [%default]",
default=os.environ.get("NOC_ENV", "test")
)
parser.add_argument(
"--jobs",
action="store",
dest="jobs",
type=int,
help="Amount of nodes fetched concurrently",
default=5
)
subparsers = parser.add_subparsers(dest="cmd")
collect_parser = subparsers.add_parser("collect") # noqa
#
options = parser.parse_args(sys.argv[1:])
cmd_options = vars(options)
args = cmd_options.pop("args", ())
if options.cmd == "collect":
collect_crashinfo(options, args)
def die(msg):
print(msg + "\n")
sys.exit(1)
def collect_crashinfo(options, args):
try:
env = Environment.get(Environment.name == options.env)
except Environment.DoesNotExist:
die("Invalid environment: '%s'" % options.env)
q = Queue.Queue()
# Start workers
workers = []
for i in range(options.jobs):
w = threading.Thread(target=collect_worker, args=(q,))
workers += [w]
w.start()
# Spool jobs
for node in Node.select().where(Node.environment == env):
q.put(node)
# Wait for workers
for i in range(options.jobs):
q.put(None)
for w in workers:
w.join()
def collect_worker(q):
while True:
node = q.get(block=True)
if not node:
return
cwd = os.path.join(
Crashinfo.CRASHINFO_ROOT,
node.environment.name,
node.name
)
if not os.path.isdir(cwd):
try:
os.makedirs(cwd)
except OSError as e:
die("Cannot create directory %s" % cwd)
log_dir = os.path.join(
"var", "tower", "log",
"crashinfo",
"collect",
node.environment.name,
)
if not os.path.isdir(log_dir):
try:
os.makedirs(log_dir)
except OSError as e:
die("Cannot create directory %s %s" % (log_dir, e))
log_file = "%s/%s.log" % (log_dir, node.name)
lf = open(log_file, "w")
cmd = [
"rsync",
"-avz",
"-e ssh",
"--rsync-path=sudo rsync",
"%s@%s:%s/var/cp/crashinfo/new/*.json" % (
node.login_as, node.address,
node.environment.sys_prefix),
"."
]
subprocess.check_call(
cmd, cwd=cwd,
stdout=lf,
stderr=lf
)
lf.close()
......@@ -33,7 +33,7 @@ class OrderedDictYAMLLoader(yaml.Loader):
key = self.construct_object(key_node, deep=deep)
try:
hash(key)
except TypeError, exc:
except TypeError as exc:
raise yaml.constructor.ConstructorError('while constructing a mapping',
node.start_mark, 'found unacceptable key (%s)' % exc,
key_node.start_mark)
......
from builtins import object
from peewee import Model, CharField, TextField
class Settings(Model):
class Meta:
class Meta(object):
db_table = "settings"
key = CharField(primary_key=True)
......
from builtins import object
from peewee import Model, CharField, BooleanField
class User(Model):
class Meta:
class Meta(object):
db_table = "user"
name = CharField(unique=True)
......
from builtins import object
from peewee import Model, CharField, BooleanField
def migrate(migrator):
class User(Model):
class Meta:
class Meta(object):
database = migrator.db
db_table = "user"
......
from builtins import object
from peewee import Model, CharField, TextField
class Environment(Model):
class Meta:
class Meta(object):
db_table = "environment"
name = CharField(unique=True)
......
from builtins import object
from peewee import Model, CharField, TextField
class Datacenter(Model):
class Meta:
class Meta(object):
db_table = "datacenter"
name = CharField(unique=True)
......
from builtins import object
from peewee import Model, CharField, TextField, ForeignKeyField
class Environment(Model):
class Meta:
class Meta(object):
db_table = "environment"
class Pool(Model):
class Meta:
class Meta(object):
db_table = "pool"
indexes = (
(("environment_id", "name"), True),
......
from builtins import object
from peewee import Model, CharField, TextField, ForeignKeyField
class Environment(Model):
class Meta:
class Meta(object):
db_table = "environment"
class Datacenter(Model):
class Meta:
class Meta(object):
db_table = "datacenter"
class Node(Model):
class Meta:
class Meta(object):
db_table = "node"
indexes = (
(("environment", "name"), True),
......
from builtins import object
from peewee import (Model, CharField, TextField, ForeignKeyField,
DateTimeField, BooleanField, IntegerField)
class Environment(Model):
class Meta:
class Meta(object):
db_table = "environment"
class JobLog(Model):
class Meta:
class Meta(object):
db_table = "joblog"
start_ts = DateTimeField()
......
from builtins import object
from peewee import Model, CharField, IntegerField, ForeignKeyField
class Environment(Model):
class Meta:
class Meta(object):
db_table = "environment"
class Pool(Model):
class Meta:
class Meta(object):
db_table = "pool"
class Node(Model):
class Meta:
class Meta(object):
db_table = "node"
class Service(Model):
class Meta:
class Meta(object):
db_table = "service"
environment = ForeignKeyField(Environment, on_delete="RESTRICT")
......
from builtins import object
from peewee import (Model, CharField, TextField, ForeignKeyField,
DateTimeField, BooleanField)
class Environment(Model):
class Meta:
class Meta(object):
db_table = "environment"
class PullLog(Model):
class Meta:
class Meta(object):
db_table = "pulllog"
start_ts = DateTimeField()
......
from builtins import object
from peewee import (Model, CharField, ForeignKeyField,
BooleanField)
class NodeType(Model):
class Meta:
class Meta(object):
db_table = "node_type"
name = CharField(max_length=64, unique=True)
......
from builtins import object
from peewee import (Model, CharField, ForeignKeyField,
BooleanField)
def migrate(migrator):
class NodeType(Model):
class Meta:
class Meta(object):
database = migrator.db
db_table = "node_type"
......@@ -17,7 +18,7 @@ def migrate(migrator):
ansible_connection = CharField(max_length=255, default="smart")
class Node(Model):
class Meta:
class Meta(object):
database = migrator.db
db_table = "node"
......
from builtins import object
from peewee import (Model, CharField, BooleanField)
def migrate(migrator):
class NodeType(Model):
class Meta:
class Meta(object):
database = migrator.db
db_table = "node_type"
......
# Third-party modules
from builtins import str
from builtins import object
import yaml
from peewee import (Model, CharField, TextField, BooleanField)
......@@ -6,7 +8,7 @@ from peewee import (Model, CharField, TextField, BooleanField)
def migrate(migrator):
class Environment(Model):
class Meta:
class Meta(object):
database = migrator.db
db_table = "environment"
......@@ -75,5 +77,5 @@ def migrate(migrator):
if "session_ttl" in config[None]["login"]:
if "d" not in str(config[None]["login"]["session_ttl"]):
config[None]["login"]["session_ttl"] = str(config[None]["login"]["session_ttl"]) + "d"
env.service_config = yaml.dump(config)
env.service_config = yaml.dump(config.decode("utf-8"))
env.save()
from builtins import object
import os
import shutil
......@@ -6,7 +7,7 @@ from peewee import (Model, CharField, TextField, BooleanField)
def migrate(migrator):
class Environment(Model):
class Meta:
class Meta(object):
database = migrator.db
db_table = "environment"
......
from __future__ import print_function
from builtins import object
import yaml
from peewee import (Model, CharField, TextField, BooleanField)
......@@ -6,7 +7,7 @@ from peewee import (Model, CharField, TextField, BooleanField)
def migrate(migrator):
class Environment(Model):
class Meta:
class Meta(object):
database = migrator.db
db_table = "environment"
......@@ -96,7 +97,7 @@ def migrate(migrator):
"mongod_engine": env.mongo_engine or "wiredTiger",
"mongod_logging_destination": config[None]["mongod"].get("mongod_logging_destination", "file")
}
env.service_config = yaml.dump(config)
env.service_config = yaml.dump(config.decode("utf-8"))
env.save()
migrator.drop_column(
......
from builtins import object
from peewee import Model, CharField, TextField, ForeignKeyField, BooleanField
import os
def migrate(migrator):
class Environment(Model):
class Meta:
class Meta(object):
database = migrator.db
db_table = "environment"
......@@ -18,7 +19,7 @@ def migrate(migrator):
)
class Role(Model):
class Meta:
class Meta(object):
database = migrator.db
db_table = "role"
......
from builtins import object
from peewee import CharField, IntegerField, ForeignKeyField
from peewee import Model, TextField, BooleanField
import yaml
......@@ -57,7 +58,7 @@ DEFAULT_ROLES = [
def migrate(migrator):
class Environment(Model):
class Meta:
class Meta(object):
database = migrator.db
db_table = "environment"
......@@ -65,7 +66,7 @@ def migrate(migrator):
service_config = TextField(default="")
class Role(Model):
class Meta:
class Meta(object):
database = migrator.db
db_table = "role"
......@@ -77,7 +78,7 @@ def migrate(migrator):
role_name = CharField()
class Node(Model):
class Meta:
class Meta(object):
database = migrator.db
db_table = "node"
......@@ -85,7 +86,7 @@ def migrate(migrator):
environment = ForeignKeyField(Environment, on_delete="RESTRICT")
class Pool(Model):
class Meta:
class Meta(object):
database = migrator.db
db_table = "pool"
......@@ -93,7 +94,7 @@ def migrate(migrator):
name = CharField()
class Service(Model):
class Meta:
class Meta(object):
database = migrator.db
db_table = "service"
......@@ -156,5 +157,5 @@ def migrate(migrator):
config[None]["telegraf"] = {
"telegraf_output_plugin": "influx"
}
env.service_config = yaml.dump(config)
env.service_config = yaml.dump(config.decode("utf-8"))
env.save()
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment