Merge branch 'pu3-use-internal-pip' into 'master'

Use internal pip for python3.

See merge request !63

.gitlab-ci.yml

@@ -48,7 +48,6 @@ upload:
   tags:
     - docker
   only:
-    - master@noc/tower
     - tags@noc/tower
 
 check_docker_buildable_debian:
@@ -81,7 +80,6 @@ build_image_debian:
   tags:
     - shell
   only:
-    - master@noc/tower
     - tags@noc/tower
 
 build_image_alpine:
@@ -98,5 +96,4 @@ build_image_alpine:
   tags:
     - shell
   only:
-    - master@noc/tower
     - tags@noc/tower

requirements.txt

-pip==9.0.1
 pyparsing==2.2.0
 bcrypt==3.1.4
 certifi==2016.9.26
@@ -9,7 +8,7 @@ peewee==2.10.2
 pycparser>=2.10
 singledispatch>=3.4.0.0
 six>=1.8.0
-tornado==4.4.2
+tornado==5.1.1
 ansible==2.9.6
 setuptools>=11.3
 packaging

tower/api/base.py

@@ -7,8 +7,7 @@
 # ----------------------------------------------------------------------
 
 # Python modules
-from builtins import object
-import base64
+import codecs
 
 # Third-party modules
 import tornado.web
@@ -30,7 +29,7 @@ class BaseHandler(tornado.web.RequestHandler):
             # Fallback to basic
             auth_header = self.request.headers.get("Authorization")
             if auth_header and auth_header.startswith("Basic "):
-                auth = base64.decodestring(auth_header[6:])
+                auth = codecs.decode(auth_header[6:].encode("utf-8"), "base64").decode("utf-8")
                 u, p = auth.split(":", 2)
                 au = User.authenticate(u, p)
                 if au.is_active:

tower/api/datacenter.py

@@ -7,7 +7,6 @@
 # ----------------------------------------------------------------------
 
 # Tower modules
-from __future__ import absolute_import
 from .model import ModelAPI
 from tower.models.datacenter import Datacenter
 

tower/api/deploy.py

@@ -6,21 +6,17 @@
 # See LICENSE for details
 # ----------------------------------------------------------------------
 
-from __future__ import absolute_import
-from builtins import str
-from builtins import zip
-from builtins import range
-import datetime
 # Python modules
+import datetime
 import logging
 import os
 import re
 import subprocess
 
+# Third-party modules
 import tornado.ioloop
 import tornado.iostream
 import tornado.process
-# Third-party modules
 import tornado.web
 
 # Tower modules

tower/api/direct.py

@@ -7,7 +7,6 @@
 # ----------------------------------------------------------------------
 
 # Python modules
-from __future__ import absolute_import
 import json
 
 # Third-party modules
@@ -78,4 +77,4 @@ class DirectRequestHandler(BaseHandler):
                 response += [{}]
         if is_scalar:
             response = response[0]
-        self.write(json.dumps(response.decode("utf-8")))
+        self.write(json.dumps(response))

tower/api/environment.py

@@ -7,7 +7,6 @@
 # ----------------------------------------------------------------------
 
 # Third-party models
-from __future__ import absolute_import
 import yaml
 
 # Tower models
@@ -22,5 +21,4 @@ class EnvironmentAPI(ModelAPI):
     @api
     def ansible_inventory(self, env_id):
         e = Environment.get(Environment.id == int(env_id))
-        return yaml.safe_dump(e.ansible_inventory.decode("utf-8"),
-                              default_flow_style=False)
+        return yaml.safe_dump(e.ansible_inventory, default_flow_style=False)

tower/api/jsonrpc.py

@@ -7,11 +7,10 @@
 # ----------------------------------------------------------------------
 
 # Python modules
-from __future__ import absolute_import
-from builtins import str
 import json
 import logging
 import peewee
+
 # Third-party modules
 import tornado.gen
 from tornado.web import HTTPError
@@ -46,8 +45,8 @@ class JSONRPCHandler(BaseHandler):
         # Parse request
         try:
             req = json.loads(self.request.body)
-        except ValueError as why:
-            raise HTTPError(400, "Bad request: %s" % why)
+        except ValueError as e:
+            raise HTTPError(400, "Bad request: %s" % e)
         # Parse request
         id = req.get("id", None)
         params = req.get("params", [])
@@ -74,10 +73,10 @@ class JSONRPCHandler(BaseHandler):
             if (tornado.gen.is_future(result)):
                 result = yield result
             response["result"] = result
-        except APIError as why:
-            response["error"] = str(why)
-        except peewee.IntegrityError as why:
-            response["error"] = str(why)
+        except APIError as e:
+            response["error"] = str(e)
+        except peewee.IntegrityError as e:
+            response["error"] = str(e)
         # Return response
         self.set_header("Content-Type", self.MIME_TYPE)
         self.write(json.dumps(response))

tower/api/model.py

@@ -7,8 +7,6 @@
 # ----------------------------------------------------------------------
 
 # Third-party modules
-from __future__ import absolute_import
-from builtins import str
 import peewee
 
 from .base import API, api, APIError

tower/api/node.py

@@ -7,7 +7,6 @@
 # ----------------------------------------------------------------------
 
 # Tower modules
-from __future__ import absolute_import
 from .model import ModelAPI, api, APIError
 from tower.models.node import Node
 

tower/api/nodetype.py

@@ -7,7 +7,6 @@
 # ----------------------------------------------------------------------
 
 # Tower modules
-from __future__ import absolute_import
 from .model import ModelAPI
 from tower.models.nodetype import NodeType
 

tower/api/pool.py

@@ -7,7 +7,6 @@
 # ----------------------------------------------------------------------
 
 # Tower modules
-from __future__ import absolute_import
 from .model import ModelAPI
 from tower.models.pool import Pool
 

tower/api/pull.py

@@ -6,26 +6,27 @@
 # See LICENSE for details
 # ----------------------------------------------------------------------
 
-from __future__ import absolute_import
-
-import datetime
 # Python modules
-import logging
 import os
+import logging
+import datetime
+import shutil
 
 # Third-party modules
 from concurrent.futures import ThreadPoolExecutor
-from pip.download import unpack_url
-from pip.index import Link
-from pip.vcs import VersionControl
-from tower.contrib.utils import unpack, check_destination
+from pip._internal.network.session import PipSession
+from pip._internal.network.download import Downloader
+from pip._internal.operations.prepare import unpack_url
+from pip._internal.index.collector import Link
+from pip._internal.vcs.versioncontrol import VersionControl
 
-from tower.models.db import db
-from tower.models.environment import Environment
-from tower.models.pulllog import PullLog
 # Tower modules
 from .base import API, api
-from tower.models.role import Role
+from ..contrib.utils import unpack, check_destination
+from ..models.db import db
+from ..models.environment import Environment
+from ..models.pulllog import PullLog
+from ..models.role import Role
 
 logger = logging.getLogger(__name__)
 logger.setLevel("DEBUG")
@@ -102,7 +103,13 @@ class PullAPI(API):
         env = job.environment
         status = True
         log = []
-        self.pull(env.playbook_link, env.playbook_path)
+        self.pull(env.playbook_link, env.repo_path)
+        repo_playbooks_path = os.path.join(env.repo_path, "ansible_deploy")
+        if not os.path.isdir(repo_playbooks_path):
+            # Playbooks on repo root
+            repo_playbooks_path = env.repo_path
+        shutil.rmtree(env.playbook_path, ignore_errors=True)
+        shutil.move(repo_playbooks_path, env.playbook_path)
         for role in Role.select().where(Role.environment == env, Role.is_enabled == True):  # noqa
             self.pull(role.link, role.role_path)
 
@@ -114,8 +121,9 @@ class PullAPI(API):
 
     @staticmethod
     def pull(link, path):
+        logger.debug("Pull link: %s, path: %s", link, path)
         try:
-            unpack_url(Link(link), path)
+            unpack_url(Link(link), path, Downloader(PipSession(), ""))
         except KeyboardInterrupt:
             raise
         except Exception as e:

tower/api/role.py

@@ -7,7 +7,6 @@
 # ----------------------------------------------------------------------
 
 # Tower modules
-from __future__ import absolute_import
 from .model import ModelAPI
 from tower.models.role import Role
 

tower/api/service.py

@@ -7,13 +7,9 @@
 # ----------------------------------------------------------------------
 
 # Python modules
-from __future__ import absolute_import
-from builtins import str
-from builtins import range
 import os
 
 # Third-party modules
-import yaml
 import json
 
 # Tower modules
@@ -24,7 +20,7 @@ from tower.models.pool import Pool
 from tower.models.service import Service
 from tower.models.db import db
 from itertools import product
-from tower.contrib.yaml_ordered_dict import OrderedDictYAMLLoader
+from tower.contrib.yaml_ordered_dict import ordered_load
 
 
 class ServiceAPI(API):
@@ -36,7 +32,7 @@ class ServiceAPI(API):
             if not os.path.exists(path):
                 continue
             with open(path) as f:
-                descr = yaml.load(f, OrderedDictYAMLLoader)
+                descr = ordered_load(f)
                 if not descr:
                     continue
                 if "services" not in descr or not descr["services"]:
@@ -158,7 +154,7 @@ class ServiceAPI(API):
                 if ck - (ck - nk) < nk:
                     updated_config = dict(service_config)
                     updated_config.update(current_config)
-                    Service.update(config=json.dumps(updated_config.decode("utf-8"), sort_keys=True)).where(
+                    Service.update(config=json.dumps(updated_config, sort_keys=True)).where(
                         Service.id == srv[0]).execute()
 
     def init_services(self, env):

tower/api/settings.py

@@ -7,7 +7,6 @@
 # ----------------------------------------------------------------------
 
 # Tower modules
-from __future__ import absolute_import
 from .base import API, api
 from tower.models.settings import Settings
 

tower/api/ui.py

@@ -6,9 +6,9 @@
 # See LICENSE for details
 # ----------------------------------------------------------------------
 
+# Python modules
 import hashlib
 import logging
-# Python modules
 import os
 
 # Third-party modules

tower/cli/backup.py

@@ -6,11 +6,11 @@
 # See LICENSE for details
 # -----------------------------------------------------------------------
 
+# Python modules
+import subprocess
 import argparse
 import os
 import shutil
-# Python modules
-import subprocess
 
 
 def sqlite_path():

tower/cli/inv.py

@@ -6,11 +6,10 @@
 # See LICENSE for details
 # -----------------------------------------------------------------------
 
-from __future__ import print_function
-import json
 # Python modules
 import os
 import sys
+import json
 from optparse import OptionParser
 
 # Tower modules

tower/cli/pull.py

@@ -6,9 +6,8 @@
 # See LICENSE for details
 # -----------------------------------------------------------------------
 
-from __future__ import print_function
-import datetime
 # Python modules
+import datetime
 import logging
 import os
 import sys