Merge branch 'add-ansible-ci' into 'master'

noc/noc#1241 Add ansible-CI

See merge request noc/noc!4482

.gitlab-ci.yml

@@ -9,6 +9,7 @@ include:
   - project: 'noc/common-jobs'
     ref: stable
     file: '/templates/upload.yml'
+  - local: '/ansible/.ansible-ci.yml'
 
 check labels:
   stage: Lint

ansible/.ansible-ci.yml

+---
+
+variables:
+  ANSIBLE_FORCE_COLOR: 'true'
+
+lint:
+  stage: Lint
+  image: registry.getnoc.com/infrastructure/ansible_linter:master
+  script:
+    - export ANSIBLE_ROLES_PATH=./ansible/additional_roles:./ansible/system_roles:./ansible/noc_roles
+    - export ANSIBLE_LIBRARY=./ansible/library:./ansible/system_roles/mongod/library
+    - yamllint --version
+    - yamllint -s ansible/*
+    - ansible-playbook --version
+    - for inv in ansible/molecule/*/inv.yml; do ansible-playbook ansible/*/*/service.yml --syntax-check -i $inv; done
+    - ansible-lint --version
+    #- ansible-lint */*/service.yml -v
+    - ansible-review --version
+    - git ls-files ansible | grep -v molecule/ | xargs -P 32 -n 4 ansible-review -c .ansible-review -q
+  rules:
+    - changes:
+        - ansible/**/*
+        - ansible/*
+    - when: manual
+  tags:
+    - docker
+
+.base_builder:
+  stage: Build
+  image: registry.getnoc.com/infrastructure/molecule-docker:master
+  before_script:
+    - echo ${ssh_identity_text} | tr -d ' ' | base64 -d > /tmp/temporary_ssh_key
+    - chmod 0400 /tmp/temporary_ssh_key
+    - echo ${ssh_identity_text_pub} | tr -d ' ' | base64 -d > /tmp/temporary_ssh_key_pub
+    - chmod 0644 /tmp/temporary_ssh_key_pub
+  script:
+    - molecule --version
+    - ansible-playbook --version
+    - rm -rf noc
+    - cd ansible/ && molecule test -s $SCENARIO_NAME
+  cache:
+    key: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
+    paths:
+      - /tmp/dist
+  variables:
+    PIP_CACHE: /root/.cache/pip
+    PKG_CACHE: /var/cache/apt/archives
+  rules:
+    - changes:
+        - ansible/**/*
+        - ansible/*
+    - when: manual
+
+build_centos7:
+   extends: .base_builder
+   variables:
+     PKG_CACHE: /var/cache/yum
+     SCENARIO_NAME: default
+   tags:
+     - cloud
+     - docker
+
+# build_centos8:
+#   extends: .base_builder
+#   variables:
+#     PKG_CACHE: /var/cache/yum
+#     SCENARIO_NAME: centos8
+#   tags:
+#     - cloud
+#     - docker
+
+build_debian9:
+   extends: .base_builder
+   variables:
+     SCENARIO_NAME: debian9
+   tags:
+     - cloud
+     - docker
+
+build_debian10:
+  extends: .base_builder
+  variables:
+    SCENARIO_NAME: debian10
+  tags:
+    - cloud
+    - docker
+
+build_ubuntu16:
+   extends: .base_builder
+   variables:
+     SCENARIO_NAME: ubuntu16
+   tags:
+     - cloud
+     - docker
+
+build_ubuntu18:
+   extends: .base_builder
+   variables:
+     SCENARIO_NAME: ubuntu18
+   tags:
+     - cloud
+     - docker
+
+build_freebsd:
+   extends: .base_builder
+   variables:
+     PKG_CACHE: /var/cache/pkg
+     SCENARIO_NAME: freebsd
+     USE_CACHE: "False"  # freebsd minio-client "broken"
+   tags:
+     - satel
+     - docker
+   allow_failure: true  # too hard to maintain without maintainer. often incompatible changes
+
+build_oel7:
+   extends: .base_builder
+   variables:
+     PKG_CACHE: /var/cache/yum
+     SCENARIO_NAME: oel7
+   tags:
+     - satel
+     - docker
+   allow_failure: true  # no active users for platform

ansible/INSTALL

-See online documentation at http://kb.nocproject.org/display/DOC/Home
-for complete installation instructions.
\ No newline at end of file

ansible/README.md

-# Ansible playbooks for NOC Tower
-
-Primary goal of that playbook provide simple and very basic install of NOC.
-
-Several notes that it will not do:
-* remove external service from node (postgres, mongodb and so on) after disabling service in Tower
-* backup your data, please do it by yourself
-* move data from one host to another.
-
-Thar repo should not intended to be used by itself.
-Use https://code.getnoc.com/noc/tower/blob/master/Readme.md instead.
-
-# Additional roles
-
-Want to use noc tower tu rule them all? you can check this group: https://code.getnoc.com/ansible-roles
-
-# Supported platforms are:
-
-* Debian 9
-* Debian 10
-* CentOS 7
-* RHEL 7
-* Ubuntu 16.04
-* Ubuntu 18.04
-* FreeBSD 12+
-
-# Related work
-
-Be aware that this type of install get less love than current repo and can be not in best shape.
-Also be aware that each of them have some limitation. Read limitations sections carefully before apply.
-
-* https://code.getnoc.com/noc/noc-dc -- want to install noc via docker compose and just sneak peak? That is the way.
-* https://code.getnoc.com/noc/noc-k8s -- Already cloud native? Probably it will help you.

ansible/molecule/debian10/molecule.yml

@@ -14,8 +14,6 @@ driver:
   network_id: ${yc_network_id}
   subnet_id: ${yc_subnet_id}
   image_id: ${yc_deb10_disk}
-lint:
-  name: yamllint
 platforms:
   - name: noc-node-debian10
     distr: debian10
@@ -56,7 +54,3 @@ scenario:
     - verify
     - side_effect
     - destroy
-verifier:
-  name: goss
-  lint:
-    name: yamllint

ansible/molecule/debian9/molecule.yml

@@ -14,8 +14,6 @@ driver:
   network_id: ${yc_network_id}
   subnet_id: ${yc_subnet_id}
   image_id: ${yc_deb9_disk}
-lint:
-  name: yamllint
 platforms:
   - name: noc-node-debian9
     distr: debian9
@@ -56,7 +54,3 @@ scenario:
     - verify
     - side_effect
     - destroy
-verifier:
-  name: goss
-  lint:
-    name: yamllint

ansible/molecule/default/molecule.yml

@@ -14,8 +14,6 @@ driver:
   network_id: ${yc_network_id}
   subnet_id: ${yc_subnet_id}
   image_id: ${yc_cen7_disk}
-lint:
-  name: yamllint
 platforms:
   - name: noc-node-centos7
     distr: centos7
@@ -56,7 +54,3 @@ scenario:
     - verify
     - side_effect
     - destroy
-verifier:
-  name: goss
-  lint:
-    name: yamllint

ansible/molecule/freebsd/molecule.yml

@@ -15,8 +15,6 @@ driver:
   ssh_identity_file: "/tmp/temporary_ssh_key"
   ssh_user: vagrant
   ssh_port: 22
-lint:
-  name: yamllint
 platforms:
   - name: noc-node-freebsd12
     template: packer-freebsd12
@@ -59,7 +57,3 @@ scenario:
     - verify
     - side_effect
     - destroy
-verifier:
-  name: goss
-  lint:
-    name: yamllint

ansible/molecule/oel7/molecule.yml

@@ -15,8 +15,6 @@ driver:
   ssh_identity_file: "/tmp/temporary_ssh_key"
   ssh_user: vagrant
   ssh_port: 22
-lint:
-  name: yamllint
 platforms:
   - name: noc_node_oel7
     template: packer-oel7
@@ -59,7 +57,3 @@ scenario:
     - verify
     - side_effect
     - destroy
-verifier:
-  name: goss
-  lint:
-    name: yamllint

ansible/molecule/ubuntu16/molecule.yml

@@ -14,8 +14,6 @@ driver:
   network_id: ${yc_network_id}
   subnet_id: ${yc_subnet_id}
   image_id: ${yc_ub16_disk}
-lint:
-  name: yamllint
 platforms:
   - name: noc-node-ubuntu16
     distr: ubuntu16
@@ -56,7 +54,3 @@ scenario:
     - verify
     - side_effect
     - destroy
-verifier:
-  name: goss
-  lint:
-    name: yamllint

ansible/molecule/ubuntu18/molecule.yml

@@ -14,8 +14,6 @@ driver:
   network_id: ${yc_network_id}
   subnet_id: ${yc_subnet_id}
   image_id: ${yc_ub18_disk}
-lint:
-  name: yamllint
 platforms:
   - name: noc-node-ubuntu18
     distr: ubuntu18
@@ -56,7 +54,3 @@ scenario:
     - verify
     - side_effect
     - destroy
-verifier:
-  name: goss
-  lint:
-    name: yamllint

ansible/molecule/yc/create.yml

@@ -34,7 +34,6 @@
           'user': "{{ item.item.ssh_user }}"
           'port': "22"
           'identity_file': "{{ molecule_yml.driver.ssh_identity_file }}"
-      register: instance_config_dict
       with_items: "{{ server.results }}"
 
     - name: Create instance config
@@ -43,16 +42,13 @@
 
     - name: Dump instance config
       copy:
-        # NOTE(retr0h): Workaround for Ansible 2.2.
-        #               https://github.com/ansible/ansible/issues/20885
-        content: "{{ instance_conf | to_json | from_json | molecule_to_yaml | molecule_header }}"
+        content: "{{ instance_conf | to_json | from_json }}"
         dest: "{{ molecule_instance_config }}"
 
     - name: Wait for SSH
       wait_for:
         port: "22"
-        host: "{{ item.address }}"
+        host: "{{ instance_config_dict.address }}"
         search_regex: SSH
         delay: 10
         timeout: 600
-      with_items: "{{ lookup('file', molecule_instance_config) | molecule_from_yaml }}"

ansible/molecule/yc/prepare.yml

@@ -20,7 +20,7 @@
 
     - name: Check if not python2 on host
       raw: test -e /usr/bin/python || (apt -qy update && apt install -y python-minimal)
-      when: molecule_yml.platforms[0].distr is match("ubuntu.*|debian.*")
+      when: molecule_yml.platforms[0].distr is match("debian.*")
       changed_when: false
 
     - name: download caches

ansible/molecule/yc/templates/template.tf

@@ -25,7 +25,7 @@ resource "yandex_compute_instance" "vm-1" {
   platform_id = "standard-v2"
 
   scheduling_policy {
-    preemptible = true
+    preemptible = false
   }
 
   boot_disk {

ansible/molecule/yc/tower.yml

 ---
-- import_playbook: ../../system_roles/goss/service.yml
 - import_playbook: ../../system_roles/consul/service.yml
 - import_playbook: ../../system_roles/postgres/service.yml
 - import_playbook: ../../system_roles/mongod/service.yml
 - import_playbook: ../../system_roles/nsqlookupd/service.yml
 - import_playbook: ../../system_roles/consul-template/service.yml
-- import_playbook: ../../system_roles/nats/service.yml
-- import_playbook: ../../system_roles/liftbridge/service.yml
 - import_playbook: ../../system_roles/nsqd/service.yml
 - import_playbook: ../../system_roles/nginx/service.yml
 - import_playbook: ../../system_roles/grafana/service.yml
 - import_playbook: ../../system_roles/clickhouse/service.yml
 - import_playbook: ../../noc_roles/noc/service.yml
 - import_playbook: ../../noc_roles/migrate/service.yml
+- import_playbook: ../../system_roles/goss/service.yml
+- import_playbook: ../../system_roles/liftbridge/service.yml
+- import_playbook: ../../system_roles/nats/service.yml

ansible/noc_roles/noc/lookup_plugins/supervisorctl_key.py

@@ -3,6 +3,7 @@
 
 # Python modules
 import os
+import base64
 
 # Ansible modules
 from ansible.plugins.lookup import LookupBase
@@ -30,7 +31,7 @@ class LookupModule(LookupBase):
                 if not os.path.exists(d):
                     os.makedirs(d, mode=0o700)
                 # Generate key file
-                key = os.urandom(48).encode("base64").strip()
+                key = str(base64.b64encode(os.urandom(48)).decode())
                 with open(path, "w") as f:
                     os.chmod(path, 0o600)
                     f.write(key)

ansible/noc_roles/noc/templates/noc.goss.yml.j2

@@ -145,7 +145,7 @@ command:
     - '!/\?\?\\s/'
     - '!/\\sM/'
     stderr: []
-    timeout: 10000
+    timeout: 20000
 {% if noc_python_interpreter != "python3" %}
   {{ noc_root }}/bin/python -c 'import sys;print sys.getdefaultencoding()':
     exit-status: 0

ansible/noc_roles/web/tasks/main.yml

 ---
-- name: Install packages for web
-  pip:
-    chdir: "{{ noc_root }}"
-    requirements: "requirements/web.txt"
-    virtualenv: "{{ noc_root }}"
-    extra_args: "--trusted-host cdn.getnoc.com --find-links https://cdn.getnoc.com/npkg/simple/ --upgrade"
-  environment:
-    https_proxy: "{{ http_proxy }}"
-    http_proxy: "{{ http_proxy }}"
-    CFLAGS: "{{ pip_cflags | default('') }}"
-    LDFLAGS: "{{ pip_ldflags | default('') }}"
-  when:
-    - not single_req.stat.exists
-  tags:
-    - requirements
 
 - name: "Include OS-specific tasks"
   include_tasks: "os/{{ ansible_distribution }}/main.yml"

ansible/system_roles/consul/lookup_plugins/consul_key.py

@@ -3,6 +3,7 @@
 
 # Python modules
 import os
+import base64
 
 # Ansible modules
 from ansible.plugins.lookup import LookupBase
@@ -30,7 +31,7 @@ class LookupModule(LookupBase):
                 if not os.path.exists(d):
                     os.makedirs(d, mode=0o700)
                 # Generate key file
-                key = os.urandom(16).encode("base64").strip()
+                key = str(base64.b64encode(os.urandom(16)).decode())
                 with open(path, "w") as f:
                     os.chmod(path, 0o600)
                     f.write(key)

ansible/system_roles/consul/tasks/config.yml

@@ -104,4 +104,4 @@
   when:
     - "'Linux' in ansible_system"
     - "'prod' in noc_env_type"
-    - " ansible_kernel.split('-') is version('4.1.0', '<')"
+    - "ansible_kernel is version('4.1.0', '<')"