Merge branch 'ansible-nsqd-drop' into 'master'

Remove nsq, nsqlookup services from nodes at deploy

See merge request noc/noc!5949

ansible/molecule/common/tower.yml

@@ -3,11 +3,9 @@
 - import_playbook: ../../system_roles/consul/service.yml
 - import_playbook: ../../system_roles/postgres/service.yml
 - import_playbook: ../../system_roles/mongod/service.yml
-- import_playbook: ../../system_roles/nsqlookupd/service.yml
 - import_playbook: ../../system_roles/consul-template/service.yml
 - import_playbook: ../../system_roles/nats/service.yml
 - import_playbook: ../../system_roles/liftbridge/service.yml
-- import_playbook: ../../system_roles/nsqd/service.yml
 - import_playbook: ../../system_roles/nginx/service.yml
 - import_playbook: ../../system_roles/grafana/service.yml
 - import_playbook: ../../system_roles/clickhouse/service.yml

ansible/molecule/debian10/inv.yml

@@ -90,8 +90,6 @@ all:
           has_svc_nbi: true
           has_svc_nginx: true
           has_svc_noc: true
-          has_svc_nsqd: true
-          has_svc_nsqlookupd: true
           has_svc_kafkasender: true
           has_svc_pgbouncer: true
           has_svc_ping: true
@@ -366,8 +364,6 @@ all:
                 category: internal
                 depends:
                   - noc
-                  - nsqlookupd
-                  - nsqd
                   - clickhouse
                 level: global
                 supervisord:
@@ -660,8 +656,6 @@ all:
                 category: internal
                 depends:
                   - noc
-                  - nsqlookupd
-                  - nsqd
                 level: pool
                 supervisord:
                   command: ./services/ping/service.py
@@ -853,8 +847,6 @@ all:
           noc_version: microservices
           noc_web_host: noc-node-debian10
           node_id: 1
-          nsqd_max_messages_in_memory: 10000
-          nsqd_max_msg_size: 1048576
           pgbouncer_max_clients: 3000
           pgbouncer_max_db_connections: 10
           ping_backup_power: 1
@@ -1161,28 +1153,6 @@ all:
     svc-noc-exec:
       hosts:
         noc-node-debian10: {}
-    svc-nsqd:
-      children:
-        cfg-nsqd-noc-node-debian10:
-          hosts:
-            noc-node-debian10: {}
-        svc-nsqd-read:
-          hosts:
-            noc-node-debian10: {}
-    svc-nsqd-exec:
-      hosts:
-        noc-node-debian10: {}
-    svc-nsqlookupd:
-      children:
-        cfg-nsqlookupd-noc-node-debian10:
-          hosts:
-            noc-node-debian10: {}
-        svc-nsqlookupd-read:
-          hosts:
-            noc-node-debian10: {}
-    svc-nsqlookupd-exec:
-      hosts:
-        noc-node-debian10: {}
     svc-pgbouncer:
       children:
         cfg-pgbouncer-noc-node-debian10:

ansible/molecule/debian11/inv.yml

@@ -90,8 +90,6 @@ all:
           has_svc_nbi: true
           has_svc_nginx: true
           has_svc_noc: true
-          has_svc_nsqd: true
-          has_svc_nsqlookupd: true
           has_svc_kafkasender: true
           has_svc_pgbouncer: true
           has_svc_ping: true
@@ -366,8 +364,6 @@ all:
                 category: internal
                 depends:
                   - noc
-                  - nsqlookupd
-                  - nsqd
                   - clickhouse
                 level: global
                 supervisord:
@@ -660,8 +656,6 @@ all:
                 category: internal
                 depends:
                   - noc
-                  - nsqlookupd
-                  - nsqd
                 level: pool
                 supervisord:
                   command: ./services/ping/service.py
@@ -853,8 +847,6 @@ all:
           noc_version: microservices
           noc_web_host: noc-node-debian11
           node_id: 1
-          nsqd_max_messages_in_memory: 10000
-          nsqd_max_msg_size: 1048576
           pgbouncer_max_clients: 3000
           pgbouncer_max_db_connections: 10
           ping_backup_power: 1

ansible/molecule/debian9/inv.yml

@@ -90,8 +90,6 @@ all:
           has_svc_nbi: true
           has_svc_nginx: true
           has_svc_noc: true
-          has_svc_nsqd: true
-          has_svc_nsqlookupd: true
           has_svc_kafkasender: true
           has_svc_pgbouncer: true
           has_svc_ping: true
@@ -366,8 +364,6 @@ all:
                 category: internal
                 depends:
                   - noc
-                  - nsqlookupd
-                  - nsqd
                   - clickhouse
                 level: global
                 supervisord:
@@ -660,8 +656,6 @@ all:
                 category: internal
                 depends:
                   - noc
-                  - nsqlookupd
-                  - nsqd
                 level: pool
                 supervisord:
                   command: ./services/ping/service.py
@@ -853,8 +847,6 @@ all:
           noc_version: release-20.1
           noc_web_host: noc-node-debian9
           node_id: 1
-          nsqd_max_messages_in_memory: 10000
-          nsqd_max_msg_size: 1048576
           pgbouncer_max_clients: 3000
           pgbouncer_max_db_connections: 10
           ping_backup_power: 1
@@ -1161,28 +1153,6 @@ all:
     svc-noc-exec:
       hosts:
         noc-node-debian9: {}
-    svc-nsqd:
-      children:
-        cfg-nsqd-noc-node-debian9:
-          hosts:
-            noc-node-debian9: {}
-        svc-nsqd-read:
-          hosts:
-            noc-node-debian9: {}
-    svc-nsqd-exec:
-      hosts:
-        noc-node-debian9: {}
-    svc-nsqlookupd:
-      children:
-        cfg-nsqlookupd-noc-node-debian9:
-          hosts:
-            noc-node-debian9: {}
-        svc-nsqlookupd-read:
-          hosts:
-            noc-node-debian9: {}
-    svc-nsqlookupd-exec:
-      hosts:
-        noc-node-debian9: {}
     svc-kafkasender:
       children:
         cfg-kafkasender-noc-node-debian10:

ansible/molecule/default/inv.yml

@@ -89,8 +89,6 @@ all:
           has_svc_nbi: true
           has_svc_nginx: true
           has_svc_noc: true
-          has_svc_nsqd: true
-          has_svc_nsqlookupd: true
           has_svc_kafkasender: true
           has_svc_pgbouncer: true
           has_svc_ping: true
@@ -365,8 +363,6 @@ all:
                 category: internal
                 depends:
                   - noc
-                  - nsqlookupd
-                  - nsqd
                   - clickhouse
                 level: global
                 supervisord:
@@ -659,8 +655,6 @@ all:
                 category: internal
                 depends:
                   - noc
-                  - nsqlookupd
-                  - nsqd
                 level: pool
                 supervisord:
                   command: ./services/ping/service.py
@@ -852,8 +846,6 @@ all:
           noc_version: microservices
           noc_web_host: noc-node-centos7
           node_id: 1
-          nsqd_max_messages_in_memory: 10000
-          nsqd_max_msg_size: 1048576
           pgbouncer_max_clients: 3000
           pgbouncer_max_db_connections: 10
           ping_backup_power: 1
@@ -1160,28 +1152,6 @@ all:
     svc-noc-exec:
       hosts:
         noc-node-centos7: {}
-    svc-nsqd:
-      children:
-        cfg-nsqd-noc-node-centos7:
-          hosts:
-            noc-node-centos7: {}
-        svc-nsqd-read:
-          hosts:
-            noc-node-centos7: {}
-    svc-nsqd-exec:
-      hosts:
-        noc-node-centos7: {}
-    svc-nsqlookupd:
-      children:
-        cfg-nsqlookupd-noc-node-centos7:
-          hosts:
-            noc-node-centos7: {}
-        svc-nsqlookupd-read:
-          hosts:
-            noc-node-centos7: {}
-    svc-nsqlookupd-exec:
-      hosts:
-        noc-node-centos7: {}
     svc-kafkasender:
       children:
         cfg-kafkasender-noc-node-centos7:

ansible/molecule/freebsd/inv.yml

@@ -90,8 +90,6 @@ all:
           has_svc_nbi: true
           has_svc_nginx: true
           has_svc_noc: true
-          has_svc_nsqd: true
-          has_svc_nsqlookupd: true
           has_svc_kafkasender: true
           has_svc_pgbouncer: true
           has_svc_ping: true
@@ -366,8 +364,6 @@ all:
                 category: internal
                 depends:
                   - noc
-                  - nsqlookupd
-                  - nsqd
                   - clickhouse
                 level: global
                 supervisord:
@@ -660,8 +656,6 @@ all:
                 category: internal
                 depends:
                   - noc
-                  - nsqlookupd
-                  - nsqd
                 level: pool
                 supervisord:
                   command: ./services/ping/service.py
@@ -853,8 +847,6 @@ all:
           noc_version: microservices
           noc_web_host: noc-node-freebsd12
           node_id: 1
-          nsqd_max_messages_in_memory: 10000
-          nsqd_max_msg_size: 1048576
           pgbouncer_max_clients: 3000
           pgbouncer_max_db_connections: 10
           ping_backup_power: 1
@@ -1161,28 +1153,6 @@ all:
     svc-noc-exec:
       hosts:
         noc-node-freebsd12: {}
-    svc-nsqd:
-      children:
-        cfg-nsqd-noc-node-freebsd12:
-          hosts:
-            noc-node-freebsd12: {}
-        svc-nsqd-read:
-          hosts:
-            noc-node-freebsd12: {}
-    svc-nsqd-exec:
-      hosts:
-        noc-node-freebsd12: {}
-    svc-nsqlookupd:
-      children:
-        cfg-nsqlookupd-noc-node-freebsd12:
-          hosts:
-            noc-node-freebsd12: {}
-        svc-nsqlookupd-read:
-          hosts:
-            noc-node-freebsd12: {}
-    svc-nsqlookupd-exec:
-      hosts:
-        noc-node-freebsd12: {}
     svc-kafkasender:
       children:
         cfg-kafkasender-noc-node-freebsd12:

ansible/molecule/oel7/inv.yml

@@ -90,8 +90,6 @@ all:
           has_svc_nbi: true
           has_svc_nginx: true
           has_svc_noc: true
-          has_svc_nsqd: true
-          has_svc_nsqlookupd: true
           has_svc_kafkasender: true
           has_svc_pgbouncer: true
           has_svc_ping: true
@@ -366,8 +364,6 @@ all:
                 category: internal
                 depends:
                   - noc
-                  - nsqlookupd
-                  - nsqd
                   - clickhouse
                 level: global
                 supervisord:
@@ -660,8 +656,6 @@ all:
                 category: internal
                 depends:
                   - noc
-                  - nsqlookupd
-                  - nsqd
                 level: pool
                 supervisord:
                   command: ./services/ping/service.py
@@ -853,8 +847,6 @@ all:
           noc_version: microservices
           noc_web_host: noc_node_oel7
           node_id: 1
-          nsqd_max_messages_in_memory: 10000
-          nsqd_max_msg_size: 1048576
           pgbouncer_max_clients: 3000
           pgbouncer_max_db_connections: 10
           ping_backup_power: 1
@@ -1161,28 +1153,6 @@ all:
     svc-noc-exec:
       hosts:
         noc_node_oel7: {}
-    svc-nsqd:
-      children:
-        cfg-nsqd-noc_node_oel7:
-          hosts:
-            noc_node_oel7: {}
-        svc-nsqd-read:
-          hosts:
-            noc_node_oel7: {}
-    svc-nsqd-exec:
-      hosts:
-        noc_node_oel7: {}
-    svc-nsqlookupd:
-      children:
-        cfg-nsqlookupd-noc_node_oel7:
-          hosts:
-            noc_node_oel7: {}
-        svc-nsqlookupd-read:
-          hosts:
-            noc_node_oel7: {}
-    svc-nsqlookupd-exec:
-      hosts:
-        noc_node_oel7: {}
     svc-kafkasender:
       children:
         cfg-kafkasender-noc-node-eol7:

ansible/molecule/ubuntu16/inv.yml

@@ -90,8 +90,6 @@ all:
           has_svc_nbi: true
           has_svc_nginx: true
           has_svc_noc: true
-          has_svc_nsqd: true
-          has_svc_nsqlookupd: true
           has_svc_kafkasender: true
           has_svc_pgbouncer: true
           has_svc_ping: true
@@ -366,8 +364,6 @@ all:
                 category: internal
                 depends:
                   - noc
-                  - nsqlookupd
-                  - nsqd
                   - clickhouse
                 level: global
                 supervisord:
@@ -660,8 +656,6 @@ all:
                 category: internal
                 depends:
                   - noc
-                  - nsqlookupd
-                  - nsqd
                 level: pool
                 supervisord:
                   command: ./services/ping/service.py
@@ -853,8 +847,6 @@ all:
           noc_version: release-20.1
           noc_web_host: noc-node-ubuntu16
           node_id: 1
-          nsqd_max_messages_in_memory: 10000
-          nsqd_max_msg_size: 1048576
           pgbouncer_max_clients: 3000
           pgbouncer_max_db_connections: 10
           ping_backup_power: 1

ansible/molecule/ubuntu18/inv.yml

@@ -90,8 +90,6 @@ all:
           has_svc_nbi: true
           has_svc_nginx: true
           has_svc_noc: true
-          has_svc_nsqd: true
-          has_svc_nsqlookupd: true
           has_svc_kafkasender: true
           has_svc_pgbouncer: true
           has_svc_ping: true
@@ -366,8 +364,6 @@ all:
                 category: internal
                 depends:
                   - noc
-                  - nsqlookupd
-                  - nsqd
                   - clickhouse
                 level: global
                 supervisord:
@@ -660,8 +656,6 @@ all:
                 category: internal
                 depends:
                   - noc
-                  - nsqlookupd
-                  - nsqd
                 level: pool
                 supervisord:
                   command: ./services/ping/service.py
@@ -853,8 +847,6 @@ all:
           noc_version: microservices
           noc_web_host: noc-node-ubuntu18
           node_id: 1
-          nsqd_max_messages_in_memory: 10000
-          nsqd_max_msg_size: 1048576
           pgbouncer_max_clients: 3000
           pgbouncer_max_db_connections: 10
           ping_backup_power: 1
@@ -1161,28 +1153,6 @@ all:
     svc-noc-exec:
       hosts:
         noc-node-ubuntu18: {}
-    svc-nsqd:
-      children:
-        cfg-nsqd-noc-node-ubuntu18:
-          hosts:
-            noc-node-ubuntu18: {}
-        svc-nsqd-read:
-          hosts:
-            noc-node-ubuntu18: {}
-    svc-nsqd-exec:
-      hosts:
-        noc-node-ubuntu18: {}
-    svc-nsqlookupd:
-      children:
-        cfg-nsqlookupd-noc-node-ubuntu18:
-          hosts:
-            noc-node-ubuntu18: {}
-        svc-nsqlookupd-read:
-          hosts:
-            noc-node-ubuntu18: {}
-    svc-nsqlookupd-exec:
-      hosts:
-        noc-node-ubuntu18: {}
     svc-pgbouncer:
       children:
         cfg-pgbouncer-noc-node-ubuntu18:

ansible/molecule/yc/tower.yml

@@ -2,9 +2,7 @@
 - import_playbook: ../../system_roles/consul/service.yml
 - import_playbook: ../../system_roles/postgres/service.yml
 - import_playbook: ../../system_roles/mongod/service.yml
-- import_playbook: ../../system_roles/nsqlookupd/service.yml
 - import_playbook: ../../system_roles/consul-template/service.yml
-- import_playbook: ../../system_roles/nsqd/service.yml
 - import_playbook: ../../system_roles/nginx/service.yml
 - import_playbook: ../../system_roles/grafana/service.yml
 - import_playbook: ../../system_roles/clickhouse/service.yml

ansible/noc_roles/chwriter/meta/tower.yml

@@ -4,7 +4,7 @@ forms:
     description: >
       <strong>Service type</strong>: Internal <br/>
       <strong>Short description</strong>: Clickhouse writer <br/>
-      <strong>Long description</strong>: That one is for writing data from NSQd to Clickhouse<br/>
+      <strong>Long description</strong>: That one is for writing data from Liftbridge to Clickhouse<br/>
       <strong>Scale Recommendations</strong>: At least one per installation.<br/>
       <strong>docs</strong>: <a href=https://docs.getnoc.com/master/en/admin/reference/services/chwriter/ target=_blank>Chwriter</a><br/>
       <strong>Placement constraint</strong>: On node with Clickhouse<br/>
@@ -57,6 +57,4 @@ services:
       stdout_logfile_backups: 3
     depends:
       - noc
-      - nsqlookupd
-      - nsqd
       - clickhouse

ansible/noc_roles/mailsender/meta/tower.yml

@@ -4,7 +4,7 @@ forms:
     description: >
       <strong>Service type</strong>: Internal <br/>
       <strong>Short description</strong>: Sends emails. <br/>
-      <strong>Long description</strong>: Reads mail queue from nsqd and sends via gateway<br/>
+      <strong>Long description</strong>: Reads mail queue from Liftbridge and sends via gateway<br/>
       <strong>Scale Recommendations</strong>: At least one per installation. <br/>
       <strong>docs</strong>: <a href=https://docs.getnoc.com/master/en/admin/reference/services/mailsender/ target=_blank>Mailsender</a><br/>
       <strong>Placement constraint</strong>: No special recommendations.<br/>

ansible/noc_roles/noc/meta/tower.yml

@@ -60,7 +60,6 @@ services:
     depends:
       - postgres
       - mongod
-      - nsqd
       - nginx
       - clickhouse
       - liftbridge

ansible/noc_roles/noc/templates/noc.goss.yml.j2

@@ -154,9 +154,3 @@ command:
     stderr: []
     timeout: 10000
 {% endif %}
-{#  consul catalog nodes -service=nsqd -near=_agent|sed -n 2p|cut -f1 -d " ":#}
-{#    exit-status: 0#}
-{#    stdout:#}
-{#    - {{ ansible_host }}#}
-{#    stderr: []#}
-{#    timeout: 10000#}

ansible/noc_roles/ping/meta/tower.yml

@@ -60,6 +60,4 @@ services:
       user: root
     depends:
       - noc
-      - nsqlookupd
-      - nsqd
       - datastream

ansible/noc_roles/syslogcollector/meta/tower.yml

@@ -4,7 +4,7 @@ forms:
     description: >
       <strong>Service type</strong>: Internal <br/>
       <strong>Short description</strong>: Syslog listener service<br/>
-      <strong>Long description</strong>: Listen for syslog events from network equipment and adds them to nsqd queue<br/>
+      <strong>Long description</strong>: Listen for syslog events from network equipment and adds them to Liftbridge queue<br/>
       <strong>Scale Recommendations</strong>: One per pool. <br/>
       <strong>docs</strong>: <a href=https://docs.getnoc.com/master/en/admin/reference/services/syslogcollector/ target=_blank>Syslogcollector</a><br/>
       <strong>Placement constraint</strong>: On node with activator<br/>

ansible/system_roles/consul-template/tasks/deservice.yml

+---
+- name: Remove {{ ct_service_name }} template
+  file:
+    state: absent
+    path: '{{ consul_template_templates_dir }}/{{ ct_service_source.split(".")[-1] }}'
+  when: not ct_service_source_template | bool
+  notify:
+    - reload consul-template
+
+- name: Remove {{ ct_service_name }} template
+  file:
+    state: absent
+    path: '{{ consul_template_templates_dir }}/{{ ct_service_source.split(".")[-1] }}'
+  when: ct_service_source_template | bool
+  notify:
+    - reload consul-template
+
+- name: Remove template for {{ ct_service_name }}
+  file:
+    state: absent
+    path: "{{ consul_template_config_dir }}/{{ ct_service_name }}.conf"
+  notify:
+    - reload consul-template

ansible/system_roles/consul/tasks/deservice.yml

+---
+- name: Remove "{{ consul_service_name }}" service file
+  file:
+    state: absent
+    path: "{{ consul_configd_path }}/{{ consul_service_name }}.json"
+  notify:
+    - reload consul

ansible/system_roles/nsqd/meta/tower.yml

@@ -5,7 +5,7 @@ forms:
       <strong>Service type</strong>: External <br/>
       <strong>Short description</strong>: NSQd daemon <br/>
       <strong>Long description</strong>: nsqd is the daemon that receives, queues, and delivers messages to clients. <br/>
-      <strong>Scale Recommendations</strong>: Have to be on each node. <br/>
+      <strong>Scale Recommendations</strong>: Don't need anymore <br/>
       <strong>docs</strong>: <a href=http://nsq.io/components/nsqd.html target=_blank>NSQd</a> <br/>
       <strong>Placement constraint</strong>: No special recommendations.<br/>
     max_messages_in_memory:

ansible/system_roles/nsqd/tasks/checks.yml

----
-- block:
-    - name: fail if consul is not selected
-      fail:
-        msg: "Nsqd role have to be set along with consul role to check weather nsqd is alive"
-      when:
-        - has_svc_consul is not defined
-
-    - name: fail if consul-template is not selected
-      fail:
-        msg: "Nsqd role have to be set along with consul-template role to get access to nsqlookup service."
-      when:
-        - has_svc_consul_template is not defined
-  when:
-    - has_svc_nsqd is defined
-  tags:
-    - checks