.gitlab-ci.yml 10.9 KB
Newer Older
Aleksey Shirokih's avatar
Aleksey Shirokih committed
1
---
Aleksey Shirokih's avatar
Aleksey Shirokih committed
2
stages:
uncol's avatar
uncol committed
3
  - Lint
Dmitry Volodin's avatar
Dmitry Volodin committed
4
  - Prebuild
uncol's avatar
uncol committed
5
6
7
  - Test
  - Build
  - Upload
Aleksey Shirokih's avatar
Aleksey Shirokih committed
8

EKbfh's avatar
EKbfh committed
9
include:
Dmitry Volodin's avatar
Dmitry Volodin committed
10
  - project: "noc/common-jobs"
EKbfh's avatar
EKbfh committed
11
    ref: stable
Dmitry Volodin's avatar
Dmitry Volodin committed
12
13
14
15
    file:
      - /templates/upload.yml
      - /templates/build_rust.yml
  - local: /ansible/.ansible-ci.yml
EKbfh's avatar
EKbfh committed
16

EKbfh's avatar
EKbfh committed
17
18
workflow:
  rules:
EKbfh's avatar
EKbfh committed
19
20
21
22
23
    - if: $CI_MERGE_REQUEST_IID
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
    - if: '$CI_PIPELINE_SOURCE == "web"'
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
EKbfh's avatar
EKbfh committed
24

EKbfh's avatar
EKbfh committed
25
#Linters section
Dmitry Volodin's avatar
Dmitry Volodin committed
26
check labels:
uncol's avatar
uncol committed
27
  stage: Lint
28
  image: registry.getnoc.com/infrastructure/noc-py-lint:master
Dmitry Volodin's avatar
Dmitry Volodin committed
29
  script:
Dmitry Volodin's avatar
Dmitry Volodin committed
30
    - FLIST=$(git --no-pager diff --name-only $(git merge-base --fork-point origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME) $CI_COMMIT_SHA)
Dmitry Volodin's avatar
Dmitry Volodin committed
31
    - python ./scripts/check-labels.py --verbose --junit-report=build/reports/labels-junit.xml $FLIST
Dmitry Volodin's avatar
Dmitry Volodin committed
32
33
34
35
  only:
    - merge_requests
  tags:
    - docker
36
37
38
39
  artifacts:
    reports:
      junit: ./build/reports/labels-junit.xml
    expire_in: 7 days
EKbfh's avatar
EKbfh committed
40
  allow_failure: true
41
  retry: 2
Dmitry Volodin's avatar
Dmitry Volodin committed
42

Aleksey Shirokih's avatar
Aleksey Shirokih committed
43
flake8:
uncol's avatar
uncol committed
44
  stage: Lint
45
  image: registry.getnoc.com/infrastructure/noc-py-lint:master
Aleksey Shirokih's avatar
Aleksey Shirokih committed
46
  script:
Aleksey Shirokih's avatar
debug    
Aleksey Shirokih committed
47
    - set -x
Dmitry Volodin's avatar
Dmitry Volodin committed
48
    - FLIST=$(git --no-pager diff --diff-filter=d --name-only $(git merge-base --fork-point origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME) $CI_COMMIT_SHA  | egrep ".py$" | grep -v ".docker/"|| true)
49
50
51
    - >
      if [ ! -z "$FLIST" ];
      then
Dmitry Volodin's avatar
Dmitry Volodin committed
52
53
        mkdir -p build/reports;
        flake8 --format junit-xml $FLIST > build/reports/flake8-junit.xml || flake8 $FLIST;
54
55
56
57
      else
        echo "No files to lint"
        true;
      fi
Dmitry Volodin's avatar
Dmitry Volodin committed
58
  only:
Dmitry Volodin's avatar
Dmitry Volodin committed
59
60
61
62
63
    refs:
      - merge_requests
    changes:
      - "*.py"
      - "**/*.py"
Aleksey Shirokih's avatar
Aleksey Shirokih committed
64
65
  tags:
    - docker
Dmitry Volodin's avatar
Dmitry Volodin committed
66
67
68
69
  artifacts:
    reports:
      junit: ./build/reports/flake8-junit.xml
    expire_in: 7 days
Dmitry Volodin's avatar
Dmitry Volodin committed
70
  allow_failure: false
Aleksey Shirokih's avatar
Aleksey Shirokih committed
71

Dmitry Volodin's avatar
Dmitry Volodin committed
72
black:
uncol's avatar
uncol committed
73
  stage: Lint
74
  image: registry.getnoc.com/infrastructure/noc-py-lint:master
Aleksey Shirokih's avatar
Aleksey Shirokih committed
75
76
  script:
    - set -x
Dmitry Volodin's avatar
Dmitry Volodin committed
77
    - FLIST=$(git --no-pager diff --diff-filter=d --name-only $(git merge-base --fork-point origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME) $CI_COMMIT_SHA  | egrep ".py$" | grep -v ".docker/"|| true)
78
79
80
    - >
      if [ ! -z "$FLIST" ];
      then
e_zombie's avatar
e_zombie committed
81
        black --check --diff $FLIST;
82
83
84
85
      else
        echo "No files to lint"
        true;
      fi
Dmitry Volodin's avatar
Dmitry Volodin committed
86
  only:
Dmitry Volodin's avatar
Dmitry Volodin committed
87
88
89
90
91
    refs:
      - merge_requests
    changes:
      - "*.py"
      - "**/*.py"
Aleksey Shirokih's avatar
Aleksey Shirokih committed
92
93
  tags:
    - docker
Dmitry Volodin's avatar
Dmitry Volodin committed
94
  allow_failure: false
Aleksey Shirokih's avatar
Aleksey Shirokih committed
95

Dmitry Volodin's avatar
Dmitry Volodin committed
96
97
# @todo: Rust fmt

Dmitry Volodin's avatar
Dmitry Volodin committed
98
mypy:
uncol's avatar
uncol committed
99
  stage: Lint
100
  image: registry.getnoc.com/infrastructure/noc-py-lint:master
Dmitry Volodin's avatar
Dmitry Volodin committed
101
102
  script:
    - set -x
Dmitry Volodin's avatar
Dmitry Volodin committed
103
    - FLIST=$(git --no-pager diff --diff-filter=d --name-only $(git merge-base --fork-point origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME) $CI_COMMIT_SHA  | egrep ".py$" | grep -v ".docker/"|| true)
Dmitry Volodin's avatar
Dmitry Volodin committed
104
105
106
    - >
      if [ ! -z "$FLIST" ];
      then
Dmitry Volodin's avatar
Dmitry Volodin committed
107
        mypy --follow-imports=skip --ignore-missing-imports $FLIST;
Dmitry Volodin's avatar
Dmitry Volodin committed
108
109
110
111
112
      else
        echo "No files to lint"
        true;
      fi
  only:
Dmitry Volodin's avatar
Dmitry Volodin committed
113
114
115
116
117
    refs:
      - merge_requests
    changes:
      - "*.py"
      - "**/*.py"
Dmitry Volodin's avatar
Dmitry Volodin committed
118
119
120
121
  tags:
    - docker
  allow_failure: true

122
prettier:
uncol's avatar
uncol committed
123
  stage: Lint
124
125
126
  image: registry.getnoc.com/infrastructure/prettier:master
  script:
    - set -x
Dmitry Volodin's avatar
Dmitry Volodin committed
127
    - FLIST=$(git --no-pager diff --diff-filter=d --name-only $(git merge-base --fork-point origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME) $CI_COMMIT_SHA  | egrep "\.(js|json|yml|css|md)$" | grep -v ".docker/"|| true)
128
129
130
131
132
133
134
135
136
    - >
      if [ ! -z "$FLIST" ];
      then
        /usr/local/bin/prettier --config=.prettierrc --check $FLIST
      else
        echo "No files to lint"
        true;
      fi
  only:
Dmitry Volodin's avatar
Dmitry Volodin committed
137
138
139
140
141
142
143
    refs:
      - merge_requests
    changes:
      - "*.js"
      - "*.json"
      - "*.css"
      - "*.yml"
Dmitry Volodin's avatar
Dmitry Volodin committed
144
      - "*.md"
Dmitry Volodin's avatar
Dmitry Volodin committed
145
146
147
148
      - "**/*.js"
      - "**/*.json"
      - "**/*.css"
      - "**/*.yml"
Dmitry Volodin's avatar
Dmitry Volodin committed
149
      - "**/*.md"
150
151
  tags:
    - docker
Dmitry Volodin's avatar
Dmitry Volodin committed
152
  allow_failure: false
153

Aleksey Shirokih's avatar
Aleksey Shirokih committed
154
pylint:
uncol's avatar
uncol committed
155
  stage: Lint
156
  image: registry.getnoc.com/infrastructure/noc-py-lint:master
Aleksey Shirokih's avatar
Aleksey Shirokih committed
157
  script:
Aleksey Shirokih's avatar
debug    
Aleksey Shirokih committed
158
    - set -x
Dmitry Volodin's avatar
Dmitry Volodin committed
159
    - FLIST=$(git --no-pager diff --diff-filter=d --name-only $(git merge-base --fork-point origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME) $CI_COMMIT_SHA | egrep ".py$" | grep -v ".docker/"|| true)
160
161
162
163
164
165
166
167
    - >
      if [ ! -z "$FLIST" ];
      then
        pylint "$FLIST";
      else
        echo "No files to lint"
        true;
      fi
Dmitry Volodin's avatar
Dmitry Volodin committed
168
  only:
Dmitry Volodin's avatar
Dmitry Volodin committed
169
170
171
172
173
    refs:
      - merge_requests
    changes:
      - "*.py"
      - "**/*.py"
Aleksey Shirokih's avatar
Aleksey Shirokih committed
174
175
176
  tags:
    - docker

EKbfh's avatar
EKbfh committed
177
# Test python code section
Dmitry Volodin's avatar
Dmitry Volodin committed
178
py:test:
uncol's avatar
uncol committed
179
  stage: Test
180
  before_script:
Dmitry Volodin's avatar
Dmitry Volodin committed
181
    - DOCKER_COMPOSE_PROJECT=${CI_COMMIT_REF_SLUG}-${CI_JOB_ID}-pytests
182
183
    - mv .dockerignore .dockerignore.prod
    - cp .dockerignore.tests .dockerignore
Dmitry Volodin's avatar
Dmitry Volodin committed
184
    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
Aleksey Shirokih's avatar
Aleksey Shirokih committed
185
  script:
186
    # see docker/docker-compose.yml for complete list of migrations
Dmitry Volodin's avatar
Dmitry Volodin committed
187
    - ls -al
188
189
190
    - >
      docker-compose
      -f .docker/docker-compose.yml
Dmitry Volodin's avatar
Dmitry Volodin committed
191
      -p "${DOCKER_COMPOSE_PROJECT}"
Dmitry Volodin's avatar
Dmitry Volodin committed
192
      build pytests
193
194
195
    - >
      docker-compose
      -f .docker/docker-compose.yml
Dmitry Volodin's avatar
Dmitry Volodin committed
196
      -p "${DOCKER_COMPOSE_PROJECT}"
197
      up
Dmitry Volodin's avatar
Dmitry Volodin committed
198
199
      --exit-code-from pytests
      pytests
200
  after_script:
Dmitry Volodin's avatar
Dmitry Volodin committed
201
202
    - DOCKER_COMPOSE_PROJECT=${CI_COMMIT_REF_SLUG}-${CI_JOB_ID}-pytests
    - JUNIT_XML_REPORT=build/reports/py-test-junit.xml
203
204
205
    - mkdir -p build/reports/
    - >
      docker cp
Dmitry Volodin's avatar
Dmitry Volodin committed
206
      "${DOCKER_COMPOSE_PROJECT}_pytests_1:/opt/noc/${JUNIT_XML_REPORT}"
207
      "${JUNIT_XML_REPORT}"
208
    - mv .dockerignore.prod .dockerignore
209
210
211
    - >
      docker-compose
      -f .docker/docker-compose.yml
Dmitry Volodin's avatar
Dmitry Volodin committed
212
      -p "${DOCKER_COMPOSE_PROJECT}"
213
      down -v
214
  rules:
215
216
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      changes:
217
218
219
220
        - "*.py"
        - "**/*.py"
        - requirements.txt
        - "**/*.json"
221
222
  tags:
    - shell
223
  allow_failure: false
Dmitry Volodin's avatar
Dmitry Volodin committed
224
225
  artifacts:
    reports:
Dmitry Volodin's avatar
Dmitry Volodin committed
226
      junit: ./build/reports/py-test-junit.xml
227
    expire_in: 7 days
228

EKbfh's avatar
EKbfh committed
229
# Build Docs section
230
build:Docs:ru:
231
  stage: Build
232
233
234
235
236
237
238
239
240
241
242
  environment: docs
  image: registry.getnoc.com/infrastructure/mkdocs:master
  script:
    - set -x
    - mkdocs build --config-file=docs/ru/mkdocs.yml
  tags:
    - docker
  artifacts:
    paths:
      - build/docs/ru
    expire_in: 2 hours
EKbfh's avatar
EKbfh committed
243
  rules:
244
    - if: ('$CI_PIPELINE_SOURCE == "merge_request_event"' || '$CI_COMMIT_BRANCH == "master"')
EKbfh's avatar
EKbfh committed
245
      changes:
246
247
        - "docs/*"
        - "docs/**/*"
248

EKbfh's avatar
EKbfh committed
249
build:Docs:en:
uncol's avatar
uncol committed
250
  stage: Build
251
  environment: docs
Dmitry Volodin's avatar
Dmitry Volodin committed
252
  image: registry.getnoc.com/infrastructure/mkdocs:master
253
  script:
254
    - set -x
Dmitry Volodin's avatar
Dmitry Volodin committed
255
    - mkdocs build --config-file=docs/en/mkdocs.yml
256
257
258
259
260
  tags:
    - docker
  artifacts:
    paths:
      - build/docs/en
Aleksey Shirokih's avatar
Aleksey Shirokih committed
261
    expire_in: 2 hours
EKbfh's avatar
EKbfh committed
262
  rules:
263
    - if: ('$CI_PIPELINE_SOURCE == "merge_request_event"' || '$CI_COMMIT_BRANCH == "master"')
EKbfh's avatar
EKbfh committed
264
      changes:
265
266
        - "docs/*"
        - "docs/**/*"
267

EKbfh's avatar
EKbfh committed
268
#Build Image section
Dmitry Volodin's avatar
Dmitry Volodin committed
269
build:Release Image:
uncol's avatar
uncol committed
270
  stage: Build
Aleksey Shirokih's avatar
Aleksey Shirokih committed
271
272
  script:
    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
273
    # registry.getnoc.com/noc/noc/code:stable
EKbfh's avatar
EKbfh committed
274
    - docker build --target code -t $CI_REGISTRY_IMAGE/code:$CI_COMMIT_TAG -f Dockerfile .
Aleksey Shirokih's avatar
Aleksey Shirokih committed
275
    # registry.getnoc.com/noc/noc/dev:stable
EKbfh's avatar
EKbfh committed
276
    - docker build --target dev -t $CI_REGISTRY_IMAGE/dev:$CI_COMMIT_TAG -f Dockerfile .
277
    # registry.getnoc.com/noc/noc/static:stable
EKbfh's avatar
EKbfh committed
278
    - docker build --target static -t $CI_REGISTRY_IMAGE/static:$CI_COMMIT_TAG -f Dockerfile .
Dmitry Volodin's avatar
Dmitry Volodin committed
279
280
    - docker push $CI_REGISTRY_IMAGE/code:$CI_COMMIT_TAG
    - docker push $CI_REGISTRY_IMAGE/dev:$CI_COMMIT_TAG
Aleksey Shirokih's avatar
Aleksey Shirokih committed
281
    - docker push $CI_REGISTRY_IMAGE/static:$CI_COMMIT_TAG
282
    - docker rmi $CI_REGISTRY_IMAGE/code:$CI_COMMIT_TAG $CI_REGISTRY_IMAGE/dev:$CI_COMMIT_TAG $CI_REGISTRY_IMAGE/static:$CI_COMMIT_TAG
Aleksey Shirokih's avatar
Aleksey Shirokih committed
283
  tags:
Aleksey Shirokih's avatar
Aleksey Shirokih committed
284
285
286
    - shell
  only:
    - tags
Dmitry Volodin's avatar
Dmitry Volodin committed
287

EKbfh's avatar
EKbfh committed
288
289
290
291
292
293
294
295
296
297
298
299
300
301
build:Master Image:
  stage: Build
  script:
    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
    # registry.getnoc.com/noc/noc/code:stable
    - docker build --target code -t $CI_REGISTRY_IMAGE/code:master -f Dockerfile .
    # registry.getnoc.com/noc/noc/dev:stable
    - docker build --target dev -t $CI_REGISTRY_IMAGE/dev:master -f Dockerfile .
    # registry.getnoc.com/noc/noc/static:stable
    - docker build --target static -t $CI_REGISTRY_IMAGE/static:master -f Dockerfile .
    - docker push $CI_REGISTRY_IMAGE/code:master
    - docker push $CI_REGISTRY_IMAGE/dev:master
    - docker push $CI_REGISTRY_IMAGE/static:master
    - docker rmi $CI_REGISTRY_IMAGE/code:master $CI_REGISTRY_IMAGE/dev:master $CI_REGISTRY_IMAGE/static:master
302
303
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule" && $BUILD_MASTER == "true"'
EKbfh's avatar
EKbfh committed
304
305
306
  tags:
    - shell

EKbfh's avatar
EKbfh committed
307
#Build web section
uncol's avatar
uncol committed
308
309
310
311
312
313
314
315
316
317
318
319
320
build:web:
  stage: Build
  image: node:12
  script:
    - cd scripts/build/web
    - npm install
    - node src/index.js
  tags:
    - docker
  only:
    refs:
      - merge_requests
    changes:
EKbfh's avatar
EKbfh committed
321
322
323
      - "ui/web/**/*.js"
      - "ui/web/**/*.css"
      - "scripts/build/web/**/*"
uncol's avatar
uncol committed
324
325
326
327
328
  artifacts:
    paths:
      - dist/ui/pkg/web
      - dist/ui/pkg/web.debug

EKbfh's avatar
EKbfh committed
329
#Build Rust things section
Dmitry Volodin's avatar
Dmitry Volodin committed
330
331
332
333
334
335
336
337
338
339
340
Prebuild ip.so:
  stage: Prebuild
  image: registry.getnoc.com/infrastructure/build/rust-centos7:master
  extends: .build_rust
  variables:
    BUILD_TARGET: pyip
  artifacts:
    paths:
      - build/rust/release/libip.so

Build dnssync:
Dmitry Volodin's avatar
Dmitry Volodin committed
341
  stage: Build
Dmitry Volodin's avatar
Dmitry Volodin committed
342
343
344
  extends: .build_rust
  variables:
    BUILD_TARGET: dnssync
Dmitry Volodin's avatar
Dmitry Volodin committed
345
346
  artifacts:
    paths:
347
      - build/rust/release/noc-dns-sync
Dmitry Volodin's avatar
Dmitry Volodin committed
348
349
350
351
352
353
354
355
356

Build agent:
  stage: Build
  extends: .build_rust
  variables:
    BUILD_TARGET: agent
  artifacts:
    paths:
      - build/rust/release/noc-agent
Dmitry Volodin's avatar
Dmitry Volodin committed
357

EKbfh's avatar
EKbfh committed
358
# Upload section
Aleksey Shirokih's avatar
Aleksey Shirokih committed
359
upload docs:
uncol's avatar
uncol committed
360
  stage: Upload
Aleksey Shirokih's avatar
Aleksey Shirokih committed
361
  environment: docs
EKbfh's avatar
EKbfh committed
362
363
364
  extends: .upload
  variables:
    SOURCE: build/docs/*
EKbfh's avatar
EKbfh committed
365
    DEST: cdn/$S3_BUCKET/$CI_COMMIT_REF_SLUG/
366
367
368
369
370
371
372
  rules:
    - if: ('$CI_PIPELINE_SOURCE == "merge_request_event"')
      when: never
    - if: ('$CI_COMMIT_BRANCH == "master"')
      changes:
        - "docs/*"
        - "docs/**/*"
EKbfh's avatar
EKbfh committed
373

Dmitry Volodin's avatar
Dmitry Volodin committed
374
375
376
377
378
379
380
381
382
383
384
385
386
387
upload rust libs:
  stage: Upload
  environment: libs
  extends: .upload
  variables:
    SOURCE: build/rust/release/libip.so
    DEST: cdn/$S3_BUCKET/libs/
    BUILD_TARGET: pyip
  rules:
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
      changes:
        - rust/$BUILD_TARGET/*
        - rust/$BUILD_TARGET/**/*

EKbfh's avatar
EKbfh committed
388
389
upload dnssync:
  stage: Upload
EKbfh's avatar
EKbfh committed
390
  environment: dnssync
EKbfh's avatar
EKbfh committed
391
392
393
394
395
396
397
398
399
400
401
  extends: .upload
  variables:
    SOURCE: build/rust/release/noc-dns-sync
    DEST: cdn/$S3_BUCKET/dnssync/
    BUILD_TARGET: dnssync
  rules:
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
      changes:
        - rust/$BUILD_TARGET/*
        - rust/$BUILD_TARGET/**/*

EKbfh's avatar
EKbfh committed
402
#Dependency scanning
EKbfh's avatar
EKbfh committed
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
gemnasium-python-dependency_scanning:
  stage: Test
  allow_failure: true
  variables:
    SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers"
    DS_MAJOR_VERSION: 2
  image:
    name: "$SECURE_ANALYZERS_PREFIX/gemnasium-python:$DS_MAJOR_VERSION"
  before_script:
    - apt install -y libpq-dev
  script:
    - /analyzer run
  tags:
    - docker
  artifacts:
    reports:
      dependency_scanning: gl-dependency-scanning-report.json
  rules:
421
    - if: $DEPENDENCY_SCANNING_DISABLED
EKbfh's avatar
EKbfh committed
422
      when: never
423
424
425
426
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
      when: never
    - if: '$CI_COMMIT_BRANCH &&
        $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && $CI_PIPELINE_SOURCE == "merge_request_event"'
EKbfh's avatar
EKbfh committed
427
      changes:
428
        - requirements.txt