.gitlab-ci.yml 8.85 KB
Newer Older
Aleksey Shirokih's avatar
Aleksey Shirokih committed
1
---
Aleksey Shirokih's avatar
Aleksey Shirokih committed
2
stages:
Aleksey Shirokih's avatar
Aleksey Shirokih committed
3
  - lint
Aleksey Shirokih's avatar
Aleksey Shirokih committed
4
  - test
Dmitry Volodin's avatar
Dmitry Volodin committed
5
  - Build docs
Aleksey Shirokih's avatar
Aleksey Shirokih committed
6
7
  - Build release container
  - Build additional release containers
Dmitry Volodin's avatar
Dmitry Volodin committed
8
  - Upload docs
Aleksey Shirokih's avatar
Aleksey Shirokih committed
9

Dmitry Volodin's avatar
Dmitry Volodin committed
10
11
12
13
check labels:
  stage: lint
  image: registry.getnoc.com/infrastructure/noc-lint:master
  script:
Dmitry Volodin's avatar
Dmitry Volodin committed
14
    - FLIST=$(git --no-pager diff --name-only $(git merge-base origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME $CI_COMMIT_SHA) $CI_COMMIT_SHA)
Dmitry Volodin's avatar
Dmitry Volodin committed
15
16
17
18
19
20
    - python ./scripts/check-labels.py $FLIST
  only:
    - merge_requests
  tags:
    - docker

Aleksey Shirokih's avatar
Aleksey Shirokih committed
21
flake8:
Aleksey Shirokih's avatar
Aleksey Shirokih committed
22
  stage: lint
23
  image: registry.getnoc.com/infrastructure/noc-lint:master
Aleksey Shirokih's avatar
Aleksey Shirokih committed
24
  script:
Aleksey Shirokih's avatar
debug    
Aleksey Shirokih committed
25
    - set -x
Dmitry Volodin's avatar
Dmitry Volodin committed
26
    - FLIST=$(git --no-pager diff --name-only $(git merge-base origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME $CI_COMMIT_SHA) $CI_COMMIT_SHA  | egrep ".py$" | grep -v ".docker/"|| true)
27
28
29
30
31
32
33
34
    - >
      if [ ! -z "$FLIST" ];
      then
        flake8 $FLIST;
      else
        echo "No files to lint"
        true;
      fi
Dmitry Volodin's avatar
Dmitry Volodin committed
35
36
  only:
    - merge_requests
Aleksey Shirokih's avatar
Aleksey Shirokih committed
37
38
39
  tags:
    - docker

Dmitry Volodin's avatar
Dmitry Volodin committed
40
black:
Aleksey Shirokih's avatar
Aleksey Shirokih committed
41
  stage: lint
Dmitry Volodin's avatar
Dmitry Volodin committed
42
  image: registry.getnoc.com/infrastructure/black:master
Aleksey Shirokih's avatar
Aleksey Shirokih committed
43
44
  script:
    - set -x
Dmitry Volodin's avatar
Dmitry Volodin committed
45
    - FLIST=$(git --no-pager diff --diff-filter=d --name-only $(git merge-base origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME $CI_COMMIT_SHA) $CI_COMMIT_SHA  | egrep ".py$" | grep -v ".docker/"|| true)
46
47
48
    - >
      if [ ! -z "$FLIST" ];
      then
Dmitry Volodin's avatar
Dmitry Volodin committed
49
        black --check $FLIST;
50
51
52
53
      else
        echo "No files to lint"
        true;
      fi
Dmitry Volodin's avatar
Dmitry Volodin committed
54
55
  only:
    - merge_requests
Aleksey Shirokih's avatar
Aleksey Shirokih committed
56
57
  tags:
    - docker
Dmitry Volodin's avatar
Dmitry Volodin committed
58
  allow_failure: false
Aleksey Shirokih's avatar
Aleksey Shirokih committed
59

Aleksey Shirokih's avatar
Aleksey Shirokih committed
60
# waiting for https://github.com/rubik/xenon/issues/21
Aleksey Shirokih's avatar
Aleksey Shirokih committed
61
xenon:
Aleksey Shirokih's avatar
Aleksey Shirokih committed
62
63
64
  stage: lint
  image: registry.getnoc.com/infrastructure/noc-lint:master
  script:
Aleksey Shirokih's avatar
debug    
Aleksey Shirokih committed
65
    - set -x
Dmitry Volodin's avatar
Dmitry Volodin committed
66
    - FLIST=$(git --no-pager diff --name-only $(git merge-base origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME $CI_COMMIT_SHA) $CI_COMMIT_SHA | egrep ".py$" | grep -v ".docker/"|| true)
Aleksey Shirokih's avatar
Aleksey Shirokih committed
67
    - x=0
68
69
70
71
72
73
74
75
    - >
      if [ ! -z "$FLIST" ];
      then
        for i in "${FLIST}";
        do
          (xenon --max-absolute B --max-modules B --max-average B "$i"; let x+=$?);
        done;
      fi
Aleksey Shirokih's avatar
Aleksey Shirokih committed
76
    - if [ "$x" -gt 0 ] ; then exit 1; fi
Dmitry Volodin's avatar
Dmitry Volodin committed
77
78
  only:
    - merge_requests
Aleksey Shirokih's avatar
Aleksey Shirokih committed
79
80
81
82
83
84
85
86
  tags:
    - docker
  allow_failure: true

pylint:
  stage: lint
  image: registry.getnoc.com/infrastructure/noc-lint:master
  script:
Aleksey Shirokih's avatar
debug    
Aleksey Shirokih committed
87
    - set -x
Dmitry Volodin's avatar
Dmitry Volodin committed
88
    - FLIST=$(git --no-pager diff --name-only $(git merge-base origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME $CI_COMMIT_SHA) $CI_COMMIT_SHA | egrep ".py$" | grep -v ".docker/"|| true)
89
90
91
92
93
94
95
96
    - >
      if [ ! -z "$FLIST" ];
      then
        pylint "$FLIST";
      else
        echo "No files to lint"
        true;
      fi
Dmitry Volodin's avatar
Dmitry Volodin committed
97
98
  only:
    - merge_requests
Aleksey Shirokih's avatar
Aleksey Shirokih committed
99
100
101
  tags:
    - docker

102
103
104
105
106
pylint3k:
  stage: lint
  image: registry.getnoc.com/infrastructure/noc-lint:master
  script:
    - set -x
Dmitry Volodin's avatar
Dmitry Volodin committed
107
    - FLIST=$(git --no-pager diff --name-only $(git merge-base origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME $CI_COMMIT_SHA) $CI_COMMIT_SHA | egrep ".py$" | grep -v ".docker/"|| true)
108
109
110
111
112
113
114
115
116
117
118
119
120
    - >
      if [ ! -z "$FLIST" ];
      then
        pylint --py3k "$FLIST";
      else
        echo "No files to lint"
        true;
      fi
  only:
    - merge_requests
  tags:
    - docker

Dmitry Volodin's avatar
Dmitry Volodin committed
121
122
123
124
125
bandit:
  stage: lint
  image: registry.getnoc.com/infrastructure/noc-lint:master
  script:
    - set -x
Dmitry Volodin's avatar
Dmitry Volodin committed
126
    - FLIST=$(git --no-pager diff --name-only $(git merge-base origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME $CI_COMMIT_SHA) $CI_COMMIT_SHA | egrep ".py$" | grep -v ".docker/"|| true)
Dmitry Volodin's avatar
Dmitry Volodin committed
127
128
129
130
131
132
133
134
135
136
137
138
139
140
    - >
      if [ ! -z "$FLIST" ];
      then
        bandit "$FLIST";
      else
        echo "No files to lint"
        true;
      fi
  only:
    - merge_requests
  tags:
    - docker
  allow_failure: true

141
142
143
144
145
146
147
148
149
150
151
152
caniusepython3:
  stage: lint
  image: registry.getnoc.com/infrastructure/noc-lint:master
  script:
    - caniusepython3 -r requirements/*.txt
  only:
    - merge_requests
  tags:
    - docker
  allow_failure: true
  when: manual

Aleksey Shirokih's avatar
Aleksey Shirokih committed
153
154
155
156
futurize:
  stage: lint
  image: registry.getnoc.com/infrastructure/noc-lint:master
  script:
Dmitry Volodin's avatar
Dmitry Volodin committed
157
    - FLIST=$(git --no-pager diff --name-only $(git merge-base origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME $CI_COMMIT_SHA) $CI_COMMIT_SHA | egrep ".py$" | grep -v ".docker/"|| true)
Aleksey Shirokih's avatar
Aleksey Shirokih committed
158
    - /bin/future.sh "$FLIST"
Dmitry Volodin's avatar
Dmitry Volodin committed
159
160
  only:
    - merge_requests
Aleksey Shirokih's avatar
Aleksey Shirokih committed
161
162
163
  tags:
    - docker

164
165
166
167
futurize stage2:
  stage: lint
  image: registry.getnoc.com/infrastructure/noc-lint:master
  script:
Dmitry Volodin's avatar
Dmitry Volodin committed
168
    - FLIST=$(git --no-pager diff --name-only $(git merge-base origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME $CI_COMMIT_SHA) $CI_COMMIT_SHA | egrep ".py$" | grep -v ".docker/"|| true)
169
    - /bin/future2.sh "$FLIST"
Dmitry Volodin's avatar
Dmitry Volodin committed
170
171
  only:
    - merge_requests
172
173
  tags:
    - docker
Dmitry Volodin's avatar
Dmitry Volodin committed
174
  allow_failure: true
175

176
Test migrate:
Aleksey Shirokih's avatar
Aleksey Shirokih committed
177
  stage: test
178
179
  before_script:
    - DOCKER_COMPOSE_PROJECT=${CI_COMMIT_REF_SLUG}-${CI_JOB_ID}
Aleksey Shirokih's avatar
Aleksey Shirokih committed
180
  script:
Aleksey Shirokih's avatar
Aleksey Shirokih committed
181
    # see docker/docker-compose.yml for complete list of migrations
182
183
184
185
186
187
188
189
190
191
192
193
    - >
     docker-compose
     -f .docker/docker-compose.yml
     -p "${DOCKER_COMPOSE_PROJECT}-migrate"
     build migrate
    - >
      docker-compose
      -f .docker/docker-compose.yml
      -p "${DOCKER_COMPOSE_PROJECT}-migrate"
      up
      --exit-code-from migrate
      migrate
194
  after_script:
195
196
197
198
199
200
    - DOCKER_COMPOSE_PROJECT=${CI_COMMIT_REF_SLUG}-${CI_JOB_ID}
    - >
      docker-compose
      -f .docker/docker-compose.yml
      -p "${DOCKER_COMPOSE_PROJECT}-migrate"
      down -v
Dmitry Volodin's avatar
Dmitry Volodin committed
201
202
203
204
205
  only:
    refs:
      - merge_requests
      - master
      - /^release-\d+\.\d+/
206
207
208
209
210
211
  tags:
    - shell
  allow_failure: false

Test code:
  stage: test
212
213
  before_script:
    - DOCKER_COMPOSE_PROJECT=${CI_COMMIT_REF_SLUG}-${CI_JOB_ID}
214
  script:
Dmitry Volodin's avatar
Dmitry Volodin committed
215
    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
216
217
218
219
220
221
222
223
224
225
226
227
    - >
      docker-compose
      -f .docker/docker-compose.yml
      -p "${DOCKER_COMPOSE_PROJECT}-tests"
      build tests
    - >
      docker-compose
      -f .docker/docker-compose.yml
      -p "${DOCKER_COMPOSE_PROJECT}-tests"
      up
      --exit-code-from tests
      tests
Aleksey Shirokih's avatar
Aleksey Shirokih committed
228
  after_script:
229
230
231
232
233
234
    - DOCKER_COMPOSE_PROJECT=${CI_COMMIT_REF_SLUG}-${CI_JOB_ID}
    - >
      docker-compose
      -f .docker/docker-compose.yml
      -p "${DOCKER_COMPOSE_PROJECT}-tests"
      down -v
Dmitry Volodin's avatar
Dmitry Volodin committed
235
236
237
238
239
  only:
    refs:
      - merge_requests
      - master
      - /^release-\d+\.\d+/
Aleksey Shirokih's avatar
Aleksey Shirokih committed
240
241
  tags:
    - shell
Dmitry Volodin's avatar
Dmitry Volodin committed
242
  allow_failure: false
Aleksey Shirokih's avatar
Aleksey Shirokih committed
243

244
245
246
Test python 3:
  stage: test
  before_script:
247
    - DOCKER_COMPOSE_PROJECT=${CI_COMMIT_REF_SLUG}-${CI_JOB_ID}
248
  script:
Dmitry Volodin's avatar
Dmitry Volodin committed
249
    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
250
    # see docker/docker-compose.yml for complete list of migrations
251
252
253
254
255
256
257
258
259
260
261
262
    - >
      docker-compose
      -f .docker/docker-compose.yml
      -p "${DOCKER_COMPOSE_PROJECT}-py3"
      build python3
    - >
      docker-compose
      -f .docker/docker-compose.yml
      -p "${DOCKER_COMPOSE_PROJECT}-py3"
      up
      --exit-code-from python3
      python3
263
  after_script:
264
265
266
267
268
269
    - DOCKER_COMPOSE_PROJECT=${CI_COMMIT_REF_SLUG}-${CI_JOB_ID}
    - >
      docker-compose
      -f .docker/docker-compose.yml
      -p "${DOCKER_COMPOSE_PROJECT}-py3"
      down -v
270
271
272
273
274
  tags:
    - shell
  allow_failure: true
  when: manual

Dmitry Volodin's avatar
Dmitry Volodin committed
275
276
Build docs:
  stage: Build docs
Aleksey Shirokih's avatar
Aleksey Shirokih committed
277
  environment: docs
Dmitry Volodin's avatar
Dmitry Volodin committed
278
  image: registry.getnoc.com/infrastructure/docs:master
279
280
  before_script:
    - NOC_SRC_PATH=${CI_PROJECT_DIR}
Dmitry Volodin's avatar
Dmitry Volodin committed
281
  script:
282
    - set -x
Dmitry Volodin's avatar
Dmitry Volodin committed
283
    - mkdir -p docs/src/en/src
284
285
    - sphinx-build -a -E -j auto -b html docs/src/en build/docs/en
    - python ./docs/scripts/build-go.py build/docs/en/objects.inv
Dmitry Volodin's avatar
Dmitry Volodin committed
286
287
288
289
290
  tags:
    - docker
  artifacts:
    paths:
      - build/docs/en
Aleksey Shirokih's avatar
Aleksey Shirokih committed
291
    expire_in: 2 hours
Dmitry Volodin's avatar
Dmitry Volodin committed
292
293
294
  only:
    changes:
      - docs/**
Dmitry Volodin's avatar
Dmitry Volodin committed
295

296
297
298
299
300
301
302
303
304
305
306
307
308
Build Master Docs:
  stage: Build docs
  environment: docs
  image: registry.getnoc.com/infrastructure/docs:master
  script:
    - cd docs/
    - sphinx-build -a -E -b html src/en ../build/docs/en
    - python ./scripts/build-go.py ../build/docs/en/objects.inv
  tags:
    - docker
  artifacts:
    paths:
      - build/docs/en
Aleksey Shirokih's avatar
Aleksey Shirokih committed
309
    expire_in: 2 hours
310
311
312
313
314
  only:
    refs:
      - master
      - /^release-\d+\.\d+/

Aleksey Shirokih's avatar
Aleksey Shirokih committed
315
316
317
318
319
Build release image:
  stage: Build release container
  before_script:
  script:
    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
320
321
    # registry.getnoc.com/noc/noc/code:stable
    - docker build --target code -t $CI_REGISTRY_IMAGE/code:$CI_COMMIT_TAG -f .docker/Dockerfile .
Aleksey Shirokih's avatar
Aleksey Shirokih committed
322
    # registry.getnoc.com/noc/noc/dev:stable
323
324
325
    - docker build --target dev -t $CI_REGISTRY_IMAGE/dev:$CI_COMMIT_TAG -f .docker/Dockerfile .
    # registry.getnoc.com/noc/noc/static:stable
    - docker build --target static -t $CI_REGISTRY_IMAGE/static:$CI_COMMIT_TAG -f .docker/Dockerfile .
Aleksey Shirokih's avatar
Aleksey Shirokih committed
326
327
328
    - docker push $CI_REGISTRY_IMAGE/code:$CI_COMMIT_TAG 
    - docker push $CI_REGISTRY_IMAGE/dev:$CI_COMMIT_TAG 
    - docker push $CI_REGISTRY_IMAGE/static:$CI_COMMIT_TAG
329
    - docker rmi $CI_REGISTRY_IMAGE/code:$CI_COMMIT_TAG $CI_REGISTRY_IMAGE/dev:$CI_COMMIT_TAG $CI_REGISTRY_IMAGE/static:$CI_COMMIT_TAG
Aleksey Shirokih's avatar
Aleksey Shirokih committed
330
  tags:
Aleksey Shirokih's avatar
Aleksey Shirokih committed
331
332
333
    - shell
  only:
    - tags
Dmitry Volodin's avatar
Dmitry Volodin committed
334

Aleksey Shirokih's avatar
Aleksey Shirokih committed
335
upload docs:
Dmitry Volodin's avatar
Dmitry Volodin committed
336
  stage: Upload docs
Aleksey Shirokih's avatar
Aleksey Shirokih committed
337
  environment: docs
Dmitry Volodin's avatar
Dmitry Volodin committed
338
339
340
341
342
343
344
345
346
  image: registry.getnoc.com/infrastructure/s3helper:master
  script:
    - /tmp/mc -q cp -r build/docs/* cdn/$S3_BUCKET/$CI_COMMIT_REF_SLUG
  tags:
    - docker
  only:
    refs:
      - master
      - /^release-\d+\.\d+/