diff --git a/.docker/Dockerfile.build b/.docker/Dockerfile.build index ed63a84c8769d865f2af488bff5e776260c9bfaa..c9dff0b7dfea14e0c3bf2ee3f609046a10ebdaf6 100644 --- a/.docker/Dockerfile.build +++ b/.docker/Dockerfile.build @@ -9,9 +9,6 @@ ENV\ NOC_THREAD_STACK_SIZE=524288 \ NOC_PYTHON_INTERPRETER=/usr/bin/python3 -COPY requirements/docker.txt /opt/noc/requirements/docker.txt -COPY requirements/test.txt /opt/noc/requirements/test.txt - RUN apk add --update --no-cache \ ca-certificates \ libpq \ @@ -37,8 +34,12 @@ RUN apk add --no-cache --virtual .build-deps \ cmake \ proj-util \ proj-dev -RUN pip3 install --src=/tmp/src --no-cache --upgrade -r /opt/noc/requirements/docker.txt \ - && pip3 install --src=/tmp/src --no-cache --upgrade -r /opt/noc/requirements/test.txt + +COPY requirements.txt /opt/noc/requirements.txt +COPY scripts/build/get-noc-requirements.py /opt/noc/scripts/build/get-noc-requirements.py +RUN ./scripts/build/get-noc-requirements.py \ + testing activator classifier login-ldap login-radius prod-tools cache-redis cache-memcached \ + | pip3 install --src=/tmp/src --no-cache --upgrade -r /dev/stdin RUN pip3 install cython diff --git a/.gitignore b/.gitignore index 40bc0fc65fc664944255e16a74d1add9b8e407b8..d52997c7478f76e7c1ec255ebca3d56a26388629 100644 --- a/.gitignore +++ b/.gitignore @@ -12,10 +12,6 @@ include/* lib/python* lib64 local/* -static/doc/* -contrib/lib/* -contrib/bin/* -contrib/share/* .* !/.docker/* !/.gitignore @@ -25,9 +21,9 @@ contrib/share/* !/.pylintrc !/.coveragerc !/.prettierrc +!/scripts/build/ ext_* etc/requirements/local.txt -static/pkg/ share/ static/test/ pip-selfcheck.json diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..6ebb216f907330108aac5373cebcbb0e420eaf03 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,45 @@ +# Base layer containing system packages and requirements +FROM alpine:3.12 AS base +ENV\ + DJANGO_SETTINGS_MODULE=noc.settings \ + NOC_THREAD_STACK_SIZE=524288 \ + NOC_PYTHON_INTERPRETER=/usr/bin/python3 \ + PYTHONPATH=/opt/noc:/opt:/usr/bin/python3.8 \ + PROJ_DIR=/usr +ADD thin.tgz / +RUN\ + apk add --no-cache \ + python3 \ + libffi \ + libpq \ + snappy \ + cyrus-sasl \ + ca-certificates \ + tzdata \ + git \ + zlib \ + proj \ + && apk add --virtual .build-dependencies --no-cache \ + build-base \ + python3-dev \ + libffi-dev \ + postgresql-dev \ + snappy-dev \ + zlib-dev \ + autoconf \ + automake \ + libtool \ + m4 \ + cyrus-sasl-dev \ + cmake \ + proj-util \ + proj-dev \ + && chmod a+x /usr/bin/get-noc-requirements \ + && /usr/bin/python3 -m ensurepip --upgrade \ + && /usr/bin/pip3 install --upgrade pip wheel \ + && (/usr/bin/get-noc-requirements activator dev test cython | /usr/bin/pip3 install -r /dev/stdin )\ + && /usr/bin/cythonize -i /opt/noc/speedup/*.pyx \ + && /usr/bin/pip3 uninstall -y Cython \ + && rm /requirements.txt \ + && apk del .build-dependencies \ + && rm -rf /var/cache/apk/* /root/.cache/ diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0d6868138893be40d03e13bef212dc7f353cf5a7 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,94 @@ +# ---------------------------------------------------------------------- +# NOC Python requirements +# Following extras can be set: +# * activator +# * cache-redis +# * cache-memcached +# * classifier +# * cython +# * dev +# * login-ldap +# * login-pam +# * login-radius +# * prod-tools +# * supervisor +# * testing + +# ---------------------------------------------------------------------- +# Copyright (C) 2007-2020 The NOC Project +# See LICENSE for details +# ---------------------------------------------------------------------- + +# Basic node requirements +Django==3.0.6 +Jinja2==2.8 +PyYAML==5.3.1 +Pygments==2.2.0 +XlsxWriter==0.9.3 +atomicl==0.1.1 +blinker==1.3 +bsdiff4==1.1.9 +cachetools==4.1.0 +crontab==0.22.6 +csiphash==0.0.5 +demjson==2.2.4 +fs==2.4.11 +future==0.17.1 +geojson==2.5.0 +geopy==0.97 +http-parser==0.9.0 +jsonschema==2.4.0 +manhole==1.6.0 +markerlib==0.6.0 +mistune==0.5 +mongoengine==0.20.0 +networkx==2.4 +numpy==1.18.3 +progressbar2==3.51.0 +psycopg2==2.8.5 +pymongo==3.10.1 +git+https://github.com/ploxiln/pynsq.git@a9f9a75eaec62904f06d497397d1b019cdb9581a +pyproj==2.6.0 +python-consul==1.1.0 +python-creole==1.3.2 +python-dateutil==2.4.0 +pytz==2020.1 +requests==2.20.0 +setproctitle==1.1.10 +tornadis==0.8.1 +tornado==6.0.4 +ujson-x==1.37 +yappi==1.2.5 +# Activator requirements +ssh2-python==0.18.0.post1; extra == "activator" +# Classifier +bitarray==0.8.1; extra == "classifier" +# Cache-Redis +hiredis==1.0.1; extra == "cache-redis" +redis==3.5.2; extra == "cache-redis" +# Cache-Memcached +pylibmc==1.5.1; extra == "cache-memcached" +# Dev requirements +Babel==2.3.4; extra == "dev" +ipython==7.14.0; extra == "dev" +pojson==0.6; extra == "dev" +# Login LDAP +ldap3==2.6.1; extra == "login-ldap" +# Login PAM +pam==0.1.4; extra == "login-pam" +# Login RADIUS +pyrad==2.0; extra == "login-radius" +# Standalone node with supervisor +supervisor==4.1.0; extra == "supervisor" +supervisor-serialrestart==0.1.1; extra == "supervisor" +# Additional prod tools +alerta==4.7.16; extra == "prod-tools" +raven==6.1.0; extra == "prod-tools" +supervisor-wildcards==0.1.3; extra == "prod-tools" +# Test +Coverage[toml]==5.0.3; extra == "testing" +pytest==5.4.1; extra == "testing" +pytest-assume==2.2.1; extra == "testing" +pytest-dependency==0.5.1; extra == "testing" +# Cython +Cython==0.29.20; extra == "cython" diff --git a/requirements/activator.txt b/requirements/activator.txt deleted file mode 100644 index f4105cee6ab769d924fbc9e6eb142a5c70325016..0000000000000000000000000000000000000000 --- a/requirements/activator.txt +++ /dev/null @@ -1 +0,0 @@ -ssh2-python==0.18.0.post1 diff --git a/requirements/classifier.txt b/requirements/classifier.txt deleted file mode 100644 index 35d412b044992a1a337fc7c46243e4c588f9b8a1..0000000000000000000000000000000000000000 --- a/requirements/classifier.txt +++ /dev/null @@ -1,2 +0,0 @@ -esmre==0.3.1;python_version<="2.7" -bitarray==0.8.1 diff --git a/requirements/dev.txt b/requirements/dev.txt deleted file mode 100644 index b975206d3a68e197c8c39cd41a1dab73a45319ee..0000000000000000000000000000000000000000 --- a/requirements/dev.txt +++ /dev/null @@ -1,4 +0,0 @@ -ipython==7.14.0 -Babel==2.3.4 -pojson==0.6 - diff --git a/requirements/docker.txt b/requirements/docker.txt deleted file mode 100644 index 0608cb795a9163b5f111000408c648530cc32806..0000000000000000000000000000000000000000 --- a/requirements/docker.txt +++ /dev/null @@ -1,48 +0,0 @@ -ssh2-python==0.18.0.post1 -pylibmc==1.5.1 -ldap3==2.6.1 -pyrad==2.0 -atomicl==0.1.1 -bsdiff4==1.1.9 -csiphash==0.0.5 -http-parser==0.9.0 -psycopg2==2.8.5 -pymongo==3.10.1 -geopy==0.97 -geojson==2.5.0 -Django==3.0.5 -blinker==1.3 -Jinja2==2.8 -jsonschema==2.4.0 -git+https://github.com/ploxiln/pynsq.git@a9f9a75eaec62904f06d497397d1b019cdb9581a -pyproj==2.6.0 -python-creole==1.3.2 -pytz>=2014.4 -python-consul==1.1.0 -python-dateutil==2.4.0 -progressbar2==3.51.0 -manhole>=1.3.0 -mistune==0.5 -mongoengine==0.20.0 -networkx==2.2 -numpy==1.18.3 -requests==2.20.0 -cachetools==4.1.0 -Pygments==2.2.0 -markerlib>=0.6.0 -PyYAML>=3.11 -setproctitle>=1.1.9 -ujson-x==1.37 -yappi==1.2.5 -XlsxWriter==0.9.3 -sleekxmpp==1.1.11 -raven==6.0.0 -argparse>=1.4.0 -demjson==2.2.4 -crontab==0.22.6 -fs==2.4.11 -tornado==6.0.4 -tornadis==0.8.1 -typing==3.7.4.1 -hiredis==1.0.1 -redis==3.5.2 diff --git a/requirements/login-ldap.txt b/requirements/login-ldap.txt deleted file mode 100644 index d8a192af4896e1337a2ffa47d8f672fa62b99cf4..0000000000000000000000000000000000000000 --- a/requirements/login-ldap.txt +++ /dev/null @@ -1 +0,0 @@ -ldap3==2.6.1 diff --git a/requirements/login-pam.txt b/requirements/login-pam.txt deleted file mode 100644 index 5ed584a68ad7e3d4b0438e11309ef2474a7da8fd..0000000000000000000000000000000000000000 --- a/requirements/login-pam.txt +++ /dev/null @@ -1,2 +0,0 @@ -pam==0.1.4 - diff --git a/requirements/login-radius.txt b/requirements/login-radius.txt deleted file mode 100644 index 5086613808e6b0b7e40e81ee89e998c4e49ea643..0000000000000000000000000000000000000000 --- a/requirements/login-radius.txt +++ /dev/null @@ -1 +0,0 @@ -pyrad==2.0 diff --git a/requirements/node.txt b/requirements/node.txt deleted file mode 100644 index 8c3be13dc50cdd2ddcb5e3b0017296bc106eb440..0000000000000000000000000000000000000000 --- a/requirements/node.txt +++ /dev/null @@ -1,47 +0,0 @@ -atomicl==0.1.1 -bsdiff4==1.1.9 -csiphash==0.0.5 -http-parser==0.9.0 -psycopg2==2.8.5 -pymongo==3.10.1 -geopy==0.97 -geojson==2.5.0 -Cython>=0.24 -Django==3.0.6 -blinker==1.3 -Jinja2==2.8 -jsonschema==2.4.0 -git+https://github.com/ploxiln/pynsq.git@a9f9a75eaec62904f06d497397d1b019cdb9581a -pyproj==2.6.0 -python-creole==1.3.2 -pytz>=2014.4 -python-consul==1.1.0 -python-dateutil==2.4.0 -progressbar2==3.51.0 -manhole>=1.3.0 -mistune==0.5 -mongoengine==0.20.0 -networkx==2.4 -numpy==1.18.3 -tornado==6.0.4 -typing==3.7.4.1 -requests==2.20.0 -supervisor==4.1.0 -supervisor-serialrestart==0.1.1 -cachetools==4.1.0 -Pygments==2.2.0 -markerlib>=0.6.0 -distribute>=0.7.3;python_version<="2.7" -PyYAML>=3.11 -setproctitle>=1.1.9 -ujson-x==1.37 -yappi==1.2.5 -XlsxWriter==0.9.3 -pylibmc==1.5.1 -demjson==2.2.4 -crontab==0.22.6 -fs==2.4.11 -tornadis==0.8.1 -hiredis==1.0.1 -redis==3.5.2 -future==0.17.1 diff --git a/requirements/notebook.txt b/requirements/notebook.txt deleted file mode 100644 index 64b2fccc7de005d3d811b290176131efab73c3b7..0000000000000000000000000000000000000000 --- a/requirements/notebook.txt +++ /dev/null @@ -1 +0,0 @@ -jupyter==1.0.0 diff --git a/requirements/notifier.txt b/requirements/notifier.txt deleted file mode 100644 index 11d53e89561e48cb43d06e9017dee1a6c5f76dd0..0000000000000000000000000000000000000000 --- a/requirements/notifier.txt +++ /dev/null @@ -1 +0,0 @@ -sleekxmpp==1.1.11 diff --git a/requirements/prod.txt b/requirements/prod.txt deleted file mode 100644 index 8f61e7bce5906cdf36c6d1418b29ef8c79eee657..0000000000000000000000000000000000000000 --- a/requirements/prod.txt +++ /dev/null @@ -1,3 +0,0 @@ -alerta==4.7.16 -raven==6.1.0 -supervisor-wildcards==0.1.3 \ No newline at end of file diff --git a/requirements/test.txt b/requirements/test.txt deleted file mode 100644 index 40c3b12edad05cb35dcbde0ba2b8d9594c3d46df..0000000000000000000000000000000000000000 --- a/requirements/test.txt +++ /dev/null @@ -1,4 +0,0 @@ -Coverage[toml]==5.1 -pytest==5.4.1 -pytest-assume==2.2.1 -pytest-dependency==0.5.1 diff --git a/requirements/web.txt b/requirements/web.txt deleted file mode 100644 index 8b137891791fe96927ad78e64b0aad7bded08bdc..0000000000000000000000000000000000000000 --- a/requirements/web.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/scripts/build/get-noc-requirements.py b/scripts/build/get-noc-requirements.py new file mode 100755 index 0000000000000000000000000000000000000000..68c1ee178ea57b5a3cf5a58430588e02f6179e10 --- /dev/null +++ b/scripts/build/get-noc-requirements.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +# ---------------------------------------------------------------------- +# Dump requirements for given extras +# ---------------------------------------------------------------------- +# Copyright (C) 2007-2020 The NOC Project +# See LICENSE for details +# ---------------------------------------------------------------------- + +# Python modules +import sys +import re + +REQUIREMENTS = "requirements.txt" + +rx_extra = re.compile(r"extra\s*==\s*\"([^\"]+)\"") + + +def is_matched(req_extra, extras): + if extras == "--all": + return True + return req_extra in extras + + +def main(extras): + with open(REQUIREMENTS) as f: + for line in f: + line = line.strip() + if not line: + continue + if not line.startswith("#") and ";" in line: + req, env = line.split(";", 1) + match = rx_extra.search(env) + if match: + req_extra = match.group(1).strip() + if is_matched(req_extra, extras): + new_env = (env[: match.start()] + env[match.end() :]).strip() + if new_env: + print("%s;%s" % (req, new_env)) + else: + print(req) + continue + if not line.startswith("#"): + print(line) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/scripts/build/setup-docker-context.sh b/scripts/build/setup-docker-context.sh new file mode 100755 index 0000000000000000000000000000000000000000..f007a3306a6e799e089727b3cea8ef370da4098f --- /dev/null +++ b/scripts/build/setup-docker-context.sh @@ -0,0 +1,18 @@ +#!/bin/sh +set -x +CTX=build/docker +mkdir -p $CTX +cp Dockerfile $CTX +# Prepare thin context +ROOT=$CTX/thin +echo "Preparing thin context at $ROOT" +rm -r $ROOT +mkdir -p $ROOT +mkdir -p $ROOT/usr/bin +mkdir -p $ROOT/opt/noc/speedup +cp requirements.txt $ROOT +cp scripts/build/get-noc-requirements.py $ROOT/usr/bin/get-noc-requirements +chmod a+x $ROOT/usr/bin/get-noc-requirements +cp speedup/*.pyx $ROOT/opt/noc/speedup +(cd $ROOT && tar cfz ../thin.tgz .) +rm -r $ROOT \ No newline at end of file