extapplication.py 10.8 KB
Newer Older
Dmitry Lukhtionov's avatar
Dmitry Lukhtionov committed
1
2
3
# ---------------------------------------------------------------------
# ExtApplication implementation
# ---------------------------------------------------------------------
4
# Copyright (C) 2007-2020 The NOC Project
Dmitry Lukhtionov's avatar
Dmitry Lukhtionov committed
5
6
# See LICENSE for details
# ---------------------------------------------------------------------
Dmitry Volodin's avatar
Dmitry Volodin committed
7

Dmitry Lukhtionov's avatar
Dmitry Lukhtionov committed
8
# Python modules
9
from builtins import str
Dmitry Volodin's avatar
Dmitry Volodin committed
10
import os
Dmitry Volodin's avatar
Dmitry Volodin committed
11

Dmitry Volodin's avatar
Dmitry Volodin committed
12
# Third-party modules
Dmitry Volodin's avatar
Dmitry Volodin committed
13
from django.http import HttpResponse
MaksimSmile13's avatar
MaksimSmile13 committed
14
from django.db.models.query import QuerySet
Dmitry Volodin's avatar
Dmitry Volodin committed
15
import orjson
Dmitry Volodin's avatar
Dmitry Volodin committed
16

Dmitry Lukhtionov's avatar
Dmitry Lukhtionov committed
17
# NOC modules
18
from noc.main.models.favorites import Favorites
19
from noc.main.models.slowop import SlowOp
20
from noc.config import config
Dmitry Volodin's avatar
Dmitry Volodin committed
21
22
from .application import Application, view
from .access import HasPerm, PermitLogged
Dmitry Volodin's avatar
Dmitry Volodin committed
23
24
25


class ExtApplication(Application):
Dmitry Volodin's avatar
Dmitry Volodin committed
26
    menu = None
Dmitry Volodin's avatar
Dmitry Volodin committed
27
    icon = "icon_application_form"
28
    # HTTP Result Codes
Dmitry Volodin's avatar
Dmitry Volodin committed
29
30
31
32
33
34
35
36
    OK = 200
    CREATED = 201
    DELETED = 204
    BAD_REQUEST = 400
    FORBIDDEN = 401
    NOT_FOUND = 404
    CONFLICT = 409
    NOT_HERE = 410
37
    TOO_LARGE = 413
Dmitry Volodin's avatar
Dmitry Volodin committed
38
39
40
    INTERNAL_ERROR = 500
    NOT_IMPLEMENTED = 501
    THROTTLED = 503
41
    # Recognized GET parameters
Dmitry Volodin's avatar
Dmitry Volodin committed
42
43
44
45
46
47
48
49
    ignored_params = ["_dc"]
    page_param = "__page"
    start_param = "__start"
    limit_param = "__limit"
    sort_param = "__sort"
    format_param = "__format"  # List output format
    query_param = "__query"
    only_param = "__only"
Nikolay Fedoseev's avatar
Nikolay Fedoseev committed
50
    in_param = "__in"
51
    fav_status = "fav_status"
52
    default_ordering = []
Dmitry Volodin's avatar
Dmitry Volodin committed
53

Dmitry Volodin's avatar
Dmitry Volodin committed
54
    def __init__(self, *args, **kwargs):
Dmitry Volodin's avatar
Dmitry Volodin committed
55
        super().__init__(*args, **kwargs)
56
        self.document_root = os.path.join("services", "web", "apps", self.module, self.app)
57
        self.row_limit = config.web.api_row_limit
58
        self.unlimited_row_limit = config.web.api_unlimited_row_limit
59
        self.pk = "id"
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
        # Bulk fields API
        self.bulk_fields = []
        for fn in [n for n in dir(self) if n.startswith("bulk_field_")]:
            h = getattr(self, fn)
            if callable(h):
                self.bulk_fields += [h]

    def apply_bulk_fields(self, data):
        """
        Pass data through bulk_field_* handlers to enrich instance_to_dict result
        :param data: dict or list of dicts
        :return: dict or list of dicts
        """
        if not self.bulk_fields:
            return data
        if isinstance(data, dict):
            # Single dict
            nd = [data]
            for h in self.bulk_fields:
                h(nd)
            return data
        # List of dicts
        for h in self.bulk_fields:
            h(data)
        return data
Dmitry Volodin's avatar
Dmitry Volodin committed
85
86

    @property
87
    def js_app_class(self):
Dmitry Volodin's avatar
Dmitry Volodin committed
88
        m, a = self.get_app_id().split(".")
89
        return "NOC.%s.%s.Application" % (m, a)
Dmitry Volodin's avatar
Dmitry Volodin committed
90
91
92
93
94
95

    @property
    def launch_access(self):
        m, a = self.get_app_id().split(".")
        return HasPerm("%s:%s:launch" % (m, a))

Dmitry Volodin's avatar
Dmitry Volodin committed
96
    def deserialize(self, data):
Dmitry Volodin's avatar
Dmitry Volodin committed
97
        return orjson.loads(data)
Dmitry Volodin's avatar
Dmitry Volodin committed
98

99
    def deserialize_form(self, request):
Dmitry Volodin's avatar
Dmitry Volodin committed
100
        return {str(k): v[0] if len(v) == 1 else v for k, v in request.POST.lists()}
101

Dmitry Volodin's avatar
Dmitry Volodin committed
102
    def response(self, content="", status=200):
103
        if not isinstance(content, str):
Dmitry Volodin's avatar
Dmitry Volodin committed
104
            return HttpResponse(
Dmitry Volodin's avatar
Dmitry Volodin committed
105
106
107
                orjson.dumps(content, option=orjson.OPT_SERIALIZE_NUMPY | orjson.OPT_NON_STR_KEYS),
                content_type="text/json; charset=utf-8",
                status=status,
Dmitry Volodin's avatar
Dmitry Volodin committed
108
            )
Dmitry Volodin's avatar
Dmitry Volodin committed
109
        else:
Dmitry Volodin's avatar
Dmitry Volodin committed
110
            return HttpResponse(content, content_type="text/plain; charset=utf-8", status=status)
Dmitry Volodin's avatar
Dmitry Volodin committed
111

112
113
114
115
116
117
118
119
120
121
    def fav_convert(self, item):
        """
        Convert favorite item from string to storage format
        """
        return str(item)

    def get_favorite_items(self, user):
        """
        Returns a set of user's favorite items
        """
Dmitry Volodin's avatar
Dmitry Volodin committed
122
        f = Favorites.objects.filter(user=user.id, app=self.app_id).first()
123
124
125
126
127
        if f:
            return set(f.favorites)
        else:
            return set()

128
129
130
131
    def extra_query(self, q, order):
        # raise NotImplementedError
        return {}, order

Dmitry Volodin's avatar
Dmitry Volodin committed
132
133
134
135
136
137
    def cleaned_query(self, q):
        raise NotImplementedError

    def queryset(self, request, query=None):
        raise NotImplementedError

138
139
140
    def instance_to_dict(self, o):
        raise NotImplementedError

Dmitry Volodin's avatar
Dmitry Volodin committed
141
142
143
144
145
    def list_data(self, request, formatter):
        """
        Returns a list of requested object objects
        """
        # Todo: Fix
146
        if request.method == "POST":
147
            if self.site.is_json(request.META.get("CONTENT_TYPE")):
Dmitry Volodin's avatar
Dmitry Volodin committed
148
                q = orjson.loads(request.body)
149
            else:
Dmitry Volodin's avatar
Dmitry Volodin committed
150
                q = {str(k): v[0] if len(v) == 1 else v for k, v in request.POST.lists()}
151
        else:
Dmitry Volodin's avatar
Dmitry Volodin committed
152
            q = {str(k): v[0] if len(v) == 1 else v for k, v in request.GET.lists()}
153
154
        # Apply row limit if necessary
        limit = q.get(self.limit_param, self.unlimited_row_limit)
155
        if limit:
156
            try:
157
                limit = max(int(limit), 0)
158
            except ValueError:
Andrey Vertiprahov's avatar
Andrey Vertiprahov committed
159
                return HttpResponse(400, "Invalid %s param" % self.limit_param)
160
        if limit and limit < 0:
Andrey Vertiprahov's avatar
Andrey Vertiprahov committed
161
            return HttpResponse(400, "Invalid %s param" % self.limit_param)
Dmitry Volodin's avatar
Dmitry Volodin committed
162
        # page = q.get(self.page_param)
163
        start = q.get(self.start_param) or 0
164
        if start:
165
            try:
166
                start = max(int(start), 0)
167
            except ValueError:
Andrey Vertiprahov's avatar
Andrey Vertiprahov committed
168
                return HttpResponse(400, "Invalid %s param" % self.start_param)
169
        elif start and start < 0:
Andrey Vertiprahov's avatar
Andrey Vertiprahov committed
170
            return HttpResponse(400, "Invalid %s param" % self.start_param)
Dmitry Volodin's avatar
Dmitry Volodin committed
171
172
173
174
175
        query = q.get(self.query_param)
        only = q.get(self.only_param)
        if only:
            only = only.split(",")
        ordering = []
176
        if request.is_extjs and self.sort_param in q:
Dmitry Volodin's avatar
Dmitry Volodin committed
177
178
179
180
181
            for r in self.deserialize(q[self.sort_param]):
                if r["direction"] == "DESC":
                    ordering += ["-%s" % r["property"]]
                else:
                    ordering += [r["property"]]
182
183
184
185
        fs = None
        fav_items = None
        if self.fav_status in q:
            fs = q.pop(self.fav_status) == "true"
186
        xaa, ordering = self.extra_query(q, ordering)
Dmitry Volodin's avatar
Dmitry Volodin committed
187
188
        q = self.cleaned_query(q)
        if None in q:
Dmitry Volodin's avatar
Dmitry Volodin committed
189
190
191
192
193
194
195
196
197
198
199
            w = []
            p = []
            for x in q.pop(None):
                if type(x) in (list, tuple):
                    w += [x[0]]
                    p += x[1]
                else:
                    w += [x]
            xa = {"where": w}
            if p:
                xa["params"] = p
200
201
202
203
            if xaa:
                data = self.queryset(request, query).filter(**q).extra(**xaa)
            else:
                data = self.queryset(request, query).filter(**q).extra(**xa)
204
205
206
        elif xaa:
            # ExtraQuery
            data = self.queryset(request, query).filter(**q).extra(**xaa)
Dmitry Volodin's avatar
Dmitry Volodin committed
207
208
        else:
            data = self.queryset(request, query).filter(**q)
209
210
211
212
213
        # Favorites filter
        if fs is not None:
            fav_items = self.get_favorite_items(request.user)
            if fs:
                data = data.filter(id__in=fav_items)
MaksimSmile13's avatar
MaksimSmile13 committed
214
            elif isinstance(data, QuerySet):  # Model
215
                data = data.exclude(id__in=fav_items)
MaksimSmile13's avatar
MaksimSmile13 committed
216
217
            else:  # Doc
                data = data.filter(id__nin=fav_items)
218
219
220
        # Store unpaged/unordered queryset
        unpaged_data = data
        # Select related records when fetching for models
Dmitry Volodin's avatar
Fix    
Dmitry Volodin committed
221
        if hasattr(data, "_as_sql"):  # For Models only
Dmitry Volodin's avatar
Dmitry Volodin committed
222
223
            data = data.select_related()
        # Apply sorting
224
        ordering = ordering or self.default_ordering
Dmitry Volodin's avatar
Dmitry Volodin committed
225
226
        if ordering:
            data = data.order_by(*ordering)
227
228
        # Apply paging
        if limit:
Dmitry Volodin's avatar
Dmitry Volodin committed
229
            data = data[start : start + limit]
230
231
        # Fetch and format data
        out = [formatter(o, fields=only) for o in data]
Andrey Vertiprahov's avatar
Andrey Vertiprahov committed
232
        if self.row_limit and len(out) > self.row_limit + 1:
233
            return self.response(
Dmitry Volodin's avatar
Dmitry Volodin committed
234
235
                "System records limit exceeded (%d records)" % self.row_limit, status=self.TOO_LARGE
            )
236
237
238
239
240
        # Set favorites
        if not only and formatter == self.instance_to_dict:
            if fav_items is None:
                fav_items = self.get_favorite_items(request.user)
            for r in out:
241
                r[self.fav_status] = r[self.pk] in fav_items
Dmitry Volodin's avatar
Dmitry Volodin committed
242
243
244
        # Bulk update result. Enrich with proper fields
        out = self.clean_list_data(out)
        #
245
        if request.is_extjs:
246
            ld = len(out)
247
            if limit and (ld == limit or start > 0):
248
249
250
                total = unpaged_data.count()
            else:
                total = ld
Dmitry Volodin's avatar
Dmitry Volodin committed
251
            out = {"total": total, "success": True, "data": out}
Dmitry Volodin's avatar
Dmitry Volodin committed
252
253
        return self.response(out, status=self.OK)

Dmitry Volodin's avatar
Dmitry Volodin committed
254
255
256
257
258
259
260
    def clean_list_data(self, data):
        """
        Finally process list_data result. Override to enrich with
        additional fields
        :param data:
        :return:
        """
261
        return self.apply_bulk_fields(data)
Dmitry Volodin's avatar
Dmitry Volodin committed
262

Dmitry Volodin's avatar
Dmitry Volodin committed
263
    @view(
264
        url=r"^favorites/app/(?P<action>set|reset)/$",
Dmitry Volodin's avatar
Dmitry Volodin committed
265
266
267
268
        method=["POST"],
        access=PermitLogged(),
        api=True,
    )
269
270
271
272
273
    def api_favorites_app(self, request, action):
        """
        Set/reset favorite app status
        """
        v = action == "set"
Dmitry Volodin's avatar
Dmitry Volodin committed
274
        fv = Favorites.objects.filter(user=request.user.id, app=self.app_id).first()
275
276
277
278
279
        if fv:
            if fv.favorite_app != v:
                fv.favorite_app = v
                fv.save()
        elif v:
Dmitry Volodin's avatar
Dmitry Volodin committed
280
            Favorites(user=request.user, app=self.app_id, favorite_app=v).save()
281
282
        return True

Dmitry Volodin's avatar
Dmitry Volodin committed
283
    @view(
284
        url=r"^favorites/item/(?P<item>[0-9a-f]+)/(?P<action>set|reset)/$",
Dmitry Volodin's avatar
Dmitry Volodin committed
285
286
287
288
        method=["POST"],
        access=PermitLogged(),
        api=True,
    )
289
290
291
292
293
    def api_favorites_items(self, request, item, action):
        """
        Set/reset favorite items
        """
        item = self.fav_convert(item)
Dmitry Volodin's avatar
Dmitry Volodin committed
294
295
296
297
        if action == "set":
            Favorites.add_item(request.user, self.app_id, item)
        else:
            Favorites.remove_item(request.user, self.app_id, item)
298
        return True
Dmitry Volodin's avatar
Dmitry Volodin committed
299

300
    @view(url=r"^futures/(?P<f_id>[0-9a-f]{24})/$", method=["GET"], access="launch", api=True)
301
    def api_future_status(self, request, f_id):
Dmitry Volodin's avatar
Dmitry Volodin committed
302
303
304
        op = self.get_object_or_404(
            SlowOp, id=f_id, app_id=self.get_app_id(), user=request.user.username
        )
305
306
307
308
        if op.is_ready():
            # Note: the slow operation will be purged by TTL index
            result = op.result()
            if isinstance(result, Exception):
Dmitry Volodin's avatar
Dmitry Volodin committed
309
310
311
312
                return self.render_json(
                    {"success": False, "message": "Error", "traceback": str(result)},
                    status=self.INTERNAL_ERROR,
                )
313
314
315
316
317
318
            else:
                return result
        else:
            return self.response_accepted(request.path)

    def submit_slow_op(self, request, fn, *args, **kwargs):
Dmitry Volodin's avatar
Dmitry Volodin committed
319
        f = SlowOp.submit(fn, self.get_app_id(), request.user.username, *args, **kwargs)
320
321
322
        if f.done():
            return f.result()
        else:
Dmitry Volodin's avatar
Dmitry Volodin committed
323
            return self.response_accepted(location="%sfutures/%s/" % (self.base_url, f.slow_op.id))