fileutils.py 5 KB
Newer Older
Dmitry Volodin's avatar
Dmitry Volodin committed
1
# -*- coding: utf-8 -*-
Dmitry Lukhtionov's avatar
Dmitry Lukhtionov committed
2
# ---------------------------------------------------------------------
Dmitry Volodin's avatar
Dmitry Volodin committed
3
# Copyright (C) 2007-2019 The NOC Project
Dmitry Lukhtionov's avatar
Dmitry Lukhtionov committed
4
5
# See LICENSE for details
# ---------------------------------------------------------------------
Dmitry Volodin's avatar
Dmitry Volodin committed
6

Dmitry Lukhtionov's avatar
Dmitry Lukhtionov committed
7
# Python modules
Dmitry Volodin's avatar
Dmitry Volodin committed
8
9
10
import os
import tempfile
import hashlib
11
import tarfile
Dmitry Volodin's avatar
Dmitry Volodin committed
12
import gzip
Dmitry Volodin's avatar
Dmitry Volodin committed
13

Dmitry Volodin's avatar
Dmitry Volodin committed
14
15
# Third-party modules
import six
Dmitry Volodin's avatar
Dmitry Volodin committed
16

Dmitry Lukhtionov's avatar
Dmitry Lukhtionov committed
17
# NOC modules
Dmitry Volodin's avatar
Dmitry Volodin committed
18
from noc.core.version import version
Dmitry Volodin's avatar
Dmitry Volodin committed
19
20
21
22
23
24


def safe_rewrite(path, text, mode=None):
    """
    Create new file filled with "text" safely
    """
Dmitry Lukhtionov's avatar
Fix    
Dmitry Lukhtionov committed
25
26
    if isinstance(text, unicode):
        text = text.encode("utf-8")
Dmitry Volodin's avatar
Dmitry Volodin committed
27
    d = os.path.dirname(path)
28
    if d and not os.path.exists(d):
29
        os.makedirs(d)
Dmitry Volodin's avatar
Dmitry Volodin committed
30
31
32
    b = os.path.basename(path)
    h, p = tempfile.mkstemp(suffix=".tmp", prefix=b, dir=d)
    f = os.fdopen(h, "w")
Dmitry Volodin's avatar
Dmitry Volodin committed
33
    f.write(text)
Dmitry Volodin's avatar
Dmitry Volodin committed
34
    f.flush()
Dmitry Volodin's avatar
Dmitry Volodin committed
35
36
37
    f.close()
    if os.path.exists(path):
        os.unlink(path)
Dmitry Volodin's avatar
Dmitry Volodin committed
38
    os.link(p, path)
39
    os.unlink(p)
40
    if mode:
Dmitry Volodin's avatar
Dmitry Volodin committed
41
42
43
        os.chmod(path, mode)


Dmitry Volodin's avatar
Dmitry Volodin committed
44
45
46
47
48
49
50
51
52
53
54
55
56
def safe_append(path, text):
    """
    Append the text to the end of the file
    """
    if isinstance(text, unicode):
        text = text.encode("utf-8")
    d = os.path.dirname(path)
    if d and not os.path.exists(d):
        os.makedirs(d)
    with open(path, "a") as f:
        f.write(text)


Dmitry Volodin's avatar
Dmitry Volodin committed
57
58
59
60
def is_differ(path, content):
    """
    Check file content is differ from string
    """
61
    if os.path.isfile(path):
Dmitry Volodin's avatar
Dmitry Volodin committed
62
63
64
65
        with open(path) as f:
            cs1 = hashlib.sha1(f.read()).digest()
        cs2 = hashlib.sha1(content).digest()
        return cs1 != cs2
66
67
    else:
        return True
Dmitry Volodin's avatar
Dmitry Volodin committed
68
69
70
71
72
73
74
75


def rewrite_when_differ(path, content):
    """
    Rewrites file when content is differ
    Returns boolean signalling wherher file was rewritten
    """
    d = is_differ(path, content)
76
    if d:
Dmitry Volodin's avatar
Dmitry Volodin committed
77
        safe_rewrite(path, content)
Dmitry Volodin's avatar
Dmitry Volodin committed
78
    return d
Dmitry Volodin's avatar
Dmitry Volodin committed
79
80


Dmitry Volodin's avatar
Dmitry Volodin committed
81
def read_file(path):
Dmitry Volodin's avatar
Dmitry Volodin committed
82
83
84
85
86
87
    """
    Read file and return file's content.
    Return None when file does not exists
    """
    if os.path.isfile(path) and os.access(path, os.R_OK):
        with open(path, "r") as f:
88
            return f.read()
Dmitry Volodin's avatar
Dmitry Volodin committed
89
90
    else:
        return None
Dmitry Volodin's avatar
Dmitry Volodin committed
91
92


93
def copy_file(f, t, mode=None):
Dmitry Volodin's avatar
Dmitry Volodin committed
94
95
96
97
    """
    Copy File
    """
    d = read_file(f)
Dmitry Volodin's avatar
Dmitry Volodin committed
98
    if d is None:
Dmitry Volodin's avatar
Dmitry Volodin committed
99
        d = ""
100
    safe_rewrite(t, d, mode=mode)
Dmitry Volodin's avatar
Dmitry Volodin committed
101
102


103
def write_tempfile(text):
Dmitry Volodin's avatar
Dmitry Volodin committed
104
105
106
107
108
    """
    Create temporary file, write content and return path
    """
    h, p = tempfile.mkstemp()
    f = os.fdopen(h, "w")
109
110
111
    f.write(text)
    f.close()
    return p
Dmitry Volodin's avatar
Dmitry Volodin committed
112
113


114
class temporary_file(object):
Dmitry Volodin's avatar
Dmitry Volodin committed
115
116
117
118
119
120
121
122
    """
    Temporary file context manager.
    Writes data to temporary file an returns path.
    Unlinks temporary file on exit
    USAGE:
         with temporary_file("line1\nline2") as p:
             subprocess.Popen(["wc","-l",p])
    """
Dmitry Volodin's avatar
Dmitry Volodin committed
123

Dmitry Volodin's avatar
Dmitry Volodin committed
124
125
126
    def __init__(self, text=""):
        self.text = text

127
    def __enter__(self):
Dmitry Volodin's avatar
Dmitry Volodin committed
128
        self.p = write_tempfile(self.text)
129
        return self.p
Dmitry Volodin's avatar
Dmitry Volodin committed
130

131
132
    def __exit__(self, type, value, tb):
        os.unlink(self.p)
Dmitry Volodin's avatar
Dmitry Volodin committed
133
134
135
136
137
138
139
140
141
142
143
144
145


def in_dir(file, dir):
    """
    Check file is inside dir
    """
    return os.path.commonprefix([dir, os.path.normpath(file)]) == dir


def urlopen(url, auto_deflate=False):
    """
    urlopen wrapper
    """
Dmitry Volodin's avatar
Dmitry Volodin committed
146
    from future.moves.urllib.request import urlopen, Request
147
148
149
    from noc.core.http.proxy import setup_urllib_proxies

    setup_urllib_proxies()
150

Dmitry Volodin's avatar
urlopen    
Dmitry Volodin committed
151
    if url.startswith("http://") or url.startswith("https://"):
152
        r = Request(url, headers={"User-Agent": "NOC/%s" % version.version.strip()})
Dmitry Volodin's avatar
urlopen    
Dmitry Volodin committed
153
    else:
Dmitry Volodin's avatar
Dmitry Volodin committed
154
        r = url
155
    if auto_deflate and url.endswith(".gz"):
Dmitry Volodin's avatar
Dmitry Volodin committed
156
        u = urlopen(r)
Dmitry Volodin's avatar
Dmitry Volodin committed
157
        f = six.StringIO(u.read())
158
        return gzip.GzipFile(fileobj=f)
Dmitry Volodin's avatar
Dmitry Volodin committed
159
    return urlopen(r)
Dmitry Volodin's avatar
Dmitry Volodin committed
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175


def search_path(file):
    """
    Search for executable file in $PATH
    :param file: File name
    :return: path or None
    :rtype: str or None
    """
    if os.path.exists(file):
        return file  # Found
    for d in os.environ["PATH"].split(os.pathsep):
        f = os.path.join(d, file)
        if os.path.exists(f):
            return f
    return None
Dmitry Volodin's avatar
Dmitry Volodin committed
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191


def tail(path, lines):
    """
    Return string containing last lines of file
    :param lines:
    :return:
    """
    with open(path) as f:
        avg = 74
        while True:
            try:
                f.seek(-avg * lines, 2)
            except IOError:
                f.seek(0)
            pos = f.tell()
Dmitry Volodin's avatar
Dmitry Volodin committed
192
193
194
            ln = f.read().splitlines()
            if len(ln) >= lines or not pos:
                return ln[-lines:]
Dmitry Volodin's avatar
Dmitry Volodin committed
195
            avg *= 1.61
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222


def iter_open(path):
    """
    Generator yielding file-like objects from path
    :param path:
    :return:
    """
    if path.endswith("tar.gz") or path.endswith("tgz"):
        tf = tarfile.open(path, "r:gz")
        for name in tf:
            f = tf.extractfile(name)
            yield f
        tf.close()
    elif path.endswith("tar.bz2") or path.endswith("tbz"):
        tf = tarfile.open(path, "r:bz")
        for f in tf:
            yield f
        tf.close()
    elif path.endswith(".gz"):
        f = gzip.open(path, "r")
        yield f
        f.close()
    else:
        f = open(path, "r")
        yield f
        f.close()