1
2
3
4 """
5 This file is part of the web2py Web Framework
6 Copyrighted by Massimo Di Pierro <mdipierro@cs.depaul.edu>
7 License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html)
8 """
9
10 import sys
11 import storage
12 import os
13 import re
14 import tarfile
15 import glob
16 import time
17 import datetime
18 import logging
19 from http import HTTP
20 from gzip import open as gzopen
21
22
23 __all__ = [
24 'parse_version',
25 'read_file',
26 'write_file',
27 'readlines_file',
28 'up',
29 'abspath',
30 'mktree',
31 'listdir',
32 'recursive_unlink',
33 'cleanpath',
34 'tar',
35 'untar',
36 'tar_compiled',
37 'get_session',
38 'check_credentials',
39 'w2p_pack',
40 'w2p_unpack',
41 'w2p_pack_plugin',
42 'w2p_unpack_plugin',
43 'fix_newlines',
44 'make_fake_file_like_object',
45 ]
46
47
48 -def parse_version(version="Version 1.99.0 (2011-09-19 08:23:26)"):
49 re_version = re.compile('[^\d]+ (\d+)\.(\d+)\.(\d+)\s*\((?P<datetime>.+?)\)\s*(?P<type>[a-z]+)?')
50 m = re_version.match(version)
51 a, b, c = int(m.group(1)), int(m.group(2)), int(m.group(3)),
52 s = m.group('type') or 'dev'
53 d = datetime.datetime.strptime(m.group('datetime'), '%Y-%m-%d %H:%M:%S')
54 return (a, b, c, d, s)
55
56
58 "returns content from filename, making sure to close the file explicitly on exit."
59 f = open(filename, mode)
60 try:
61 return f.read()
62 finally:
63 f.close()
64
65
67 "writes <value> to filename, making sure to close the file explicitly on exit."
68 f = open(filename, mode)
69 try:
70 return f.write(value)
71 finally:
72 f.close()
73
74
76 "applies .split('\n') to the output of read_file()"
77 return read_file(filename, mode).split('\n')
78
79
81 head, tail = os.path.split(path)
82 if head:
83 if tail:
84 mktree(head)
85 if not os.path.exists(head):
86 os.mkdir(head)
87
88
89 -def listdir(
90 path,
91 expression='^.+$',
92 drop=True,
93 add_dirs=False,
94 sort=True,
95 ):
96 """
97 like os.listdir() but you can specify a regex pattern to filter files.
98 if add_dirs is True, the returned items will have the full path.
99 """
100 if path[-1:] != os.path.sep:
101 path = path + os.path.sep
102 if drop:
103 n = len(path)
104 else:
105 n = 0
106 regex = re.compile(expression)
107 items = []
108 for (root, dirs, files) in os.walk(path, topdown=True):
109 for dir in dirs[:]:
110 if dir.startswith('.'):
111 dirs.remove(dir)
112 if add_dirs:
113 items.append(root[n:])
114 for file in sorted(files):
115 if regex.match(file) and not file.startswith('.'):
116 items.append(os.path.join(root, file)[n:])
117 if sort:
118 return sorted(items)
119 else:
120 return items
121
122
130
131
133 """
134 turns any expression/path into a valid filename. replaces / with _ and
135 removes special characters.
136 """
137
138 items = path.split('.')
139 if len(items) > 1:
140 path = re.sub('[^\w\.]+', '_', '_'.join(items[:-1]) + '.'
141 + ''.join(items[-1:]))
142 else:
143 path = re.sub('[^\w\.]+', '_', ''.join(items[-1:]))
144 return path
145
146
148 if not hasattr(tarfile.TarFile, 'extractall'):
149 from tarfile import ExtractError
150
151 class TarFile(tarfile.TarFile):
152
153 def extractall(self, path='.', members=None):
154 """Extract all members from the archive to the current working
155 directory and set owner, modification time and permissions on
156 directories afterwards. `path' specifies a different directory
157 to extract to. `members' is optional and must be a subset of the
158 list returned by getmembers().
159 """
160
161 directories = []
162 if members is None:
163 members = self
164 for tarinfo in members:
165 if tarinfo.isdir():
166
167
168
169
170 try:
171 os.makedirs(os.path.join(path,
172 tarinfo.name), 0777)
173 except EnvironmentError:
174 pass
175 directories.append(tarinfo)
176 else:
177 self.extract(tarinfo, path)
178
179
180
181 directories.sort(lambda a, b: cmp(a.name, b.name))
182 directories.reverse()
183
184
185
186 for tarinfo in directories:
187 path = os.path.join(path, tarinfo.name)
188 try:
189 self.chown(tarinfo, path)
190 self.utime(tarinfo, path)
191 self.chmod(tarinfo, path)
192 except ExtractError, e:
193 if self.errorlevel > 1:
194 raise
195 else:
196 self._dbg(1, 'tarfile: %s' % e)
197
198 _cls = TarFile
199 else:
200 _cls = tarfile.TarFile
201
202 tar = _cls(filename, 'r')
203 ret = tar.extractall(path, members)
204 tar.close()
205 return ret
206
207
208 -def tar(file, dir, expression='^.+$'):
209 """
210 tars dir into file, only tars file that match expression
211 """
212
213 tar = tarfile.TarFile(file, 'w')
214 try:
215 for file in listdir(dir, expression, add_dirs=True):
216 tar.add(os.path.join(dir, file), file, False)
217 finally:
218 tar.close()
219
220
222 """
223 untar file into dir
224 """
225
226 _extractall(file, dir)
227
228
229 -def w2p_pack(filename, path, compiled=False):
230 filename = abspath(filename)
231 path = abspath(path)
232 tarname = filename + '.tar'
233 if compiled:
234 tar_compiled(tarname, path, '^[\w\.\-]+$')
235 else:
236 tar(tarname, path, '^[\w\.\-]+$')
237 w2pfp = gzopen(filename, 'wb')
238 tarfp = open(tarname, 'rb')
239 w2pfp.write(tarfp.read())
240 w2pfp.close()
241 tarfp.close()
242 os.unlink(tarname)
243
245 if not os.path.exists('welcome.w2p') or os.path.exists('NEWINSTALL'):
246 try:
247 w2p_pack('welcome.w2p', 'applications/welcome')
248 os.unlink('NEWINSTALL')
249 logging.info("New installation: created welcome.w2p file")
250 except:
251 logging.error("New installation error: unable to create welcome.w2p file")
252
253
255
256 if filename=='welcome.w2p':
257 create_welcome_w2p()
258 filename = abspath(filename)
259 path = abspath(path)
260 if filename[-4:] == '.w2p' or filename[-3:] == '.gz':
261 if filename[-4:] == '.w2p':
262 tarname = filename[:-4] + '.tar'
263 else:
264 tarname = filename[:-3] + '.tar'
265 fgzipped = gzopen(filename, 'rb')
266 tarfile = open(tarname, 'wb')
267 tarfile.write(fgzipped.read())
268 tarfile.close()
269 fgzipped.close()
270 else:
271 tarname = filename
272 untar(tarname, path)
273 if delete_tar:
274 os.unlink(tarname)
275
276
278 """Pack the given plugin into a w2p file.
279 Will match files at:
280 <path>/*/plugin_[name].*
281 <path>/*/plugin_[name]/*
282 """
283 filename = abspath(filename)
284 path = abspath(path)
285 if not filename.endswith('web2py.plugin.%s.w2p' % plugin_name):
286 raise Exception("Not a web2py plugin name")
287 plugin_tarball = tarfile.open(filename, 'w:gz')
288 try:
289 app_dir = path
290 while app_dir[-1] == '/':
291 app_dir = app_dir[:-1]
292 files1 = glob.glob(
293 os.path.join(app_dir, '*/plugin_%s.*' % plugin_name))
294 files2 = glob.glob(
295 os.path.join(app_dir, '*/plugin_%s/*' % plugin_name))
296 for file in files1 + files2:
297 plugin_tarball.add(file, arcname=file[len(app_dir) + 1:])
298 finally:
299 plugin_tarball.close()
300
301
308
309
311 """
312 used to tar a compiled application.
313 the content of models, views, controllers is not stored in the tar file.
314 """
315
316 tar = tarfile.TarFile(file, 'w')
317 for file in listdir(dir, expression, add_dirs=True):
318 filename = os.path.join(dir, file)
319 if os.path.islink(filename):
320 continue
321 if os.path.isfile(filename) and file[-4:] != '.pyc':
322 if file[:6] == 'models':
323 continue
324 if file[:5] == 'views':
325 continue
326 if file[:11] == 'controllers':
327 continue
328 if file[:7] == 'modules':
329 continue
330 tar.add(filename, file, False)
331 tar.close()
332
333
335 return os.path.dirname(os.path.normpath(path))
336
337
339 """ checks that user is authorized to access other_application"""
340 if request.application == other_application:
341 raise KeyError
342 try:
343 session_id = request.cookies['session_id_' + other_application].value
344 osession = storage.load_storage(os.path.join(
345 up(request.folder), other_application, 'sessions', session_id))
346 except Exception, e:
347 osession = storage.Storage()
348 return osession
349
350
352 """ checks that user is authorized to access other_application"""
353 if request.env.web2py_runtime_gae:
354 from google.appengine.api import users
355 if users.is_current_user_admin():
356 return True
357 else:
358 login_html = '<a href="%s">Sign in with your google account</a>.' \
359 % users.create_login_url(request.env.path_info)
360 raise HTTP(200, '<html><body>%s</body></html>' % login_html)
361 else:
362 dt = time.time() - expiration
363 s = get_session(request, other_application)
364 return (s.authorized and s.last_time and s.last_time > dt)
365
366
368 regex = re.compile(r'''(\r
369 |\r|
370 )''')
371 for filename in listdir(path, '.*\.(py|html)$', drop=False):
372 rdata = read_file(filename, 'rb')
373 wdata = regex.sub('\n', rdata)
374 if wdata != rdata:
375 write_file(filename, wdata, 'wb')
376
377
378 -def copystream(
379 src,
380 dest,
381 size,
382 chunk_size=10 ** 5,
383 ):
384 """
385 this is here because I think there is a bug in shutil.copyfileobj
386 """
387 while size > 0:
388 if size < chunk_size:
389 data = src.read(size)
390 else:
391 data = src.read(chunk_size)
392 length = len(data)
393 if length > size:
394 (data, length) = (data[:size], size)
395 size -= length
396 if length == 0:
397 break
398 dest.write(data)
399 if length < chunk_size:
400 break
401 dest.seek(0)
402 return
403
404
406 class LogFile(object):
407 def write(self, value):
408 pass
409
410 def close(self):
411 pass
412 return LogFile()
413
414
415 from settings import global_settings
416
417
418
420 "convert relative path to absolute path based (by default) on applications_parent"
421 path = os.path.join(*relpath)
422 gluon = base.get('gluon', False)
423 if os.path.isabs(path):
424 return path
425 if gluon:
426 return os.path.join(global_settings.gluon_parent, path)
427 return os.path.join(global_settings.applications_parent, path)
428