Package web2py :: Package gluon :: Module fileutils
[hide private]
[frames] | no frames]

Source Code for Module web2py.gluon.fileutils

  1  #!/usr/bin/env python 
  2  # -*- coding: utf-8 -*- 
  3   
  4  """ 
  5  This file is part of the web2py Web Framework 
  6  Copyrighted by Massimo Di Pierro <mdipierro@cs.depaul.edu> 
  7  License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html) 
  8  """ 
  9   
 10  import sys 
 11  import storage 
 12  import os 
 13  import re 
 14  import tarfile 
 15  import glob 
 16  import time 
 17  import datetime 
 18  import logging 
 19  from http import HTTP 
 20  from gzip import open as gzopen 
 21   
 22   
 23  __all__ = [ 
 24      'parse_version', 
 25      'read_file', 
 26      'write_file', 
 27      'readlines_file', 
 28      'up', 
 29      'abspath', 
 30      'mktree', 
 31      'listdir', 
 32      'recursive_unlink', 
 33      'cleanpath', 
 34      'tar', 
 35      'untar', 
 36      'tar_compiled', 
 37      'get_session', 
 38      'check_credentials', 
 39      'w2p_pack', 
 40      'w2p_unpack', 
 41      'w2p_pack_plugin', 
 42      'w2p_unpack_plugin', 
 43      'fix_newlines', 
 44      'make_fake_file_like_object', 
 45  ] 
 46   
 47   
48 -def parse_version(version="Version 1.99.0 (2011-09-19 08:23:26)"):
49 re_version = re.compile('[^\d]+ (\d+)\.(\d+)\.(\d+)\s*\((?P<datetime>.+?)\)\s*(?P<type>[a-z]+)?') 50 m = re_version.match(version) 51 a, b, c = int(m.group(1)), int(m.group(2)), int(m.group(3)), 52 s = m.group('type') or 'dev' 53 d = datetime.datetime.strptime(m.group('datetime'), '%Y-%m-%d %H:%M:%S') 54 return (a, b, c, d, s)
55 56
57 -def read_file(filename, mode='r'):
58 "returns content from filename, making sure to close the file explicitly on exit." 59 f = open(filename, mode) 60 try: 61 return f.read() 62 finally: 63 f.close()
64 65
66 -def write_file(filename, value, mode='w'):
67 "writes <value> to filename, making sure to close the file explicitly on exit." 68 f = open(filename, mode) 69 try: 70 return f.write(value) 71 finally: 72 f.close()
73 74
75 -def readlines_file(filename, mode='r'):
76 "applies .split('\n') to the output of read_file()" 77 return read_file(filename, mode).split('\n')
78 79
80 -def mktree(path):
81 head, tail = os.path.split(path) 82 if head: 83 if tail: 84 mktree(head) 85 if not os.path.exists(head): 86 os.mkdir(head)
87 88
89 -def listdir( 90 path, 91 expression='^.+$', 92 drop=True, 93 add_dirs=False, 94 sort=True, 95 ):
96 """ 97 like os.listdir() but you can specify a regex pattern to filter files. 98 if add_dirs is True, the returned items will have the full path. 99 """ 100 if path[-1:] != os.path.sep: 101 path = path + os.path.sep 102 if drop: 103 n = len(path) 104 else: 105 n = 0 106 regex = re.compile(expression) 107 items = [] 108 for (root, dirs, files) in os.walk(path, topdown=True): 109 for dir in dirs[:]: 110 if dir.startswith('.'): 111 dirs.remove(dir) 112 if add_dirs: 113 items.append(root[n:]) 114 for file in sorted(files): 115 if regex.match(file) and not file.startswith('.'): 116 items.append(os.path.join(root, file)[n:]) 117 if sort: 118 return sorted(items) 119 else: 120 return items
121 122 130 131
132 -def cleanpath(path):
133 """ 134 turns any expression/path into a valid filename. replaces / with _ and 135 removes special characters. 136 """ 137 138 items = path.split('.') 139 if len(items) > 1: 140 path = re.sub('[^\w\.]+', '_', '_'.join(items[:-1]) + '.' 141 + ''.join(items[-1:])) 142 else: 143 path = re.sub('[^\w\.]+', '_', ''.join(items[-1:])) 144 return path
145 146
147 -def _extractall(filename, path='.', members=None):
148 if not hasattr(tarfile.TarFile, 'extractall'): 149 from tarfile import ExtractError 150 151 class TarFile(tarfile.TarFile): 152 153 def extractall(self, path='.', members=None): 154 """Extract all members from the archive to the current working 155 directory and set owner, modification time and permissions on 156 directories afterwards. `path' specifies a different directory 157 to extract to. `members' is optional and must be a subset of the 158 list returned by getmembers(). 159 """ 160 161 directories = [] 162 if members is None: 163 members = self 164 for tarinfo in members: 165 if tarinfo.isdir(): 166 167 # Extract directory with a safe mode, so that 168 # all files below can be extracted as well. 169 170 try: 171 os.makedirs(os.path.join(path, 172 tarinfo.name), 0777) 173 except EnvironmentError: 174 pass 175 directories.append(tarinfo) 176 else: 177 self.extract(tarinfo, path) 178 179 # Reverse sort directories. 180 181 directories.sort(lambda a, b: cmp(a.name, b.name)) 182 directories.reverse() 183 184 # Set correct owner, mtime and filemode on directories. 185 186 for tarinfo in directories: 187 path = os.path.join(path, tarinfo.name) 188 try: 189 self.chown(tarinfo, path) 190 self.utime(tarinfo, path) 191 self.chmod(tarinfo, path) 192 except ExtractError, e: 193 if self.errorlevel > 1: 194 raise 195 else: 196 self._dbg(1, 'tarfile: %s' % e)
197 198 _cls = TarFile 199 else: 200 _cls = tarfile.TarFile 201 202 tar = _cls(filename, 'r') 203 ret = tar.extractall(path, members) 204 tar.close() 205 return ret 206 207
208 -def tar(file, dir, expression='^.+$'):
209 """ 210 tars dir into file, only tars file that match expression 211 """ 212 213 tar = tarfile.TarFile(file, 'w') 214 try: 215 for file in listdir(dir, expression, add_dirs=True): 216 tar.add(os.path.join(dir, file), file, False) 217 finally: 218 tar.close()
219 220
221 -def untar(file, dir):
222 """ 223 untar file into dir 224 """ 225 226 _extractall(file, dir)
227 228
229 -def w2p_pack(filename, path, compiled=False):
230 filename = abspath(filename) 231 path = abspath(path) 232 tarname = filename + '.tar' 233 if compiled: 234 tar_compiled(tarname, path, '^[\w\.\-]+$') 235 else: 236 tar(tarname, path, '^[\w\.\-]+$') 237 w2pfp = gzopen(filename, 'wb') 238 tarfp = open(tarname, 'rb') 239 w2pfp.write(tarfp.read()) 240 w2pfp.close() 241 tarfp.close() 242 os.unlink(tarname)
243
244 -def create_welcome_w2p():
245 if not os.path.exists('welcome.w2p') or os.path.exists('NEWINSTALL'): 246 try: 247 w2p_pack('welcome.w2p', 'applications/welcome') 248 os.unlink('NEWINSTALL') 249 logging.info("New installation: created welcome.w2p file") 250 except: 251 logging.error("New installation error: unable to create welcome.w2p file")
252 253
254 -def w2p_unpack(filename, path, delete_tar=True):
255 256 if filename=='welcome.w2p': 257 create_welcome_w2p() 258 filename = abspath(filename) 259 path = abspath(path) 260 if filename[-4:] == '.w2p' or filename[-3:] == '.gz': 261 if filename[-4:] == '.w2p': 262 tarname = filename[:-4] + '.tar' 263 else: 264 tarname = filename[:-3] + '.tar' 265 fgzipped = gzopen(filename, 'rb') 266 tarfile = open(tarname, 'wb') 267 tarfile.write(fgzipped.read()) 268 tarfile.close() 269 fgzipped.close() 270 else: 271 tarname = filename 272 untar(tarname, path) 273 if delete_tar: 274 os.unlink(tarname)
275 276
277 -def w2p_pack_plugin(filename, path, plugin_name):
278 """Pack the given plugin into a w2p file. 279 Will match files at: 280 <path>/*/plugin_[name].* 281 <path>/*/plugin_[name]/* 282 """ 283 filename = abspath(filename) 284 path = abspath(path) 285 if not filename.endswith('web2py.plugin.%s.w2p' % plugin_name): 286 raise Exception("Not a web2py plugin name") 287 plugin_tarball = tarfile.open(filename, 'w:gz') 288 try: 289 app_dir = path 290 while app_dir[-1] == '/': 291 app_dir = app_dir[:-1] 292 files1 = glob.glob( 293 os.path.join(app_dir, '*/plugin_%s.*' % plugin_name)) 294 files2 = glob.glob( 295 os.path.join(app_dir, '*/plugin_%s/*' % plugin_name)) 296 for file in files1 + files2: 297 plugin_tarball.add(file, arcname=file[len(app_dir) + 1:]) 298 finally: 299 plugin_tarball.close()
300 301
302 -def w2p_unpack_plugin(filename, path, delete_tar=True):
303 filename = abspath(filename) 304 path = abspath(path) 305 if not os.path.basename(filename).startswith('web2py.plugin.'): 306 raise Exception("Not a web2py plugin") 307 w2p_unpack(filename, path, delete_tar)
308 309
310 -def tar_compiled(file, dir, expression='^.+$'):
311 """ 312 used to tar a compiled application. 313 the content of models, views, controllers is not stored in the tar file. 314 """ 315 316 tar = tarfile.TarFile(file, 'w') 317 for file in listdir(dir, expression, add_dirs=True): 318 filename = os.path.join(dir, file) 319 if os.path.islink(filename): 320 continue 321 if os.path.isfile(filename) and file[-4:] != '.pyc': 322 if file[:6] == 'models': 323 continue 324 if file[:5] == 'views': 325 continue 326 if file[:11] == 'controllers': 327 continue 328 if file[:7] == 'modules': 329 continue 330 tar.add(filename, file, False) 331 tar.close()
332 333
334 -def up(path):
335 return os.path.dirname(os.path.normpath(path))
336 337
338 -def get_session(request, other_application='admin'):
339 """ checks that user is authorized to access other_application""" 340 if request.application == other_application: 341 raise KeyError 342 try: 343 session_id = request.cookies['session_id_' + other_application].value 344 osession = storage.load_storage(os.path.join( 345 up(request.folder), other_application, 'sessions', session_id)) 346 except Exception, e: 347 osession = storage.Storage() 348 return osession
349 350
351 -def check_credentials(request, other_application='admin', expiration=60 * 60):
352 """ checks that user is authorized to access other_application""" 353 if request.env.web2py_runtime_gae: 354 from google.appengine.api import users 355 if users.is_current_user_admin(): 356 return True 357 else: 358 login_html = '<a href="%s">Sign in with your google account</a>.' \ 359 % users.create_login_url(request.env.path_info) 360 raise HTTP(200, '<html><body>%s</body></html>' % login_html) 361 else: 362 dt = time.time() - expiration 363 s = get_session(request, other_application) 364 return (s.authorized and s.last_time and s.last_time > dt)
365 366
367 -def fix_newlines(path):
368 regex = re.compile(r'''(\r 369 |\r| 370 )''') 371 for filename in listdir(path, '.*\.(py|html)$', drop=False): 372 rdata = read_file(filename, 'rb') 373 wdata = regex.sub('\n', rdata) 374 if wdata != rdata: 375 write_file(filename, wdata, 'wb')
376 377
378 -def copystream( 379 src, 380 dest, 381 size, 382 chunk_size=10 ** 5, 383 ):
384 """ 385 this is here because I think there is a bug in shutil.copyfileobj 386 """ 387 while size > 0: 388 if size < chunk_size: 389 data = src.read(size) 390 else: 391 data = src.read(chunk_size) 392 length = len(data) 393 if length > size: 394 (data, length) = (data[:size], size) 395 size -= length 396 if length == 0: 397 break 398 dest.write(data) 399 if length < chunk_size: 400 break 401 dest.seek(0) 402 return
403 404
405 -def make_fake_file_like_object():
406 class LogFile(object): 407 def write(self, value): 408 pass
409 410 def close(self): 411 pass 412 return LogFile() 413 414 415 from settings import global_settings # we need to import settings here because 416 # settings imports fileutils too 417 418
419 -def abspath(*relpath, **base):
420 "convert relative path to absolute path based (by default) on applications_parent" 421 path = os.path.join(*relpath) 422 gluon = base.get('gluon', False) 423 if os.path.isabs(path): 424 return path 425 if gluon: 426 return os.path.join(global_settings.gluon_parent, path) 427 return os.path.join(global_settings.applications_parent, path)
428