Package logilab :: Package common :: Module modutils
[frames] | no frames]

Source Code for Module logilab.common.modutils

  1  # -*- coding: utf-8 -*- 
  2  # copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved. 
  3  # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr 
  4  # 
  5  # This file is part of logilab-common. 
  6  # 
  7  # logilab-common is free software: you can redistribute it and/or modify it under 
  8  # the terms of the GNU Lesser General Public License as published by the Free 
  9  # Software Foundation, either version 2.1 of the License, or (at your option) any 
 10  # later version. 
 11  # 
 12  # logilab-common is distributed in the hope that it will be useful, but WITHOUT 
 13  # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
 14  # FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more 
 15  # details. 
 16  # 
 17  # You should have received a copy of the GNU Lesser General Public License along 
 18  # with logilab-common.  If not, see <http://www.gnu.org/licenses/>. 
 19  """Python modules manipulation utility functions. 
 20   
 21  :type PY_SOURCE_EXTS: tuple(str) 
 22  :var PY_SOURCE_EXTS: list of possible python source file extension 
 23   
 24  :type STD_LIB_DIR: str 
 25  :var STD_LIB_DIR: directory where standard modules are located 
 26   
 27  :type BUILTIN_MODULES: dict 
 28  :var BUILTIN_MODULES: dictionary with builtin module names as key 
 29  """ 
 30   
 31  __docformat__ = "restructuredtext en" 
 32   
 33  import sys 
 34  import os 
 35  from os.path import splitext, join, abspath, isdir, dirname, exists, basename 
 36  from imp import find_module, load_module, C_BUILTIN, PY_COMPILED, PKG_DIRECTORY 
 37  from distutils.sysconfig import get_config_var, get_python_lib, get_python_version 
 38  from distutils.errors import DistutilsPlatformError 
 39   
 40  from six.moves import range 
 41   
 42  try: 
 43      import zipimport 
 44  except ImportError: 
 45      zipimport = None 
 46   
 47  ZIPFILE = object() 
 48   
 49  from logilab.common import STD_BLACKLIST, _handle_blacklist 
 50   
 51  # Notes about STD_LIB_DIR 
 52  # Consider arch-specific installation for STD_LIB_DIR definition 
 53  # :mod:`distutils.sysconfig` contains to much hardcoded values to rely on 
 54  # 
 55  # :see: `Problems with /usr/lib64 builds <http://bugs.python.org/issue1294959>`_ 
 56  # :see: `FHS <http://www.pathname.com/fhs/pub/fhs-2.3.html#LIBLTQUALGTALTERNATEFORMATESSENTIAL>`_ 
 57  if sys.platform.startswith('win'): 
 58      PY_SOURCE_EXTS = ('py', 'pyw') 
 59      PY_COMPILED_EXTS = ('dll', 'pyd') 
 60  else: 
 61      PY_SOURCE_EXTS = ('py',) 
 62      PY_COMPILED_EXTS = ('so',) 
 63   
 64  try: 
 65      STD_LIB_DIR = get_python_lib(standard_lib=True) 
 66  # get_python_lib(standard_lib=1) is not available on pypy, set STD_LIB_DIR to 
 67  # non-valid path, see https://bugs.pypy.org/issue1164 
 68  except DistutilsPlatformError: 
 69      STD_LIB_DIR = '//' 
 70   
 71  EXT_LIB_DIR = get_python_lib() 
 72   
 73  BUILTIN_MODULES = dict.fromkeys(sys.builtin_module_names, True) 
 74   
 75   
76 -class NoSourceFile(Exception):
77 """exception raised when we are not able to get a python 78 source file for a precompiled file 79 """
80
81 -class LazyObject(object):
82 - def __init__(self, module, obj):
83 self.module = module 84 self.obj = obj 85 self._imported = None
86
87 - def _getobj(self):
88 if self._imported is None: 89 self._imported = getattr(load_module_from_name(self.module), 90 self.obj) 91 return self._imported
92
93 - def __getattribute__(self, attr):
94 try: 95 return super(LazyObject, self).__getattribute__(attr) 96 except AttributeError as ex: 97 return getattr(self._getobj(), attr)
98
99 - def __call__(self, *args, **kwargs):
100 return self._getobj()(*args, **kwargs)
101 102
103 -def load_module_from_name(dotted_name, path=None, use_sys=True):
104 """Load a Python module from its name. 105 106 :type dotted_name: str 107 :param dotted_name: python name of a module or package 108 109 :type path: list or None 110 :param path: 111 optional list of path where the module or package should be 112 searched (use sys.path if nothing or None is given) 113 114 :type use_sys: bool 115 :param use_sys: 116 boolean indicating whether the sys.modules dictionary should be 117 used or not 118 119 120 :raise ImportError: if the module or package is not found 121 122 :rtype: module 123 :return: the loaded module 124 """ 125 return load_module_from_modpath(dotted_name.split('.'), path, use_sys)
126 127
128 -def load_module_from_modpath(parts, path=None, use_sys=True):
129 """Load a python module from its splitted name. 130 131 :type parts: list(str) or tuple(str) 132 :param parts: 133 python name of a module or package splitted on '.' 134 135 :type path: list or None 136 :param path: 137 optional list of path where the module or package should be 138 searched (use sys.path if nothing or None is given) 139 140 :type use_sys: bool 141 :param use_sys: 142 boolean indicating whether the sys.modules dictionary should be used or not 143 144 :raise ImportError: if the module or package is not found 145 146 :rtype: module 147 :return: the loaded module 148 """ 149 if use_sys: 150 try: 151 return sys.modules['.'.join(parts)] 152 except KeyError: 153 pass 154 modpath = [] 155 prevmodule = None 156 for part in parts: 157 modpath.append(part) 158 curname = '.'.join(modpath) 159 module = None 160 if len(modpath) != len(parts): 161 # even with use_sys=False, should try to get outer packages from sys.modules 162 module = sys.modules.get(curname) 163 elif use_sys: 164 # because it may have been indirectly loaded through a parent 165 module = sys.modules.get(curname) 166 if module is None: 167 mp_file, mp_filename, mp_desc = find_module(part, path) 168 module = load_module(curname, mp_file, mp_filename, mp_desc) 169 if prevmodule: 170 setattr(prevmodule, part, module) 171 _file = getattr(module, '__file__', '') 172 prevmodule = module 173 if not _file and _is_namespace(curname): 174 continue 175 if not _file and len(modpath) != len(parts): 176 raise ImportError('no module in %s' % '.'.join(parts[len(modpath):]) ) 177 path = [dirname( _file )] 178 return module
179 180
181 -def load_module_from_file(filepath, path=None, use_sys=True, extrapath=None):
182 """Load a Python module from it's path. 183 184 :type filepath: str 185 :param filepath: path to the python module or package 186 187 :type path: list or None 188 :param path: 189 optional list of path where the module or package should be 190 searched (use sys.path if nothing or None is given) 191 192 :type use_sys: bool 193 :param use_sys: 194 boolean indicating whether the sys.modules dictionary should be 195 used or not 196 197 198 :raise ImportError: if the module or package is not found 199 200 :rtype: module 201 :return: the loaded module 202 """ 203 modpath = modpath_from_file(filepath, extrapath) 204 return load_module_from_modpath(modpath, path, use_sys)
205 206
207 -def _check_init(path, mod_path):
208 """check there are some __init__.py all along the way""" 209 modpath = [] 210 for part in mod_path: 211 modpath.append(part) 212 path = join(path, part) 213 if not _is_namespace('.'.join(modpath)) and not _has_init(path): 214 return False 215 return True
216 217
218 -def modpath_from_file(filename, extrapath=None):
219 """given a file path return the corresponding splitted module's name 220 (i.e name of a module or package splitted on '.') 221 222 :type filename: str 223 :param filename: file's path for which we want the module's name 224 225 :type extrapath: dict 226 :param extrapath: 227 optional extra search path, with path as key and package name for the path 228 as value. This is usually useful to handle package splitted in multiple 229 directories using __path__ trick. 230 231 232 :raise ImportError: 233 if the corresponding module's name has not been found 234 235 :rtype: list(str) 236 :return: the corresponding splitted module's name 237 """ 238 base = splitext(abspath(filename))[0] 239 if extrapath is not None: 240 for path_ in extrapath: 241 path = abspath(path_) 242 if path and base[:len(path)] == path: 243 submodpath = [pkg for pkg in base[len(path):].split(os.sep) 244 if pkg] 245 if _check_init(path, submodpath[:-1]): 246 return extrapath[path_].split('.') + submodpath 247 for path in sys.path: 248 path = abspath(path) 249 if path and base.startswith(path): 250 modpath = [pkg for pkg in base[len(path):].split(os.sep) if pkg] 251 if _check_init(path, modpath[:-1]): 252 return modpath 253 raise ImportError('Unable to find module for %s in %s' % ( 254 filename, ', \n'.join(sys.path)))
255 256 257
258 -def file_from_modpath(modpath, path=None, context_file=None):
259 """given a mod path (i.e. splitted module / package name), return the 260 corresponding file, giving priority to source file over precompiled 261 file if it exists 262 263 :type modpath: list or tuple 264 :param modpath: 265 splitted module's name (i.e name of a module or package splitted 266 on '.') 267 (this means explicit relative imports that start with dots have 268 empty strings in this list!) 269 270 :type path: list or None 271 :param path: 272 optional list of path where the module or package should be 273 searched (use sys.path if nothing or None is given) 274 275 :type context_file: str or None 276 :param context_file: 277 context file to consider, necessary if the identifier has been 278 introduced using a relative import unresolvable in the actual 279 context (i.e. modutils) 280 281 :raise ImportError: if there is no such module in the directory 282 283 :rtype: str or None 284 :return: 285 the path to the module's file or None if it's an integrated 286 builtin module such as 'sys' 287 """ 288 if context_file is not None: 289 context = dirname(context_file) 290 else: 291 context = context_file 292 if modpath[0] == 'xml': 293 # handle _xmlplus 294 try: 295 return _file_from_modpath(['_xmlplus'] + modpath[1:], path, context) 296 except ImportError: 297 return _file_from_modpath(modpath, path, context) 298 elif modpath == ['os', 'path']: 299 # FIXME: currently ignoring search_path... 300 return os.path.__file__ 301 return _file_from_modpath(modpath, path, context)
302 303 304
305 -def get_module_part(dotted_name, context_file=None):
306 """given a dotted name return the module part of the name : 307 308 >>> get_module_part('logilab.common.modutils.get_module_part') 309 'logilab.common.modutils' 310 311 :type dotted_name: str 312 :param dotted_name: full name of the identifier we are interested in 313 314 :type context_file: str or None 315 :param context_file: 316 context file to consider, necessary if the identifier has been 317 introduced using a relative import unresolvable in the actual 318 context (i.e. modutils) 319 320 321 :raise ImportError: if there is no such module in the directory 322 323 :rtype: str or None 324 :return: 325 the module part of the name or None if we have not been able at 326 all to import the given name 327 328 XXX: deprecated, since it doesn't handle package precedence over module 329 (see #10066) 330 """ 331 # os.path trick 332 if dotted_name.startswith('os.path'): 333 return 'os.path' 334 parts = dotted_name.split('.') 335 if context_file is not None: 336 # first check for builtin module which won't be considered latter 337 # in that case (path != None) 338 if parts[0] in BUILTIN_MODULES: 339 if len(parts) > 2: 340 raise ImportError(dotted_name) 341 return parts[0] 342 # don't use += or insert, we want a new list to be created ! 343 path = None 344 starti = 0 345 if parts[0] == '': 346 assert context_file is not None, \ 347 'explicit relative import, but no context_file?' 348 path = [] # prevent resolving the import non-relatively 349 starti = 1 350 while parts[starti] == '': # for all further dots: change context 351 starti += 1 352 context_file = dirname(context_file) 353 for i in range(starti, len(parts)): 354 try: 355 file_from_modpath(parts[starti:i+1], 356 path=path, context_file=context_file) 357 except ImportError: 358 if not i >= max(1, len(parts) - 2): 359 raise 360 return '.'.join(parts[:i]) 361 return dotted_name
362 363
364 -def get_modules(package, src_directory, blacklist=STD_BLACKLIST):
365 """given a package directory return a list of all available python 366 modules in the package and its subpackages 367 368 :type package: str 369 :param package: the python name for the package 370 371 :type src_directory: str 372 :param src_directory: 373 path of the directory corresponding to the package 374 375 :type blacklist: list or tuple 376 :param blacklist: 377 optional list of files or directory to ignore, default to 378 the value of `logilab.common.STD_BLACKLIST` 379 380 :rtype: list 381 :return: 382 the list of all available python modules in the package and its 383 subpackages 384 """ 385 modules = [] 386 for directory, dirnames, filenames in os.walk(src_directory): 387 _handle_blacklist(blacklist, dirnames, filenames) 388 # check for __init__.py 389 if not '__init__.py' in filenames: 390 dirnames[:] = () 391 continue 392 if directory != src_directory: 393 dir_package = directory[len(src_directory):].replace(os.sep, '.') 394 modules.append(package + dir_package) 395 for filename in filenames: 396 if _is_python_file(filename) and filename != '__init__.py': 397 src = join(directory, filename) 398 module = package + src[len(src_directory):-3] 399 modules.append(module.replace(os.sep, '.')) 400 return modules
401 402 403
404 -def get_module_files(src_directory, blacklist=STD_BLACKLIST):
405 """given a package directory return a list of all available python 406 module's files in the package and its subpackages 407 408 :type src_directory: str 409 :param src_directory: 410 path of the directory corresponding to the package 411 412 :type blacklist: list or tuple 413 :param blacklist: 414 optional list of files or directory to ignore, default to the value of 415 `logilab.common.STD_BLACKLIST` 416 417 :rtype: list 418 :return: 419 the list of all available python module's files in the package and 420 its subpackages 421 """ 422 files = [] 423 for directory, dirnames, filenames in os.walk(src_directory): 424 _handle_blacklist(blacklist, dirnames, filenames) 425 # check for __init__.py 426 if not '__init__.py' in filenames: 427 dirnames[:] = () 428 continue 429 for filename in filenames: 430 if _is_python_file(filename): 431 src = join(directory, filename) 432 files.append(src) 433 return files
434 435
436 -def get_source_file(filename, include_no_ext=False):
437 """given a python module's file name return the matching source file 438 name (the filename will be returned identically if it's a already an 439 absolute path to a python source file...) 440 441 :type filename: str 442 :param filename: python module's file name 443 444 445 :raise NoSourceFile: if no source file exists on the file system 446 447 :rtype: str 448 :return: the absolute path of the source file if it exists 449 """ 450 base, orig_ext = splitext(abspath(filename)) 451 for ext in PY_SOURCE_EXTS: 452 source_path = '%s.%s' % (base, ext) 453 if exists(source_path): 454 return source_path 455 if include_no_ext and not orig_ext and exists(base): 456 return base 457 raise NoSourceFile(filename)
458 459
460 -def cleanup_sys_modules(directories):
461 """remove submodules of `directories` from `sys.modules`""" 462 cleaned = [] 463 for modname, module in list(sys.modules.items()): 464 modfile = getattr(module, '__file__', None) 465 if modfile: 466 for directory in directories: 467 if modfile.startswith(directory): 468 cleaned.append(modname) 469 del sys.modules[modname] 470 break 471 return cleaned
472 473
474 -def is_python_source(filename):
475 """ 476 rtype: bool 477 return: True if the filename is a python source file 478 """ 479 return splitext(filename)[1][1:] in PY_SOURCE_EXTS
480 481
482 -def is_standard_module(modname, std_path=(STD_LIB_DIR,)):
483 """try to guess if a module is a standard python module (by default, 484 see `std_path` parameter's description) 485 486 :type modname: str 487 :param modname: name of the module we are interested in 488 489 :type std_path: list(str) or tuple(str) 490 :param std_path: list of path considered as standard 491 492 493 :rtype: bool 494 :return: 495 true if the module: 496 - is located on the path listed in one of the directory in `std_path` 497 - is a built-in module 498 499 Note: this function is known to return wrong values when inside virtualenv. 500 See https://www.logilab.org/ticket/294756. 501 """ 502 modname = modname.split('.')[0] 503 try: 504 filename = file_from_modpath([modname]) 505 except ImportError as ex: 506 # import failed, i'm probably not so wrong by supposing it's 507 # not standard... 508 return False 509 # modules which are not living in a file are considered standard 510 # (sys and __builtin__ for instance) 511 if filename is None: 512 # we assume there are no namespaces in stdlib 513 return not _is_namespace(modname) 514 filename = abspath(filename) 515 if filename.startswith(EXT_LIB_DIR): 516 return False 517 for path in std_path: 518 if filename.startswith(abspath(path)): 519 return True 520 return False
521 522 523
524 -def is_relative(modname, from_file):
525 """return true if the given module name is relative to the given 526 file name 527 528 :type modname: str 529 :param modname: name of the module we are interested in 530 531 :type from_file: str 532 :param from_file: 533 path of the module from which modname has been imported 534 535 :rtype: bool 536 :return: 537 true if the module has been imported relatively to `from_file` 538 """ 539 if not isdir(from_file): 540 from_file = dirname(from_file) 541 if from_file in sys.path: 542 return False 543 try: 544 find_module(modname.split('.')[0], [from_file]) 545 return True 546 except ImportError: 547 return False
548 549 550 # internal only functions ##################################################### 551
552 -def _file_from_modpath(modpath, path=None, context=None):
553 """given a mod path (i.e. splitted module / package name), return the 554 corresponding file 555 556 this function is used internally, see `file_from_modpath`'s 557 documentation for more information 558 """ 559 assert len(modpath) > 0 560 if context is not None: 561 try: 562 mtype, mp_filename = _module_file(modpath, [context]) 563 except ImportError: 564 mtype, mp_filename = _module_file(modpath, path) 565 else: 566 mtype, mp_filename = _module_file(modpath, path) 567 if mtype == PY_COMPILED: 568 try: 569 return get_source_file(mp_filename) 570 except NoSourceFile: 571 return mp_filename 572 elif mtype == C_BUILTIN: 573 # integrated builtin module 574 return None 575 elif mtype == PKG_DIRECTORY: 576 mp_filename = _has_init(mp_filename) 577 return mp_filename
578
579 -def _search_zip(modpath, pic):
580 for filepath, importer in pic.items(): 581 if importer is not None: 582 if importer.find_module(modpath[0]): 583 if not importer.find_module('/'.join(modpath)): 584 raise ImportError('No module named %s in %s/%s' % ( 585 '.'.join(modpath[1:]), filepath, modpath)) 586 return ZIPFILE, abspath(filepath) + '/' + '/'.join(modpath), filepath 587 raise ImportError('No module named %s' % '.'.join(modpath))
588 589 try: 590 import pkg_resources 591 except ImportError: 592 pkg_resources = None 593 594
595 -def _is_namespace(modname):
596 return (pkg_resources is not None 597 and modname in pkg_resources._namespace_packages)
598 599
600 -def _module_file(modpath, path=None):
601 """get a module type / file path 602 603 :type modpath: list or tuple 604 :param modpath: 605 splitted module's name (i.e name of a module or package splitted 606 on '.'), with leading empty strings for explicit relative import 607 608 :type path: list or None 609 :param path: 610 optional list of path where the module or package should be 611 searched (use sys.path if nothing or None is given) 612 613 614 :rtype: tuple(int, str) 615 :return: the module type flag and the file path for a module 616 """ 617 # egg support compat 618 try: 619 pic = sys.path_importer_cache 620 _path = (path is None and sys.path or path) 621 for __path in _path: 622 if not __path in pic: 623 try: 624 pic[__path] = zipimport.zipimporter(__path) 625 except zipimport.ZipImportError: 626 pic[__path] = None 627 checkeggs = True 628 except AttributeError: 629 checkeggs = False 630 # pkg_resources support (aka setuptools namespace packages) 631 if (_is_namespace(modpath[0]) and modpath[0] in sys.modules): 632 # setuptools has added into sys.modules a module object with proper 633 # __path__, get back information from there 634 module = sys.modules[modpath.pop(0)] 635 path = module.__path__ 636 if not modpath: 637 return C_BUILTIN, None 638 imported = [] 639 while modpath: 640 modname = modpath[0] 641 # take care to changes in find_module implementation wrt builtin modules 642 # 643 # Python 2.6.6 (r266:84292, Sep 11 2012, 08:34:23) 644 # >>> imp.find_module('posix') 645 # (None, 'posix', ('', '', 6)) 646 # 647 # Python 3.3.1 (default, Apr 26 2013, 12:08:46) 648 # >>> imp.find_module('posix') 649 # (None, None, ('', '', 6)) 650 try: 651 _, mp_filename, mp_desc = find_module(modname, path) 652 except ImportError: 653 if checkeggs: 654 return _search_zip(modpath, pic)[:2] 655 raise 656 else: 657 if checkeggs and mp_filename: 658 fullabspath = [abspath(x) for x in _path] 659 try: 660 pathindex = fullabspath.index(dirname(abspath(mp_filename))) 661 emtype, emp_filename, zippath = _search_zip(modpath, pic) 662 if pathindex > _path.index(zippath): 663 # an egg takes priority 664 return emtype, emp_filename 665 except ValueError: 666 # XXX not in _path 667 pass 668 except ImportError: 669 pass 670 checkeggs = False 671 imported.append(modpath.pop(0)) 672 mtype = mp_desc[2] 673 if modpath: 674 if mtype != PKG_DIRECTORY: 675 raise ImportError('No module %s in %s' % ('.'.join(modpath), 676 '.'.join(imported))) 677 # XXX guess if package is using pkgutil.extend_path by looking for 678 # those keywords in the first four Kbytes 679 try: 680 with open(join(mp_filename, '__init__.py')) as stream: 681 data = stream.read(4096) 682 except IOError: 683 path = [mp_filename] 684 else: 685 if 'pkgutil' in data and 'extend_path' in data: 686 # extend_path is called, search sys.path for module/packages 687 # of this name see pkgutil.extend_path documentation 688 path = [join(p, *imported) for p in sys.path 689 if isdir(join(p, *imported))] 690 else: 691 path = [mp_filename] 692 return mtype, mp_filename
693
694 -def _is_python_file(filename):
695 """return true if the given filename should be considered as a python file 696 697 .pyc and .pyo are ignored 698 """ 699 for ext in ('.py', '.so', '.pyd', '.pyw'): 700 if filename.endswith(ext): 701 return True 702 return False
703 704
705 -def _has_init(directory):
706 """if the given directory has a valid __init__ file, return its path, 707 else return None 708 """ 709 mod_or_pack = join(directory, '__init__') 710 for ext in PY_SOURCE_EXTS + ('pyc', 'pyo'): 711 if exists(mod_or_pack + '.' + ext): 712 return mod_or_pack + '.' + ext 713 return None
714