shell bypass 403
# -*- coding: utf-8 -*-
# Copyright (c) 2014-2018 Claudiu Popa <pcmanticore@gmail.com>
# Copyright (c) 2014 Google, Inc.
# Copyright (c) 2014 Denis Laxalde <denis.laxalde@logilab.fr>
# Copyright (c) 2014 LOGILAB S.A. (Paris, FRANCE) <contact@logilab.fr>
# Copyright (c) 2014 Eevee (Alex Munroe) <amunroe@yelp.com>
# Copyright (c) 2015 Florian Bruhin <me@the-compiler.org>
# Copyright (c) 2015 Radosław Ganczarek <radoslaw@ganczarek.in>
# Copyright (c) 2016 Derek Gustafson <degustaf@gmail.com>
# Copyright (c) 2016 Jakub Wilk <jwilk@jwilk.net>
# Copyright (c) 2016 Ceridwen <ceridwenv@gmail.com>
# Copyright (c) 2018 Mario Corchero <mcorcherojim@bloomberg.net>
# Copyright (c) 2018 Mario Corchero <mariocj89@gmail.com>
# Copyright (c) 2018 Anthony Sottile <asottile@umich.edu>
# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html
# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER
"""Python modules manipulation utility functions.
:type PY_SOURCE_EXTS: tuple(str)
:var PY_SOURCE_EXTS: list of possible python source file extension
:type STD_LIB_DIRS: set of str
:var STD_LIB_DIRS: directories where standard modules are located
:type BUILTIN_MODULES: dict
:var BUILTIN_MODULES: dictionary with builtin module names has key
"""
import imp
import os
import platform
import sys
import itertools
from distutils.sysconfig import get_python_lib # pylint: disable=import-error
# pylint: disable=import-error, no-name-in-module
from distutils.errors import DistutilsPlatformError
# distutils is replaced by virtualenv with a module that does
# weird path manipulations in order to get to the
# real distutils module.
from .interpreter._import import spec
from .interpreter._import import util
if sys.platform.startswith("win"):
PY_SOURCE_EXTS = ("py", "pyw")
PY_COMPILED_EXTS = ("dll", "pyd")
else:
PY_SOURCE_EXTS = ("py",)
PY_COMPILED_EXTS = ("so",)
try:
# The explicit sys.prefix is to work around a patch in virtualenv that
# replaces the 'real' sys.prefix (i.e. the location of the binary)
# with the prefix from which the virtualenv was created. This throws
# off the detection logic for standard library modules, thus the
# workaround.
STD_LIB_DIRS = {
get_python_lib(standard_lib=True, prefix=sys.prefix),
# Take care of installations where exec_prefix != prefix.
get_python_lib(standard_lib=True, prefix=sys.exec_prefix),
get_python_lib(standard_lib=True),
}
# get_python_lib(standard_lib=1) is not available on pypy, set STD_LIB_DIR to
# non-valid path, see https://bugs.pypy.org/issue1164
except DistutilsPlatformError:
STD_LIB_DIRS = set()
if os.name == "nt":
STD_LIB_DIRS.add(os.path.join(sys.prefix, "dlls"))
try:
# real_prefix is defined when running inside virtual environments,
# created with the **virtualenv** library.
STD_LIB_DIRS.add(os.path.join(sys.real_prefix, "dlls"))
except AttributeError:
# sys.base_exec_prefix is always defined, but in a virtual environment
# created with the stdlib **venv** module, it points to the original
# installation, if the virtual env is activated.
try:
STD_LIB_DIRS.add(os.path.join(sys.base_exec_prefix, "dlls"))
except AttributeError:
pass
if platform.python_implementation() == "PyPy":
_root = os.path.join(sys.prefix, "lib_pypy")
STD_LIB_DIRS.add(_root)
try:
# real_prefix is defined when running inside virtualenv.
STD_LIB_DIRS.add(os.path.join(sys.real_prefix, "lib_pypy"))
except AttributeError:
pass
del _root
if os.name == "posix":
# Need the real prefix is we're under a virtualenv, otherwise
# the usual one will do.
try:
prefix = sys.real_prefix
except AttributeError:
prefix = sys.prefix
def _posix_path(path):
base_python = "python%d.%d" % sys.version_info[:2]
return os.path.join(prefix, path, base_python)
STD_LIB_DIRS.add(_posix_path("lib"))
if sys.maxsize > 2 ** 32:
# This tries to fix a problem with /usr/lib64 builds,
# where systems are running both 32-bit and 64-bit code
# on the same machine, which reflects into the places where
# standard library could be found. More details can be found
# here http://bugs.python.org/issue1294959.
# An easy reproducing case would be
# https://github.com/PyCQA/pylint/issues/712#issuecomment-163178753
STD_LIB_DIRS.add(_posix_path("lib64"))
EXT_LIB_DIR = get_python_lib()
IS_JYTHON = platform.python_implementation() == "Jython"
BUILTIN_MODULES = dict.fromkeys(sys.builtin_module_names, True)
class NoSourceFile(Exception):
"""exception raised when we are not able to get a python
source file for a precompiled file
"""
def _normalize_path(path):
return os.path.normcase(os.path.abspath(path))
def _canonicalize_path(path):
return os.path.realpath(os.path.expanduser(path))
def _path_from_filename(filename, is_jython=IS_JYTHON):
if not is_jython:
if sys.version_info > (3, 0):
return filename
if filename.endswith(".pyc"):
return filename[:-1]
return filename
head, has_pyclass, _ = filename.partition("$py.class")
if has_pyclass:
return head + ".py"
return filename
def _handle_blacklist(blacklist, dirnames, filenames):
"""remove files/directories in the black list
dirnames/filenames are usually from os.walk
"""
for norecurs in blacklist:
if norecurs in dirnames:
dirnames.remove(norecurs)
elif norecurs in filenames:
filenames.remove(norecurs)
_NORM_PATH_CACHE = {}
def _cache_normalize_path(path):
"""abspath with caching"""
# _module_file calls abspath on every path in sys.path every time it's
# called; on a larger codebase this easily adds up to half a second just
# assembling path components. This cache alleviates that.
try:
return _NORM_PATH_CACHE[path]
except KeyError:
if not path: # don't cache result for ''
return _normalize_path(path)
result = _NORM_PATH_CACHE[path] = _normalize_path(path)
return result
def load_module_from_name(dotted_name, path=None, use_sys=True):
"""Load a Python module from its name.
:type dotted_name: str
:param dotted_name: python name of a module or package
:type path: list or None
:param path:
optional list of path where the module or package should be
searched (use sys.path if nothing or None is given)
:type use_sys: bool
:param use_sys:
boolean indicating whether the sys.modules dictionary should be
used or not
:raise ImportError: if the module or package is not found
:rtype: module
:return: the loaded module
"""
return load_module_from_modpath(dotted_name.split("."), path, use_sys)
def load_module_from_modpath(parts, path=None, use_sys=1):
"""Load a python module from its split name.
:type parts: list(str) or tuple(str)
:param parts:
python name of a module or package split on '.'
:type path: list or None
:param path:
optional list of path where the module or package should be
searched (use sys.path if nothing or None is given)
:type use_sys: bool
:param use_sys:
boolean indicating whether the sys.modules dictionary should be used or not
:raise ImportError: if the module or package is not found
:rtype: module
:return: the loaded module
"""
if use_sys:
try:
return sys.modules[".".join(parts)]
except KeyError:
pass
modpath = []
prevmodule = None
for part in parts:
modpath.append(part)
curname = ".".join(modpath)
module = None
if len(modpath) != len(parts):
# even with use_sys=False, should try to get outer packages from sys.modules
module = sys.modules.get(curname)
elif use_sys:
# because it may have been indirectly loaded through a parent
module = sys.modules.get(curname)
if module is None:
mp_file, mp_filename, mp_desc = imp.find_module(part, path)
module = imp.load_module(curname, mp_file, mp_filename, mp_desc)
# mp_file still needs to be closed.
if mp_file:
mp_file.close()
if prevmodule:
setattr(prevmodule, part, module)
_file = getattr(module, "__file__", "")
prevmodule = module
if not _file and util.is_namespace(curname):
continue
if not _file and len(modpath) != len(parts):
raise ImportError("no module in %s" % ".".join(parts[len(modpath) :]))
path = [os.path.dirname(_file)]
return module
def load_module_from_file(filepath, path=None, use_sys=True, extrapath=None):
"""Load a Python module from it's path.
:type filepath: str
:param filepath: path to the python module or package
:type path: list or None
:param path:
optional list of path where the module or package should be
searched (use sys.path if nothing or None is given)
:type use_sys: bool
:param use_sys:
boolean indicating whether the sys.modules dictionary should be
used or not
:raise ImportError: if the module or package is not found
:rtype: module
:return: the loaded module
"""
modpath = modpath_from_file(filepath, extrapath)
return load_module_from_modpath(modpath, path, use_sys)
def check_modpath_has_init(path, mod_path):
"""check there are some __init__.py all along the way"""
modpath = []
for part in mod_path:
modpath.append(part)
path = os.path.join(path, part)
if not _has_init(path):
old_namespace = util.is_namespace(".".join(modpath))
if not old_namespace:
return False
return True
def _get_relative_base_path(filename, path_to_check):
"""Extracts the relative mod path of the file to import from
Check if a file is within the passed in path and if so, returns the
relative mod path from the one passed in.
If the filename is no in path_to_check, returns None
Note this function will look for both abs and realpath of the file,
this allows to find the relative base path even if the file is a
symlink of a file in the passed in path
Examples:
_get_relative_base_path("/a/b/c/d.py", "/a/b") -> ["c","d"]
_get_relative_base_path("/a/b/c/d.py", "/dev") -> None
"""
importable_path = None
path_to_check = os.path.normcase(path_to_check)
abs_filename = os.path.abspath(filename)
if os.path.normcase(abs_filename).startswith(path_to_check):
importable_path = abs_filename
real_filename = os.path.realpath(filename)
if os.path.normcase(real_filename).startswith(path_to_check):
importable_path = real_filename
if importable_path:
base_path = os.path.splitext(importable_path)[0]
relative_base_path = base_path[len(path_to_check) :]
return [pkg for pkg in relative_base_path.split(os.sep) if pkg]
return None
def modpath_from_file_with_callback(filename, extrapath=None, is_package_cb=None):
filename = os.path.expanduser(_path_from_filename(filename))
if extrapath is not None:
for path_ in itertools.chain(map(_canonicalize_path, extrapath), extrapath):
path = os.path.abspath(path_)
if not path:
continue
submodpath = _get_relative_base_path(filename, path)
if not submodpath:
continue
if is_package_cb(path, submodpath[:-1]):
return extrapath[path_].split(".") + submodpath
for path in itertools.chain(map(_canonicalize_path, sys.path), sys.path):
path = _cache_normalize_path(path)
if not path:
continue
modpath = _get_relative_base_path(filename, path)
if not modpath:
continue
if is_package_cb(path, modpath[:-1]):
return modpath
raise ImportError(
"Unable to find module for %s in %s" % (filename, ", \n".join(sys.path))
)
def modpath_from_file(filename, extrapath=None):
"""given a file path return the corresponding split module's name
(i.e name of a module or package split on '.')
:type filename: str
:param filename: file's path for which we want the module's name
:type extrapath: dict
:param extrapath:
optional extra search path, with path as key and package name for the path
as value. This is usually useful to handle package split in multiple
directories using __path__ trick.
:raise ImportError:
if the corresponding module's name has not been found
:rtype: list(str)
:return: the corresponding split module's name
"""
return modpath_from_file_with_callback(filename, extrapath, check_modpath_has_init)
def file_from_modpath(modpath, path=None, context_file=None):
return file_info_from_modpath(modpath, path, context_file).location
def file_info_from_modpath(modpath, path=None, context_file=None):
"""given a mod path (i.e. split module / package name), return the
corresponding file, giving priority to source file over precompiled
file if it exists
:type modpath: list or tuple
:param modpath:
split module's name (i.e name of a module or package split
on '.')
(this means explicit relative imports that start with dots have
empty strings in this list!)
:type path: list or None
:param path:
optional list of path where the module or package should be
searched (use sys.path if nothing or None is given)
:type context_file: str or None
:param context_file:
context file to consider, necessary if the identifier has been
introduced using a relative import unresolvable in the actual
context (i.e. modutils)
:raise ImportError: if there is no such module in the directory
:rtype: (str or None, import type)
:return:
the path to the module's file or None if it's an integrated
builtin module such as 'sys'
"""
if context_file is not None:
context = os.path.dirname(context_file)
else:
context = context_file
if modpath[0] == "xml":
# handle _xmlplus
try:
return _spec_from_modpath(["_xmlplus"] + modpath[1:], path, context)
except ImportError:
return _spec_from_modpath(modpath, path, context)
elif modpath == ["os", "path"]:
# FIXME: currently ignoring search_path...
return spec.ModuleSpec(
name="os.path", location=os.path.__file__, module_type=imp.PY_SOURCE
)
return _spec_from_modpath(modpath, path, context)
def get_module_part(dotted_name, context_file=None):
"""given a dotted name return the module part of the name :
>>> get_module_part('astroid.as_string.dump')
'astroid.as_string'
:type dotted_name: str
:param dotted_name: full name of the identifier we are interested in
:type context_file: str or None
:param context_file:
context file to consider, necessary if the identifier has been
introduced using a relative import unresolvable in the actual
context (i.e. modutils)
:raise ImportError: if there is no such module in the directory
:rtype: str or None
:return:
the module part of the name or None if we have not been able at
all to import the given name
XXX: deprecated, since it doesn't handle package precedence over module
(see #10066)
"""
# os.path trick
if dotted_name.startswith("os.path"):
return "os.path"
parts = dotted_name.split(".")
if context_file is not None:
# first check for builtin module which won't be considered latter
# in that case (path != None)
if parts[0] in BUILTIN_MODULES:
if len(parts) > 2:
raise ImportError(dotted_name)
return parts[0]
# don't use += or insert, we want a new list to be created !
path = None
starti = 0
if parts[0] == "":
assert (
context_file is not None
), "explicit relative import, but no context_file?"
path = [] # prevent resolving the import non-relatively
starti = 1
while parts[starti] == "": # for all further dots: change context
starti += 1
context_file = os.path.dirname(context_file)
for i in range(starti, len(parts)):
try:
file_from_modpath(
parts[starti : i + 1], path=path, context_file=context_file
)
except ImportError:
if i < max(1, len(parts) - 2):
raise
return ".".join(parts[:i])
return dotted_name
def get_module_files(src_directory, blacklist, list_all=False):
"""given a package directory return a list of all available python
module's files in the package and its subpackages
:type src_directory: str
:param src_directory:
path of the directory corresponding to the package
:type blacklist: list or tuple
:param blacklist: iterable
list of files or directories to ignore.
:type list_all: bool
:param list_all:
get files from all paths, including ones without __init__.py
:rtype: list
:return:
the list of all available python module's files in the package and
its subpackages
"""
files = []
for directory, dirnames, filenames in os.walk(src_directory):
if directory in blacklist:
continue
_handle_blacklist(blacklist, dirnames, filenames)
# check for __init__.py
if not list_all and "__init__.py" not in filenames:
dirnames[:] = ()
continue
for filename in filenames:
if _is_python_file(filename):
src = os.path.join(directory, filename)
files.append(src)
return files
def get_source_file(filename, include_no_ext=False):
"""given a python module's file name return the matching source file
name (the filename will be returned identically if it's already an
absolute path to a python source file...)
:type filename: str
:param filename: python module's file name
:raise NoSourceFile: if no source file exists on the file system
:rtype: str
:return: the absolute path of the source file if it exists
"""
filename = os.path.abspath(_path_from_filename(filename))
base, orig_ext = os.path.splitext(filename)
for ext in PY_SOURCE_EXTS:
source_path = "%s.%s" % (base, ext)
if os.path.exists(source_path):
return source_path
if include_no_ext and not orig_ext and os.path.exists(base):
return base
raise NoSourceFile(filename)
def is_python_source(filename):
"""
rtype: bool
return: True if the filename is a python source file
"""
return os.path.splitext(filename)[1][1:] in PY_SOURCE_EXTS
def is_standard_module(modname, std_path=None):
"""try to guess if a module is a standard python module (by default,
see `std_path` parameter's description)
:type modname: str
:param modname: name of the module we are interested in
:type std_path: list(str) or tuple(str)
:param std_path: list of path considered has standard
:rtype: bool
:return:
true if the module:
- is located on the path listed in one of the directory in `std_path`
- is a built-in module
"""
modname = modname.split(".")[0]
try:
filename = file_from_modpath([modname])
except ImportError:
# import failed, i'm probably not so wrong by supposing it's
# not standard...
return False
# modules which are not living in a file are considered standard
# (sys and __builtin__ for instance)
if filename is None:
# we assume there are no namespaces in stdlib
return not util.is_namespace(modname)
filename = _normalize_path(filename)
if filename.startswith(_cache_normalize_path(EXT_LIB_DIR)):
return False
if std_path is None:
std_path = STD_LIB_DIRS
for path in std_path:
if filename.startswith(_cache_normalize_path(path)):
return True
return False
def is_relative(modname, from_file):
"""return true if the given module name is relative to the given
file name
:type modname: str
:param modname: name of the module we are interested in
:type from_file: str
:param from_file:
path of the module from which modname has been imported
:rtype: bool
:return:
true if the module has been imported relatively to `from_file`
"""
if not os.path.isdir(from_file):
from_file = os.path.dirname(from_file)
if from_file in sys.path:
return False
try:
stream, _, _ = imp.find_module(modname.split(".")[0], [from_file])
# Close the stream to avoid ResourceWarnings.
if stream:
stream.close()
return True
except ImportError:
return False
# internal only functions #####################################################
def _spec_from_modpath(modpath, path=None, context=None):
"""given a mod path (i.e. split module / package name), return the
corresponding spec
this function is used internally, see `file_from_modpath`'s
documentation for more information
"""
assert modpath
location = None
if context is not None:
try:
found_spec = spec.find_spec(modpath, [context])
location = found_spec.location
except ImportError:
found_spec = spec.find_spec(modpath, path)
location = found_spec.location
else:
found_spec = spec.find_spec(modpath, path)
if found_spec.type == spec.ModuleType.PY_COMPILED:
try:
location = get_source_file(found_spec.location)
return found_spec._replace(
location=location, type=spec.ModuleType.PY_SOURCE
)
except NoSourceFile:
return found_spec._replace(location=location)
elif found_spec.type == spec.ModuleType.C_BUILTIN:
# integrated builtin module
return found_spec._replace(location=None)
elif found_spec.type == spec.ModuleType.PKG_DIRECTORY:
location = _has_init(found_spec.location)
return found_spec._replace(location=location, type=spec.ModuleType.PY_SOURCE)
return found_spec
def _is_python_file(filename):
"""return true if the given filename should be considered as a python file
.pyc and .pyo are ignored
"""
for ext in (".py", ".so", ".pyd", ".pyw"):
if filename.endswith(ext):
return True
return False
def _has_init(directory):
"""if the given directory has a valid __init__ file, return its path,
else return None
"""
mod_or_pack = os.path.join(directory, "__init__")
for ext in PY_SOURCE_EXTS + ("pyc", "pyo"):
if os.path.exists(mod_or_pack + "." + ext):
return mod_or_pack + "." + ext
return None
def is_namespace(specobj):
return specobj.type == spec.ModuleType.PY_NAMESPACE
def is_directory(specobj):
return specobj.type == spec.ModuleType.PKG_DIRECTORY