# (c) 2005 Ian Bicking and contributors; written for Paste (http://pythonpaste.org)
# Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php
# (c) 2005 Ian Bicking and contributors; written for Paste (http://pythonpaste.org)
# Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php
"""
These are functions for use when doctest-testing a document.
"""
import subprocess
import doctest
import os
import sys
import shutil
import re
import cgi
import rfc822
from cStringIO import StringIO
from paste.util import PySourceColor
here = os.path.abspath(__file__)
paste_parent = os.path.dirname(
os.path.dirname(os.path.dirname(here)))
def run(command):
data = run_raw(command)
if data:
print data
def run_raw(command):
"""
Runs the string command, returns any output.
"""
proc = subprocess.Popen(command, shell=True,
stderr=subprocess.STDOUT,
stdout=subprocess.PIPE, env=_make_env())
data = proc.stdout.read()
proc.wait()
while data.endswith('\n') or data.endswith('\r'):
data = data[:-1]
if data:
data = '\n'.join(
[l for l in data.splitlines() if l])
return data
else:
return ''
def run_command(command, name, and_print=False):
output = run_raw(command)
data = '$ %s\n%s' % (command, output)
show_file('shell-command', name, description='shell transcript',
data=data)
if and_print and output:
print output
def _make_env():
env = os.environ.copy()
env['PATH'] = (env.get('PATH', '')
+ ':'
+ os.path.join(paste_parent, 'scripts')
+ ':'
+ os.path.join(paste_parent, 'paste', '3rd-party',
'sqlobject-files', 'scripts'))
env['PYTHONPATH'] = (env.get('PYTHONPATH', '')
+ ':'
+ paste_parent)
return env
def clear_dir(dir):
"""
Clears (deletes) the given directory
"""
shutil.rmtree(dir, True)
def ls(dir=None, recurse=False, indent=0):
"""
Show a directory listing
"""
dir = dir or os.getcwd()
fns = os.listdir(dir)
fns.sort()
for fn in fns:
full = os.path.join(dir, fn)
if os.path.isdir(full):
fn = fn + '/'
print ' '*indent + fn
if os.path.isdir(full) and recurse:
ls(dir=full, recurse=True, indent=indent+2)
default_app = None
default_url = None
def set_default_app(app, url):
global default_app
global default_url
default_app = app
default_url = url
def resource_filename(fn):
"""
Returns the filename of the resource -- generally in the directory
resources/DocumentName/fn
"""
return os.path.join(
os.path.dirname(sys.testing_document_filename),
'resources',
os.path.splitext(os.path.basename(sys.testing_document_filename))[0],
fn)
def show(path_info, example_name):
fn = resource_filename(example_name + '.html')
out = StringIO()
assert default_app is not None, (
"No default_app set")
url = default_url + path_info
out.write('<span class="doctest-url"><a href="%s">%s</a></span><br>\n'
% (url, url))
out.write('<div class="doctest-example">\n')
proc = subprocess.Popen(
['paster', 'serve' '--server=console', '--no-verbose',
'--url=' + path_info],
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
env=_make_env())
stdout, errors = proc.communicate()
stdout = StringIO(stdout)
headers = rfc822.Message(stdout)
content = stdout.read()
for header, value in headers.items():
if header.lower() == 'status' and int(value.split()[0]) == 200:
continue
if header.lower() in ('content-type', 'content-length'):
continue
if (header.lower() == 'set-cookie'
and value.startswith('_SID_')):
continue
out.write('<span class="doctest-header">%s: %s</span><br>\n'
% (header, value))
lines = [l for l in content.splitlines() if l.strip()]
for line in lines:
out.write(line + '\n')
if errors:
out.write('<pre class="doctest-errors">%s</pre>'
% errors)
out.write('</div>\n')
result = out.getvalue()
if not os.path.exists(fn):
f = open(fn, 'wb')
f.write(result)
f.close()
else:
f = open(fn, 'rb')
expected = f.read()
f.close()
if not html_matches(expected, result):
print 'Pages did not match. Expected from %s:' % fn
print '-'*60
print expected
print '='*60
print 'Actual output:'
print '-'*60
print result
def html_matches(pattern, text):
regex = re.escape(pattern)
regex = regex.replace(r'\.\.\.', '.*')
regex = re.sub(r'0x[0-9a-f]+', '.*', regex)
regex = '^%s$' % regex
return re.search(regex, text)
def convert_docstring_string(data):
if data.startswith('\n'):
data = data[1:]
lines = data.splitlines()
new_lines = []
for line in lines:
if line.rstrip() == '.':
new_lines.append('')
else:
new_lines.append(line)
data = '\n'.join(new_lines) + '\n'
return data
def create_file(path, version, data):
data = convert_docstring_string(data)
write_data(path, data)
show_file(path, version)
def append_to_file(path, version, data):
data = convert_docstring_string(data)
f = open(path, 'a')
f.write(data)
f.close()
# I think these appends can happen so quickly (in less than a second)
# that the .pyc file doesn't appear to be expired, even though it
# is after we've made this change; so we have to get rid of the .pyc
# file:
if path.endswith('.py'):
pyc_file = path + 'c'
if os.path.exists(pyc_file):
os.unlink(pyc_file)
show_file(path, version, description='added to %s' % path,
data=data)
def show_file(path, version, description=None, data=None):
ext = os.path.splitext(path)[1]
if data is None:
f = open(path, 'rb')
data = f.read()
f.close()
if ext == '.py':
html = ('<div class="source-code">%s</div>'
% PySourceColor.str2html(data, PySourceColor.dark))
else:
html = '<pre class="source-code">%s</pre>' % cgi.escape(data, 1)
html = '<span class="source-filename">%s</span><br>%s' % (
description or path, html)
write_data(resource_filename('%s.%s.gen.html' % (path, version)),
html)
def call_source_highlight(input, format):
proc = subprocess.Popen(['source-highlight', '--out-format=html',
'--no-doc', '--css=none',
'--src-lang=%s' % format], shell=False,
stdout=subprocess.PIPE)
stdout, stderr = proc.communicate(input)
result = stdout
proc.wait()
return result
def write_data(path, data):
dir = os.path.dirname(os.path.abspath(path))
if not os.path.exists(dir):
os.makedirs(dir)
f = open(path, 'wb')
f.write(data)
f.close()
def change_file(path, changes):
f = open(os.path.abspath(path), 'rb')
lines = f.readlines()
f.close()
for change_type, line, text in changes:
if change_type == 'insert':
lines[line:line] = [text]
elif change_type == 'delete':
lines[line:text] = []
else:
assert 0, (
"Unknown change_type: %r" % change_type)
f = open(path, 'wb')
f.write(''.join(lines))
f.close()
class LongFormDocTestParser(doctest.DocTestParser):
"""
This parser recognizes some reST comments as commands, without
prompts or expected output, like:
.. run:
do_this(...
...)
"""
_EXAMPLE_RE = re.compile(r"""
# Source consists of a PS1 line followed by zero or more PS2 lines.
(?: (?P<source>
(?:^(?P<indent> [ ]*) >>> .*) # PS1 line
(?:\n [ ]* \.\.\. .*)*) # PS2 lines
\n?
# Want consists of any non-blank lines that do not start with PS1.
(?P<want> (?:(?![ ]*$) # Not a blank line
(?![ ]*>>>) # Not a line starting with PS1
.*$\n? # But any other line
)*))
|
(?: # This is for longer commands that are prefixed with a reST
# comment like '.. run:' (two colons makes that a directive).
# These commands cannot have any output.
(?:^\.\.[ ]*(?P<run>run):[ ]*\n) # Leading command/command
(?:[ ]*\n)? # Blank line following
(?P<runsource>
(?:(?P<runindent> [ ]+)[^ ].*$)
(?:\n [ ]+ .*)*)
)
|
(?: # This is for shell commands
(?P<shellsource>
(?:^(P<shellindent> [ ]*) [$] .*) # Shell line
(?:\n [ ]* [>] .*)*) # Continuation
\n?
# Want consists of any non-blank lines that do not start with $
(?P<shellwant> (?:(?![ ]*$)
(?![ ]*[$]$)
.*$\n?
)*))
""", re.MULTILINE | re.VERBOSE)
def _parse_example(self, m, name, lineno):
r"""
Given a regular expression match from `_EXAMPLE_RE` (`m`),
return a pair `(source, want)`, where `source` is the matched
example's source code (with prompts and indentation stripped);
and `want` is the example's expected output (with indentation
stripped).
`name` is the string's name, and `lineno` is the line number
where the example starts; both are used for error messages.
>>> def parseit(s):
... p = LongFormDocTestParser()
... return p._parse_example(p._EXAMPLE_RE.search(s), '<string>', 1)
>>> parseit('>>> 1\n1')
('1', {}, '1', None)
>>> parseit('>>> (1\n... +1)\n2')
('(1\n+1)', {}, '2', None)
>>> parseit('.. run:\n\n test1\n test2\n')
('test1\ntest2', {}, '', None)
"""
# Get the example's indentation level.
runner = m.group('run') or ''
indent = len(m.group('%sindent' % runner))
# Divide source into lines; check that they're properly
# indented; and then strip their indentation & prompts.
source_lines = m.group('%ssource' % runner).split('\n')
if runner:
self._check_prefix(source_lines[1:], ' '*indent, name, lineno)
else:
self._check_prompt_blank(source_lines, indent, name, lineno)
self._check_prefix(source_lines[2:], ' '*indent + '.', name, lineno)
if runner:
source = '\n'.join([sl[indent:] for sl in source_lines])
else:
source = '\n'.join([sl[indent+4:] for sl in source_lines])
if runner:
want = ''
exc_msg = None
else:
# Divide want into lines; check that it's properly indented; and
# then strip the indentation. Spaces before the last newline should
# be preserved, so plain rstrip() isn't good enough.
want = m.group('want')
want_lines = want.split('\n')
if len(want_lines) > 1 and re.match(r' *$', want_lines[-1]):
del want_lines[-1] # forget final newline & spaces after it
self._check_prefix(want_lines, ' '*indent, name,
lineno + len(source_lines))
want = '\n'.join([wl[indent:] for wl in want_lines])
# If `want` contains a traceback message, then extract it.
m = self._EXCEPTION_RE.match(want)
if m:
exc_msg = m.group('msg')
else:
exc_msg = None
# Extract options from the source.
options = self._find_options(source, name, lineno)
return source, options, want, exc_msg
def parse(self, string, name='<string>'):
"""
Divide the given string into examples and intervening text,
and return them as a list of alternating Examples and strings.
Line numbers for the Examples are 0-based. The optional
argument `name` is a name identifying this string, and is only
used for error messages.
"""
string = string.expandtabs()
# If all lines begin with the same indentation, then strip it.
min_indent = self._min_indent(string)
if min_indent > 0:
string = '\n'.join([l[min_indent:] for l in string.split('\n')])
output = []
charno, lineno = 0, 0
# Find all doctest examples in the string:
for m in self._EXAMPLE_RE.finditer(string):
# Add the pre-example text to `output`.
output.append(string[charno:m.start()])
# Update lineno (lines before this example)
lineno += string.count('\n', charno, m.start())
# Extract info from the regexp match.
(source, options, want, exc_msg) = \
self._parse_example(m, name, lineno)
# Create an Example, and add it to the list.
if not self._IS_BLANK_OR_COMMENT(source):
# @@: Erg, this is the only line I need to change...
output.append(doctest.Example(
source, want, exc_msg,
lineno=lineno,
indent=min_indent+len(m.group('indent') or m.group('runindent')),
options=options))
# Update lineno (lines inside this example)
lineno += string.count('\n', m.start(), m.end())
# Update charno.
charno = m.end()
# Add any remaining post-example text to `output`.
output.append(string[charno:])
return output
if __name__ == '__main__':
if sys.argv[1:] and sys.argv[1] == 'doctest':
doctest.testmod()
sys.exit()
if not paste_parent in sys.path:
sys.path.append(paste_parent)
for fn in sys.argv[1:]:
fn = os.path.abspath(fn)
# @@: OK, ick; but this module gets loaded twice
sys.testing_document_filename = fn
doctest.testfile(
fn, module_relative=False,
optionflags=doctest.ELLIPSIS|doctest.REPORT_ONLY_FIRST_FAILURE,
parser=LongFormDocTestParser())
new = os.path.splitext(fn)[0] + '.html'
assert new != fn
os.system('rst2html.py %s > %s' % (fn, new))