537 lines
17 KiB
Python
537 lines
17 KiB
Python
"""
|
|
INLINE PATTERNS
|
|
=============================================================================
|
|
|
|
Inline patterns such as *emphasis* are handled by means of auxiliary
|
|
objects, one per pattern. Pattern objects must be instances of classes
|
|
that extend markdown.Pattern. Each pattern object uses a single regular
|
|
expression and needs support the following methods:
|
|
|
|
pattern.getCompiledRegExp() # returns a regular expression
|
|
|
|
pattern.handleMatch(m) # takes a match object and returns
|
|
# an ElementTree element or just plain text
|
|
|
|
All of python markdown's built-in patterns subclass from Pattern,
|
|
but you can add additional patterns that don't.
|
|
|
|
Also note that all the regular expressions used by inline must
|
|
capture the whole block. For this reason, they all start with
|
|
'^(.*)' and end with '(.*)!'. In case with built-in expression
|
|
Pattern takes care of adding the "^(.*)" and "(.*)!".
|
|
|
|
Finally, the order in which regular expressions are applied is very
|
|
important - e.g. if we first replace http://.../ links with <a> tags
|
|
and _then_ try to replace inline html, we would end up with a mess.
|
|
So, we apply the expressions in the following order:
|
|
|
|
* escape and backticks have to go before everything else, so
|
|
that we can preempt any markdown patterns by escaping them.
|
|
|
|
* then we handle auto-links (must be done before inline html)
|
|
|
|
* then we handle inline HTML. At this point we will simply
|
|
replace all inline HTML strings with a placeholder and add
|
|
the actual HTML to a hash.
|
|
|
|
* then inline images (must be done before links)
|
|
|
|
* then bracketed links, first regular then reference-style
|
|
|
|
* finally we apply strong and emphasis
|
|
"""
|
|
|
|
from __future__ import absolute_import
|
|
from __future__ import unicode_literals
|
|
from . import util
|
|
from . import odict
|
|
import re
|
|
try: # pragma: no cover
|
|
from urllib.parse import urlparse, urlunparse
|
|
except ImportError: # pragma: no cover
|
|
from urlparse import urlparse, urlunparse
|
|
try: # pragma: no cover
|
|
from html import entities
|
|
except ImportError: # pragma: no cover
|
|
import htmlentitydefs as entities
|
|
|
|
|
|
def build_inlinepatterns(md_instance, **kwargs):
|
|
""" Build the default set of inline patterns for Markdown. """
|
|
inlinePatterns = odict.OrderedDict()
|
|
inlinePatterns["backtick"] = BacktickPattern(BACKTICK_RE)
|
|
inlinePatterns["escape"] = EscapePattern(ESCAPE_RE, md_instance)
|
|
inlinePatterns["reference"] = ReferencePattern(REFERENCE_RE, md_instance)
|
|
inlinePatterns["link"] = LinkPattern(LINK_RE, md_instance)
|
|
inlinePatterns["image_link"] = ImagePattern(IMAGE_LINK_RE, md_instance)
|
|
inlinePatterns["image_reference"] = ImageReferencePattern(
|
|
IMAGE_REFERENCE_RE, md_instance
|
|
)
|
|
inlinePatterns["short_reference"] = ReferencePattern(
|
|
SHORT_REF_RE, md_instance
|
|
)
|
|
inlinePatterns["autolink"] = AutolinkPattern(AUTOLINK_RE, md_instance)
|
|
inlinePatterns["automail"] = AutomailPattern(AUTOMAIL_RE, md_instance)
|
|
inlinePatterns["linebreak"] = SubstituteTagPattern(LINE_BREAK_RE, 'br')
|
|
if md_instance.safeMode != 'escape':
|
|
inlinePatterns["html"] = HtmlPattern(HTML_RE, md_instance)
|
|
inlinePatterns["entity"] = HtmlPattern(ENTITY_RE, md_instance)
|
|
inlinePatterns["not_strong"] = SimpleTextPattern(NOT_STRONG_RE)
|
|
inlinePatterns["em_strong"] = DoubleTagPattern(EM_STRONG_RE, 'strong,em')
|
|
inlinePatterns["strong_em"] = DoubleTagPattern(STRONG_EM_RE, 'em,strong')
|
|
inlinePatterns["strong"] = SimpleTagPattern(STRONG_RE, 'strong')
|
|
inlinePatterns["emphasis"] = SimpleTagPattern(EMPHASIS_RE, 'em')
|
|
if md_instance.smart_emphasis:
|
|
inlinePatterns["emphasis2"] = SimpleTagPattern(SMART_EMPHASIS_RE, 'em')
|
|
else:
|
|
inlinePatterns["emphasis2"] = SimpleTagPattern(EMPHASIS_2_RE, 'em')
|
|
return inlinePatterns
|
|
|
|
|
|
"""
|
|
The actual regular expressions for patterns
|
|
-----------------------------------------------------------------------------
|
|
"""
|
|
|
|
NOBRACKET = r'[^\]\[]*'
|
|
BRK = (
|
|
r'\[(' +
|
|
(NOBRACKET + r'(\[')*6 +
|
|
(NOBRACKET + r'\])*')*6 +
|
|
NOBRACKET + r')\]'
|
|
)
|
|
NOIMG = r'(?<!\!)'
|
|
|
|
# `e=f()` or ``e=f("`")``
|
|
BACKTICK_RE = r'(?:(?<!\\)((?:\\{2})+)(?=`+)|(?<!\\)(`+)(.+?)(?<!`)\3(?!`))'
|
|
|
|
# \<
|
|
ESCAPE_RE = r'\\(.)'
|
|
|
|
# *emphasis*
|
|
EMPHASIS_RE = r'(\*)([^\*]+)\2'
|
|
|
|
# **strong**
|
|
STRONG_RE = r'(\*{2}|_{2})(.+?)\2'
|
|
|
|
# ***strongem*** or ***em*strong**
|
|
EM_STRONG_RE = r'(\*|_)\2{2}(.+?)\2(.*?)\2{2}'
|
|
|
|
# ***strong**em*
|
|
STRONG_EM_RE = r'(\*|_)\2{2}(.+?)\2{2}(.*?)\2'
|
|
|
|
# _smart_emphasis_
|
|
SMART_EMPHASIS_RE = r'(?<!\w)(_)(?!_)(.+?)(?<!_)\2(?!\w)'
|
|
|
|
# _emphasis_
|
|
EMPHASIS_2_RE = r'(_)(.+?)\2'
|
|
|
|
# [text](url) or [text](<url>) or [text](url "title")
|
|
LINK_RE = NOIMG + BRK + \
|
|
r'''\(\s*(<.*?>|((?:(?:\(.*?\))|[^\(\)]))*?)\s*((['"])(.*?)\12\s*)?\)'''
|
|
|
|
#  or 
|
|
IMAGE_LINK_RE = r'\!' + BRK + r'\s*\(\s*(<.*?>|([^"\)\s]+\s*"[^"]*"|[^\)\s]*))\s*\)'
|
|
|
|
# [Google][3]
|
|
REFERENCE_RE = NOIMG + BRK + r'\s?\[([^\]]*)\]'
|
|
|
|
# [Google]
|
|
SHORT_REF_RE = NOIMG + r'\[([^\]]+)\]'
|
|
|
|
# ![alt text][2]
|
|
IMAGE_REFERENCE_RE = r'\!' + BRK + r'\s?\[([^\]]*)\]'
|
|
|
|
# stand-alone * or _
|
|
NOT_STRONG_RE = r'((^| )(\*|_)( |$))'
|
|
|
|
# <http://www.123.com>
|
|
AUTOLINK_RE = r'<((?:[Ff]|[Hh][Tt])[Tt][Pp][Ss]?://[^>]*)>'
|
|
|
|
# <me@example.com>
|
|
AUTOMAIL_RE = r'<([^> \!]*@[^> ]*)>'
|
|
|
|
# <...>
|
|
HTML_RE = r'(\<([a-zA-Z/][^\>]*?|\!--.*?--)\>)'
|
|
|
|
# &
|
|
ENTITY_RE = r'(&[\#a-zA-Z0-9]*;)'
|
|
|
|
# two spaces at end of line
|
|
LINE_BREAK_RE = r' \n'
|
|
|
|
|
|
def dequote(string):
|
|
"""Remove quotes from around a string."""
|
|
if ((string.startswith('"') and string.endswith('"')) or
|
|
(string.startswith("'") and string.endswith("'"))):
|
|
return string[1:-1]
|
|
else:
|
|
return string
|
|
|
|
|
|
ATTR_RE = re.compile(r"\{@([^\}]*)=([^\}]*)}") # {@id=123}
|
|
|
|
|
|
def handleAttributes(text, parent):
|
|
"""Set values of an element based on attribute definitions ({@id=123})."""
|
|
def attributeCallback(match):
|
|
parent.set(match.group(1), match.group(2).replace('\n', ' '))
|
|
return ATTR_RE.sub(attributeCallback, text)
|
|
|
|
|
|
"""
|
|
The pattern classes
|
|
-----------------------------------------------------------------------------
|
|
"""
|
|
|
|
|
|
class Pattern(object):
|
|
"""Base class that inline patterns subclass. """
|
|
|
|
ANCESTOR_EXCLUDES = tuple()
|
|
|
|
def __init__(self, pattern, markdown_instance=None):
|
|
"""
|
|
Create an instant of an inline pattern.
|
|
|
|
Keyword arguments:
|
|
|
|
* pattern: A regular expression that matches a pattern
|
|
|
|
"""
|
|
self.pattern = pattern
|
|
self.compiled_re = re.compile(r"^(.*?)%s(.*)$" % pattern,
|
|
re.DOTALL | re.UNICODE)
|
|
|
|
# Api for Markdown to pass safe_mode into instance
|
|
self.safe_mode = False
|
|
if markdown_instance:
|
|
self.markdown = markdown_instance
|
|
|
|
def getCompiledRegExp(self):
|
|
""" Return a compiled regular expression. """
|
|
return self.compiled_re
|
|
|
|
def handleMatch(self, m):
|
|
"""Return a ElementTree element from the given match.
|
|
|
|
Subclasses should override this method.
|
|
|
|
Keyword arguments:
|
|
|
|
* m: A re match object containing a match of the pattern.
|
|
|
|
"""
|
|
pass # pragma: no cover
|
|
|
|
def type(self):
|
|
""" Return class name, to define pattern type """
|
|
return self.__class__.__name__
|
|
|
|
def unescape(self, text):
|
|
""" Return unescaped text given text with an inline placeholder. """
|
|
try:
|
|
stash = self.markdown.treeprocessors['inline'].stashed_nodes
|
|
except KeyError: # pragma: no cover
|
|
return text
|
|
|
|
def itertext(el): # pragma: no cover
|
|
' Reimplement Element.itertext for older python versions '
|
|
tag = el.tag
|
|
if not isinstance(tag, util.string_type) and tag is not None:
|
|
return
|
|
if el.text:
|
|
yield el.text
|
|
for e in el:
|
|
for s in itertext(e):
|
|
yield s
|
|
if e.tail:
|
|
yield e.tail
|
|
|
|
def get_stash(m):
|
|
id = m.group(1)
|
|
if id in stash:
|
|
value = stash.get(id)
|
|
if isinstance(value, util.string_type):
|
|
return value
|
|
else:
|
|
# An etree Element - return text content only
|
|
return ''.join(itertext(value))
|
|
return util.INLINE_PLACEHOLDER_RE.sub(get_stash, text)
|
|
|
|
|
|
class SimpleTextPattern(Pattern):
|
|
""" Return a simple text of group(2) of a Pattern. """
|
|
def handleMatch(self, m):
|
|
return m.group(2)
|
|
|
|
|
|
class EscapePattern(Pattern):
|
|
""" Return an escaped character. """
|
|
|
|
def handleMatch(self, m):
|
|
char = m.group(2)
|
|
if char in self.markdown.ESCAPED_CHARS:
|
|
return '%s%s%s' % (util.STX, ord(char), util.ETX)
|
|
else:
|
|
return None
|
|
|
|
|
|
class SimpleTagPattern(Pattern):
|
|
"""
|
|
Return element of type `tag` with a text attribute of group(3)
|
|
of a Pattern.
|
|
|
|
"""
|
|
def __init__(self, pattern, tag):
|
|
Pattern.__init__(self, pattern)
|
|
self.tag = tag
|
|
|
|
def handleMatch(self, m):
|
|
el = util.etree.Element(self.tag)
|
|
el.text = m.group(3)
|
|
return el
|
|
|
|
|
|
class SubstituteTagPattern(SimpleTagPattern):
|
|
""" Return an element of type `tag` with no children. """
|
|
def handleMatch(self, m):
|
|
return util.etree.Element(self.tag)
|
|
|
|
|
|
class BacktickPattern(Pattern):
|
|
""" Return a `<code>` element containing the matching text. """
|
|
def __init__(self, pattern):
|
|
Pattern.__init__(self, pattern)
|
|
self.ESCAPED_BSLASH = '%s%s%s' % (util.STX, ord('\\'), util.ETX)
|
|
self.tag = 'code'
|
|
|
|
def handleMatch(self, m):
|
|
if m.group(4):
|
|
el = util.etree.Element(self.tag)
|
|
el.text = util.AtomicString(m.group(4).strip())
|
|
return el
|
|
else:
|
|
return m.group(2).replace('\\\\', self.ESCAPED_BSLASH)
|
|
|
|
|
|
class DoubleTagPattern(SimpleTagPattern):
|
|
"""Return a ElementTree element nested in tag2 nested in tag1.
|
|
|
|
Useful for strong emphasis etc.
|
|
|
|
"""
|
|
def handleMatch(self, m):
|
|
tag1, tag2 = self.tag.split(",")
|
|
el1 = util.etree.Element(tag1)
|
|
el2 = util.etree.SubElement(el1, tag2)
|
|
el2.text = m.group(3)
|
|
if len(m.groups()) == 5:
|
|
el2.tail = m.group(4)
|
|
return el1
|
|
|
|
|
|
class HtmlPattern(Pattern):
|
|
""" Store raw inline html and return a placeholder. """
|
|
def handleMatch(self, m):
|
|
rawhtml = self.unescape(m.group(2))
|
|
place_holder = self.markdown.htmlStash.store(rawhtml)
|
|
return place_holder
|
|
|
|
def unescape(self, text):
|
|
""" Return unescaped text given text with an inline placeholder. """
|
|
try:
|
|
stash = self.markdown.treeprocessors['inline'].stashed_nodes
|
|
except KeyError: # pragma: no cover
|
|
return text
|
|
|
|
def get_stash(m):
|
|
id = m.group(1)
|
|
value = stash.get(id)
|
|
if value is not None:
|
|
try:
|
|
return self.markdown.serializer(value)
|
|
except Exception:
|
|
return r'\%s' % value
|
|
|
|
return util.INLINE_PLACEHOLDER_RE.sub(get_stash, text)
|
|
|
|
|
|
class LinkPattern(Pattern):
|
|
""" Return a link element from the given match. """
|
|
def handleMatch(self, m):
|
|
el = util.etree.Element("a")
|
|
el.text = m.group(2)
|
|
title = m.group(13)
|
|
href = m.group(9)
|
|
|
|
if href:
|
|
if href[0] == "<":
|
|
href = href[1:-1]
|
|
el.set("href", self.sanitize_url(self.unescape(href.strip())))
|
|
else:
|
|
el.set("href", "")
|
|
|
|
if title:
|
|
title = dequote(self.unescape(title))
|
|
el.set("title", title)
|
|
return el
|
|
|
|
def sanitize_url(self, url):
|
|
"""
|
|
Sanitize a url against xss attacks in "safe_mode".
|
|
|
|
Rather than specifically blacklisting `javascript:alert("XSS")` and all
|
|
its aliases (see <http://ha.ckers.org/xss.html>), we whitelist known
|
|
safe url formats. Most urls contain a network location, however some
|
|
are known not to (i.e.: mailto links). Script urls do not contain a
|
|
location. Additionally, for `javascript:...`, the scheme would be
|
|
"javascript" but some aliases will appear to `urlparse()` to have no
|
|
scheme. On top of that relative links (i.e.: "foo/bar.html") have no
|
|
scheme. Therefore we must check "path", "parameters", "query" and
|
|
"fragment" for any literal colons. We don't check "scheme" for colons
|
|
because it *should* never have any and "netloc" must allow the form:
|
|
`username:password@host:port`.
|
|
|
|
"""
|
|
if not self.markdown.safeMode:
|
|
# Return immediately bipassing parsing.
|
|
return url
|
|
|
|
try:
|
|
scheme, netloc, path, params, query, fragment = url = urlparse(url)
|
|
except ValueError: # pragma: no cover
|
|
# Bad url - so bad it couldn't be parsed.
|
|
return ''
|
|
|
|
locless_schemes = ['', 'mailto', 'news']
|
|
allowed_schemes = locless_schemes + ['http', 'https', 'ftp', 'ftps']
|
|
if scheme not in allowed_schemes:
|
|
# Not a known (allowed) scheme. Not safe.
|
|
return ''
|
|
|
|
if netloc == '' and scheme not in locless_schemes: # pragma: no cover
|
|
# This should not happen. Treat as suspect.
|
|
return ''
|
|
|
|
for part in url[2:]:
|
|
if ":" in part:
|
|
# A colon in "path", "parameters", "query"
|
|
# or "fragment" is suspect.
|
|
return ''
|
|
|
|
# Url passes all tests. Return url as-is.
|
|
return urlunparse(url)
|
|
|
|
|
|
class ImagePattern(LinkPattern):
|
|
""" Return a img element from the given match. """
|
|
def handleMatch(self, m):
|
|
el = util.etree.Element("img")
|
|
src_parts = m.group(9).split()
|
|
if src_parts:
|
|
src = src_parts[0]
|
|
if src[0] == "<" and src[-1] == ">":
|
|
src = src[1:-1]
|
|
el.set('src', self.sanitize_url(self.unescape(src)))
|
|
else:
|
|
el.set('src', "")
|
|
if len(src_parts) > 1:
|
|
el.set('title', dequote(self.unescape(" ".join(src_parts[1:]))))
|
|
|
|
if self.markdown.enable_attributes:
|
|
truealt = handleAttributes(m.group(2), el)
|
|
else:
|
|
truealt = m.group(2)
|
|
|
|
el.set('alt', self.unescape(truealt))
|
|
return el
|
|
|
|
|
|
class ReferencePattern(LinkPattern):
|
|
""" Match to a stored reference and return link element. """
|
|
|
|
NEWLINE_CLEANUP_RE = re.compile(r'[ ]?\n', re.MULTILINE)
|
|
|
|
def handleMatch(self, m):
|
|
try:
|
|
id = m.group(9).lower()
|
|
except IndexError:
|
|
id = None
|
|
if not id:
|
|
# if we got something like "[Google][]" or "[Google]"
|
|
# we'll use "google" as the id
|
|
id = m.group(2).lower()
|
|
|
|
# Clean up linebreaks in id
|
|
id = self.NEWLINE_CLEANUP_RE.sub(' ', id)
|
|
if id not in self.markdown.references: # ignore undefined refs
|
|
return None
|
|
href, title = self.markdown.references[id]
|
|
|
|
text = m.group(2)
|
|
return self.makeTag(href, title, text)
|
|
|
|
def makeTag(self, href, title, text):
|
|
el = util.etree.Element('a')
|
|
|
|
el.set('href', self.sanitize_url(href))
|
|
if title:
|
|
el.set('title', title)
|
|
|
|
el.text = text
|
|
return el
|
|
|
|
|
|
class ImageReferencePattern(ReferencePattern):
|
|
""" Match to a stored reference and return img element. """
|
|
def makeTag(self, href, title, text):
|
|
el = util.etree.Element("img")
|
|
el.set("src", self.sanitize_url(href))
|
|
if title:
|
|
el.set("title", title)
|
|
|
|
if self.markdown.enable_attributes:
|
|
text = handleAttributes(text, el)
|
|
|
|
el.set("alt", self.unescape(text))
|
|
return el
|
|
|
|
|
|
class AutolinkPattern(Pattern):
|
|
""" Return a link Element given an autolink (`<http://example/com>`). """
|
|
def handleMatch(self, m):
|
|
el = util.etree.Element("a")
|
|
el.set('href', self.unescape(m.group(2)))
|
|
el.text = util.AtomicString(m.group(2))
|
|
return el
|
|
|
|
|
|
class AutomailPattern(Pattern):
|
|
"""
|
|
Return a mailto link Element given an automail link (`<foo@example.com>`).
|
|
"""
|
|
def handleMatch(self, m):
|
|
el = util.etree.Element('a')
|
|
email = self.unescape(m.group(2))
|
|
if email.startswith("mailto:"):
|
|
email = email[len("mailto:"):]
|
|
|
|
def codepoint2name(code):
|
|
"""Return entity definition by code, or the code if not defined."""
|
|
entity = entities.codepoint2name.get(code)
|
|
if entity:
|
|
return "%s%s;" % (util.AMP_SUBSTITUTE, entity)
|
|
else:
|
|
return "%s#%d;" % (util.AMP_SUBSTITUTE, code)
|
|
|
|
letters = [codepoint2name(ord(letter)) for letter in email]
|
|
el.text = util.AtomicString(''.join(letters))
|
|
|
|
mailto = "mailto:" + email
|
|
mailto = "".join([util.AMP_SUBSTITUTE + '#%d;' %
|
|
ord(letter) for letter in mailto])
|
|
el.set('href', mailto)
|
|
return el
|