Skip to main content

Simple regex-based lexer/parser for inline markup

Project description

Simple regex-based lexer/parser for inline markup


  • Python 3



import re
from pprint import pprint
from reparser import Parser, Token, MatchGroup

boundary_chars = r'\s`!()\[\]{{}};:\'".,<>?«»“”‘’*_~='
b_left = r'(?:(?<=[' + boundary_chars + r'])|(?<=^))'  # Lookbehind
b_right = r'(?:(?=[' + boundary_chars + r'])|(?=$))'   # Lookahead

markdown_start = b_left + r'(?<!\\){tag}(?!\s)(?!{tag})'
markdown_end = r'(?<!{tag})(?<!\s)(?<!\\){tag}' + b_right
markdown_link = r'(?<!\\)\[(?P<link>.+?)\]\((?P<url>.+?)\)'
newline = r'\n|\r\n'

url_proto_regex = re.compile(r'(?i)^[a-z][\w-]+:/{1,3}')

def markdown(tag):
    """Return sequence of start and end regex patterns for simple Markdown tag"""
    return (markdown_start.format(tag=tag), markdown_end.format(tag=tag))

def url_complete(url):
    """If URL doesn't start with protocol, prepend it with http://"""
    return url if else 'http://' + url

tokens = [
    Token('bi1',  *markdown(r'\*\*\*'), is_bold=True, is_italic=True),
    Token('bi2',  *markdown(r'___'),    is_bold=True, is_italic=True),
    Token('b1',   *markdown(r'\*\*'),   is_bold=True),
    Token('b2',   *markdown(r'__'),     is_bold=True),
    Token('i1',   *markdown(r'\*'),     is_italic=True),
    Token('i2',   *markdown(r'_'),      is_italic=True),
    Token('pre3', *markdown(r'```'),    skip=True),
    Token('pre2', *markdown(r'``'),     skip=True),
    Token('pre1', *markdown(r'`'),      skip=True),
    Token('s',    *markdown(r'~~'),     is_strikethrough=True),
    Token('u',    *markdown(r'=='),     is_underline=True),
    Token('link', markdown_link, text=MatchGroup('link'),
          link_target=MatchGroup('url', func=url_complete)),
    Token('br', newline, text='\n', segment_type="LINE_BREAK")

parser = Parser(tokens)
text = ('Hello **bold** world!\n'
        'You can **try *this* awesome** [link](')

segments = parser.parse(text)
pprint([(segment.text, segment.params) for segment in segments])


[('Hello ', {}),
 ('bold', {'is_bold': True}),
 (' world!', {}),
 ('\n', {'segment_type': 'LINE_BREAK'}),
 ('You can ', {}),
 ('try ', {'is_bold': True}),
 ('this', {'is_bold': True, 'is_italic': True}),
 (' awesome', {'is_bold': True}),
 (' ', {}),
 ('link', {'link_target': ''}),
 ('.', {})]

Project details

Download files

Download the file for your platform. If you're not sure which to choose, learn more about installing packages.

Source Distribution

ReParser-1.4.3.tar.gz (4.6 kB view hashes)

Uploaded source

Supported by

AWS AWS Cloud computing and Security Sponsor Datadog Datadog Monitoring Fastly Fastly CDN Google Google Download Analytics Microsoft Microsoft PSF Sponsor Pingdom Pingdom Monitoring Sentry Sentry Error logging StatusPage StatusPage Status page