Google

Edit File: ylex.py

# lexer for yacc-grammars
#
# Author: David Beazley (dave@dabeaz.com)
# Date  : October 2, 2006

import sys
sys.path.append("../..")

from ply import *

tokens = (
    'LITERAL','SECTION','TOKEN','LEFT','RIGHT','PREC','START','TYPE','NONASSOC','UNION','CODE',
    'ID','QLITERAL','NUMBER',
)

states = (('code','exclusive'),)

literals = [ ';', ',', '<', '>', '|',':' ]
t_ignore = ' \t'

t_TOKEN     = r'%token'
t_LEFT      = r'%left'
t_RIGHT     = r'%right'
t_NONASSOC  = r'%nonassoc'
t_PREC      = r'%prec'
t_START     = r'%start'
t_TYPE      = r'%type'
t_UNION     = r'%union'
t_ID        = r'[a-zA-Z_][a-zA-Z_0-9]*'
t_QLITERAL  = r'''(?P<quote>['"]).*?(?P=quote)'''
t_NUMBER    = r'\d+'

def t_SECTION(t):
    r'%%'
    if getattr(t.lexer,"lastsection",0):
         t.value = t.lexer.lexdata[t.lexpos+2:]
         t.lexer.lexpos = len(t.lexer.lexdata)
    else:
         t.lexer.lastsection = 0
    return t

# Comments
def t_ccomment(t):
    r'/\*(.|\n)*?\*/'
    t.lexer.lineno += t.value.count('\n')

t_ignore_cppcomment = r'//.*'

def t_LITERAL(t):
   r'%\{(.|\n)*?%\}'
   t.lexer.lineno += t.value.count("\n")
   return t

def t_NEWLINE(t):
   r'\n'
   t.lexer.lineno += 1

def t_code(t):
   r'\{'
   t.lexer.codestart = t.lexpos
   t.lexer.level = 1
   t.lexer.begin('code')

def t_code_ignore_string(t):
    r'\"([^\\\n]|(\\.))*?\"'

def t_code_ignore_char(t):
    r'\'([^\\\n]|(\\.))*?\''

def t_code_ignore_comment(t):
   r'/\*(.|\n)*?\*/'

def t_code_ignore_cppcom(t):
   r'//.*'

def t_code_lbrace(t):
    r'\{'
    t.lexer.level += 1

def t_code_rbrace(t):
    r'\}'
    t.lexer.level -= 1
    if t.lexer.level == 0:
         t.type = 'CODE'
         t.value = t.lexer.lexdata[t.lexer.codestart:t.lexpos+1]
         t.lexer.begin('INITIAL')
         t.lexer.lineno += t.value.count('\n')
         return t

t_code_ignore_nonspace   = r'[^\s\}\'\"\{]+'
t_code_ignore_whitespace = r'\s+'
t_code_ignore = ""

def t_code_error(t):
    raise RuntimeError

def t_error(t):
    print "%d: Illegal character '%s'" % (t.lexer.lineno, t.value[0])
    print t.value
    t.lexer.skip(1)

lex.lex()

if __name__ == '__main__':
    lex.runmain()