path: root/debian/pyrex/pyrex-0.9.9/Pyrex/Compiler/Parsing.py

#
#   Pyrex Parser
#

import os, re
from string import join, replace
from types import ListType, TupleType
from Scanning import PyrexScanner
import Nodes
import ExprNodes
from ModuleNode import ModuleNode
from Errors import warning, error, InternalError


class Ctx(object):
    #  Parsing context
    level = 'other'
    visibility = 'private'
    extern_from = False
    cdef_flag = 0
    cplus_flag = 0
    typedef_flag = 0
    api = 0
    nogil = 0
    
    def __init__(self, **kwds):
        self.__dict__.update(kwds)
    
    def __call__(self, **kwds):
        ctx = Ctx()
        d = ctx.__dict__
        d.update(self.__dict__)
        d.update(kwds)
        return ctx

    def cplus_check(self, pos):
        #if self.visibility <> 'extern':
        #	error(pos, "C++ declarations must be 'extern'")
        if self.cplus_flag and not self.extern_from:
            error(pos, "C++ declarations must be in an 'extern from' block")


def p_ident(s, message = "Expected an identifier"):
    if s.sy == 'IDENT':
        name = s.systring
        s.next()
        return name
    else:
        s.error(message)

def p_ident_list(s):
    names = []
    while s.sy == 'IDENT':
        names.append(s.systring)
        s.next()
        if s.sy <> ',':
            break
        s.next()
    return names

#------------------------------------------
#
#   Expressions
#
#------------------------------------------

def p_binop_expr(s, ops, p_sub_expr):
    n1 = p_sub_expr(s)
    while s.sy in ops:
        op = s.sy
        pos = s.position()
        s.next()
        n2 = p_sub_expr(s)
        n1 = ExprNodes.binop_node(pos, op, n1, n2)
    return n1

#test: and_test ('or' and_test)* | lambdef

def p_simple_expr(s):
    return p_rassoc_binop_expr(s, ('or',), p_and_test)

def p_rassoc_binop_expr(s, ops, p_subexpr):
    n1 = p_subexpr(s)
    if s.sy in ops:
        pos = s.position()
        op = s.sy
        s.next()
        n2 = p_rassoc_binop_expr(s, ops, p_subexpr)
        n1 = ExprNodes.binop_node(pos, op, n1, n2)
    return n1

#and_test: not_test ('and' not_test)*

def p_and_test(s):
    #return p_binop_expr(s, ('and',), p_not_test)
    return p_rassoc_binop_expr(s, ('and',), p_not_test)

#not_test: 'not' not_test | comparison

def p_not_test(s):
    if s.sy == 'not':
        pos = s.position()
        s.next()
        return ExprNodes.NotNode(pos, operand = p_not_test(s))
    else:
        return p_comparison(s)

#comparison: expr (comp_op expr)*
#comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'

def p_comparison(s):
    n1 = p_bit_expr(s)
    if s.sy in comparison_ops:
        pos = s.position()
        op = p_cmp_op(s)
        n2 = p_bit_expr(s)
        n1 = ExprNodes.PrimaryCmpNode(pos, 
            operator = op, operand1 = n1, operand2 = n2)
        if s.sy in comparison_ops:
            n1.cascade = p_cascaded_cmp(s)
    return n1

def p_cascaded_cmp(s):
    pos = s.position()
    op = p_cmp_op(s)
    n2 = p_bit_expr(s)
    result = ExprNodes.CascadedCmpNode(pos, 
        operator = op, operand2 = n2)
    if s.sy in comparison_ops:
        result.cascade = p_cascaded_cmp(s)
    return result

def p_cmp_op(s):
    if s.sy == 'not':
        s.next()
        s.expect('in')
        op = 'not_in'
    elif s.sy == 'is':
        s.next()
        if s.sy == 'not':
            s.next()
            op = 'is_not'
        else:
            op = 'is'
    else:
        op = s.sy
        s.next()
    if op == '<>':
        op = '!='
    return op
    
comparison_ops = (
    '<', '>', '==', '>=', '<=', '<>', '!=', 
    'in', 'is', 'not'
)

#expr: xor_expr ('|' xor_expr)*

def p_bit_expr(s):
    return p_binop_expr(s, ('|',), p_xor_expr)

#xor_expr: and_expr ('^' and_expr)*

def p_xor_expr(s):
    return p_binop_expr(s, ('^',), p_and_expr)

#and_expr: shift_expr ('&' shift_expr)*

def p_and_expr(s):
    return p_binop_expr(s, ('&',), p_shift_expr)

#shift_expr: arith_expr (('<<'|'>>') arith_expr)*

def p_shift_expr(s):
    return p_binop_expr(s, ('<<', '>>'), p_arith_expr)

#arith_expr: term (('+'|'-') term)*

def p_arith_expr(s):
    return p_binop_expr(s, ('+', '-'), p_term)

#term: factor (('*'|'/'|'%') factor)*

def p_term(s):
    return p_binop_expr(s, ('*', '/', '%'), p_factor)

#factor: ('+'|'-'|'~'|'&'|typecast|sizeof) factor | power

def p_factor(s):
    sy = s.sy
    if sy in ('+', '-', '~'):
        op = s.sy
        pos = s.position()
        s.next()
        return ExprNodes.unop_node(pos, op, p_factor(s))
    elif sy == '&':
        pos = s.position()
        s.next()
        arg = p_factor(s)
        return ExprNodes.AmpersandNode(pos, operand = arg)
    elif sy == "<":
        return p_typecast(s)
    elif sy == 'IDENT' and s.systring == "sizeof":
        return p_sizeof(s)
    else:
        return p_power(s)

def p_typecast(s):
    # s.sy == "<"
    pos = s.position()
    s.next()
    base_type = p_c_base_type(s)
    declarator = p_c_declarator(s, empty = 1)
    s.expect(">")
    operand = p_factor(s)
    return ExprNodes.TypecastNode(pos, 
        base_type = base_type, 
        declarator = declarator,
        operand = operand)

def p_sizeof(s):
    # s.sy == ident "sizeof"
    pos = s.position()
    s.next()
    s.expect('(')
    if looking_at_type(s):
        base_type = p_c_base_type(s)
        declarator = p_c_declarator(s, empty = 1)
        node = ExprNodes.SizeofTypeNode(pos, 
            base_type = base_type, declarator = declarator)
    else:
        operand = p_simple_expr(s)
        node = ExprNodes.SizeofVarNode(pos, operand = operand)
    s.expect(')')
    return node

#power: atom trailer* ('**' factor)*

def p_power(s):
    n1 = p_primitive(s)
    if s.sy == '**':
        pos = s.position()
        s.next()
        n2 = p_factor(s)
        n1 = ExprNodes.binop_node(pos, '**', n1, n2)
    return n1

def p_primitive(s):
    n = p_atom(s)
    while s.sy in ('(', '[', '.'):
        n = p_trailer(s, n)
    return n

#trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME

def p_trailer(s, node1):
    pos = s.position()
    if s.sy == '(':
        return p_call(s, node1)
    elif s.sy == '[':
        return p_index(s, node1)
    else: # s.sy == '.'
        s.next()
        name = p_ident(s)
        return ExprNodes.AttributeNode(pos, 
            obj = node1, attribute = name)

# arglist:  argument (',' argument)* [',']
# argument: [test '='] test       # Really [keyword '='] test

def p_call(s, function):
    # s.sy == '('
    pos = s.position()
    s.next()
    positional_args = []
    keyword_args = []
    star_arg = None
    starstar_arg = None
    while s.sy not in ('*', '**', ')'):
        arg = p_simple_expr(s)
        if s.sy == '=':
            s.next()
            if not arg.is_name:
                s.error("Expected an identifier before '='",
                    pos = arg.pos)
            keyword = ExprNodes.StringNode(arg.pos, 
                value = arg.name)
            arg = p_simple_expr(s)
            keyword_args.append((keyword, arg))
        else:
            if keyword_args:
                s.error("Non-keyword arg following keyword arg",
                    pos = arg.pos)
            positional_args.append(arg)
        if s.sy <> ',':
            break
        s.next()
    if s.sy == '*':
        s.next()
        star_arg = p_simple_expr(s)
        if s.sy == ',':
            s.next()
    if s.sy == '**':
        s.next()
        starstar_arg = p_simple_expr(s)
        if s.sy == ',':
            s.next()
    s.expect(')')
    if not (keyword_args or star_arg or starstar_arg):
        return ExprNodes.SimpleCallNode(pos,
            function = function,
            args = positional_args)
    else:
        arg_tuple = None
        keyword_dict = None
        if positional_args or not star_arg:
            arg_tuple = ExprNodes.TupleNode(pos, 
                args = positional_args)
        if star_arg:
            star_arg_tuple = ExprNodes.AsTupleNode(pos, arg = star_arg)
            if arg_tuple:
                arg_tuple = ExprNodes.binop_node(pos, 
                    operator = '+', operand1 = arg_tuple,
                    operand2 = star_arg_tuple)
            else:
                arg_tuple = star_arg_tuple
        if keyword_args:
            keyword_dict = ExprNodes.DictNode(pos,
                key_value_pairs = keyword_args)
        return ExprNodes.GeneralCallNode(pos, 
            function = function,
            positional_args = arg_tuple,
            keyword_args = keyword_dict,
            starstar_arg = starstar_arg)

#lambdef: 'lambda' [varargslist] ':' test

#subscriptlist: subscript (',' subscript)* [',']

def p_index(s, base):
    # s.sy == '['
    pos = s.position()
    s.next()
    subscripts = p_subscript_list(s)
    if len(subscripts) == 1 and len(subscripts[0]) == 2:
        start, stop = subscripts[0]
        result = ExprNodes.SliceIndexNode(pos, 
            base = base, start = start, stop = stop)
    else:
        indexes = make_slice_nodes(pos, subscripts)
        if len(indexes) == 1:
            index = indexes[0]
        else:
            index = ExprNodes.TupleNode(pos, args = indexes)
        result = ExprNodes.IndexNode(pos,
            base = base, index = index)
    s.expect(']')
    return result

def p_subscript_list(s):
    items = [p_subscript(s)]
    while s.sy == ',':
        s.next()
        if s.sy == ']':
            break
        items.append(p_subscript(s))
    return items

#subscript: '.' '.' '.' | test | [test] ':' [test] [':' [test]]

def p_subscript(s):
    # Parse a subscript and return a list of
    # 1, 2 or 3 ExprNodes, depending on how
    # many slice elements were encountered.
    pos = s.position()
    if s.sy == '.':
        expect_ellipsis(s)
        return [ExprNodes.EllipsisNode(pos)]
    else:
        start = p_slice_element(s, (':',))
        if s.sy <> ':':
            return [start]
        s.next()
        stop = p_slice_element(s, (':', ',', ']'))
        if s.sy <> ':':
            return [start, stop]
        s.next()
        step = p_slice_element(s, (':', ',', ']'))
        return [start, stop, step]

def p_slice_element(s, follow_set):
    # Simple expression which may be missing iff
    # it is followed by something in follow_set.
    if s.sy not in follow_set:
        return p_simple_expr(s)
    else:
        return None

def expect_ellipsis(s):
    s.expect('.')
    s.expect('.')
    s.expect('.')

def make_slice_nodes(pos, subscripts):
    # Convert a list of subscripts as returned
    # by p_subscript_list into a list of ExprNodes,
    # creating SliceNodes for elements with 2 or
    # more components.
    result = []
    for subscript in subscripts:
        if len(subscript) == 1:
            result.append(subscript[0])
        else:
            result.append(make_slice_node(pos, *subscript))
    return result

def make_slice_node(pos, start, stop = None, step = None):
    if not start:
        start = ExprNodes.NoneNode(pos)
    if not stop:
        stop = ExprNodes.NoneNode(pos)
    if not step:
        step = ExprNodes.NoneNode(pos)
    return ExprNodes.SliceNode(pos,
        start = start, stop = stop, step = step)

#atom: '(' [testlist] ')' | '[' [listmaker] ']' | '{' [dictmaker] '}' | '`' testlist '`' | NAME | NUMBER | STRING+

def p_atom(s):
    pos = s.position()
    sy = s.sy
    if sy == '(':
        s.next()
        if s.sy == ')':
            result = ExprNodes.TupleNode(pos, args = [])
        else:
            result = p_expr(s)
        s.expect(')')
        return result
    elif sy == '[':
        return p_list_maker(s)
    elif sy == '{':
        return p_dict_maker(s)
    elif sy == '`':
        return p_backquote_expr(s)
    elif sy == 'INT':
        value = s.systring
        s.next()
        return ExprNodes.IntNode(pos, value = value)
    elif sy == 'LONG':
        value = s.systring
        s.next()
        return ExprNodes.LongNode(pos, value = value)
    elif sy == 'FLOAT':
        value = s.systring
        s.next()
        return ExprNodes.FloatNode(pos, value = value)
    elif sy == 'IMAG':
        value = s.systring[:-1]
        s.next()
        return ExprNodes.ImagNode(pos, value = value)
    elif sy == 'STRING' or sy == 'BEGIN_STRING':
        kind, value = p_cat_string_literal(s)
        if kind == 'c':
            return ExprNodes.CharNode(pos, value = value)
        else:
            return ExprNodes.StringNode(pos, value = value)
    elif sy == 'IDENT':
        name = s.systring
        s.next()
        if name == "None":
            return ExprNodes.NoneNode(pos)
        elif name == "new" and s.sy == 'IDENT':
            return p_new_call(s)
        else:
            return p_name_atom(s, name)
    elif sy == 'NULL':
        s.next()
        return ExprNodes.NullNode(pos)
    else:
        s.error("Expected an identifier or literal")

def p_new_call(s):
    node = p_primitive(s)
    if isinstance(node, ExprNodes.SimpleCallNode):
        node.is_new = 1
    else:
        error(s.position(), "'new' must be followed by a C++ constructor call")
    return node

def p_name(s):
    if s.sy == 'IDENT':
        pos = s.position()
        name = s.systring
        s.next()
        return ExprNodes.NameNode(pos, name = name)
    else:
        s.error("Expected a variable name")

def p_name_atom(s, name):
    pos = s.position()
    if not s.compile_time_expr:
        try:
            value = s.compile_time_env.lookup_here(name)
        except KeyError:
            pass
        else:
            rep = repr(value)
            if isinstance(value, int):
                return ExprNodes.IntNode(pos, value = rep)
            elif isinstance(value, long):
                return ExprNodes.LongNode(pos, value = rep)
            elif isinstance(value, float):
                return ExprNodes.FloatNode(pos, value = rep)
            elif isinstance(value, str):
                return ExprNodes.StringNode(pos, value = rep[1:-1])
            else:
                error(pos, "Invalid type for compile-time constant: %s"
                    % value.__class__.__name__)
    return ExprNodes.NameNode(pos, name = name)

def p_cat_string_literal(s):
    # A sequence of one or more adjacent string literals.
    # Returns (kind, value) where kind in ('', 'c', 'r')
    kind, value = p_string_literal(s)
    if kind <> 'c':
        strings = [value]
        while s.sy == 'STRING' or s.sy == 'BEGIN_STRING':
            next_kind, next_value = p_string_literal(s)
            if next_kind == 'c':
                self.error(
                    "Cannot concatenate char literal with another string or char literal")
            strings.append(next_value)
        value = ''.join(strings)
    return kind, value

def p_opt_string_literal(s):
    if s.sy == 'STRING' or s.sy == 'BEGIN_STRING':
        return p_string_literal(s)
    else:
        return None

def p_string_literal(s):
    # A single string or char literal.
    # Returns (kind, value) where kind in ('', 'c', 'r')
    if s.sy == 'STRING':
        value = unquote(s.systring)
        s.next()
        return value
    # s.sy == 'BEGIN_STRING'
    pos = s.position()
    #is_raw = s.systring[:1].lower() == "r"
    kind = s.systring[:1].lower()
    if kind not in "cr":
        kind = ''
    chars = []
    while 1:
        s.next()
        sy = s.sy
        #print "p_string_literal: sy =", sy, repr(s.systring) ###
        if sy == 'CHARS':
            systr = s.systring
            if len(systr) == 1 and systr in "'\"\n":
                chars.append('\\')
            chars.append(systr)
        elif sy == 'ESCAPE':
            systr = s.systring
            if kind == 'r':
                if systr == '\\\n':
                    chars.append(r'\\\n')
                elif systr == r'\"':
                    chars.append(r'\\\"')
                elif systr == r'\\':
                    chars.append(r'\\\\')
                else:
                    chars.append('\\' + systr)
            else:
                c = systr[1]
                if c in "'\"\\abfnrtv01234567":
                    chars.append(systr)
                elif c == 'x':
                    chars.append('\\x0' + systr[2:])
                elif c == '\n':
                    pass
                else:
                    chars.append(r'\\' + systr[1:])
        elif sy == 'NEWLINE':
            chars.append(r'\n')
        elif sy == 'END_STRING':
            break
        elif sy == 'EOF':
            s.error("Unclosed string literal", pos = pos)
        else:
            s.error(
                "Unexpected token %r:%r in string literal" %
                    (sy, s.systring))
    s.next()
    value = join(chars, '')
    #print "p_string_literal: value =", repr(value) ###
    return kind, value

def unquote(s):
    is_raw = 0
    if s[:1].lower() == "r":
        is_raw = 1
        s = s[1:]
    q = s[:3]
    if q == '"""' or q == "'''":
        s = s[3:-3]
    else:
        s = s[1:-1]
    if is_raw:
        s = s.replace('\\', '\\\\')
        s = s.replace('\n', '\\\n')
    else:
        # Split into double quotes, newlines, escape sequences 
        # and spans of regular chars
        l1 = re.split(r'((?:\\[0-7]{1,3})|(?:\\x[0-9A-Fa-f]{2})|(?:\\.)|(?:\\\n)|(?:\n)|")', s)
        print "unquote: l1 =", l1 ###
        l2 = []
        for item in l1:
            if item == '"' or item == '\n':
                l2.append('\\' + item)
            elif item == '\\\n':
                pass
            elif item[:1] == '\\':
                if len(item) == 2:
                    if item[1] in '"\\abfnrtv':
                        l2.append(item)
                    else:
                        l2.append(item[1])
                elif item[1:2] == 'x':
                    l2.append('\\x0' + item[2:])
                else:
                    # octal escape
                    l2.append(item)
            else:
                l2.append(item)
        s = "".join(l2)
    return s
        
def p_list_maker(s):
    # s.sy == '['
    pos = s.position()
    s.next()
    exprs = p_simple_expr_list(s)
    s.expect(']')
    return ExprNodes.ListNode(pos, args = exprs)

#dictmaker: test ':' test (',' test ':' test)* [',']

def p_dict_maker(s):
    # s.sy == '{'
    pos = s.position()
    s.next()
    items = []
    while s.sy <> '}':
        key = p_simple_expr(s)
        s.expect(':')
        value = p_simple_expr(s)
        items.append((key, value))
        if s.sy <> ',':
            break
        s.next()
    s.expect('}')
    return ExprNodes.DictNode(pos, key_value_pairs = items)

def p_backquote_expr(s):
    # s.sy == '`'
    pos = s.position()
    s.next()
    arg = p_expr(s)
    s.expect('`')
    return ExprNodes.BackquoteNode(pos, arg = arg)

#testlist: test (',' test)* [',']

def p_simple_expr_list(s):
    exprs = []
    while s.sy not in expr_terminators:
        exprs.append(p_simple_expr(s))
        if s.sy <> ',':
            break
        s.next()
    return exprs

def p_expr(s):
    pos = s.position()
    expr = p_simple_expr(s)
    if s.sy == ',':
        s.next()
        exprs = [expr] + p_simple_expr_list(s)
        return ExprNodes.TupleNode(pos, args = exprs)
    else:
        return expr

expr_terminators = (')', ']', '}', ':', '=', 'NEWLINE')

#-------------------------------------------------------
#
#   Statements
#
#-------------------------------------------------------

def p_global_statement(s):
    # assume s.sy == 'global'
    pos = s.position()
    s.next()
    names = p_ident_list(s)
    return Nodes.GlobalNode(pos, names = names)

inplace_operators = ('+=', '-=', '*=', '/=', '%=', '**=',
    '<<=', '>>=', '&=', '^=', '|=')

def p_expression_or_assignment(s):
    pos = s.position()
    expr = p_expr(s)
    if s.sy in inplace_operators:
        return p_inplace_operation(s, expr)
    elif s.sy <> '=':
        if isinstance(expr, ExprNodes.StringNode):
            return Nodes.PassStatNode(expr.pos)
        else:
            return Nodes.ExprStatNode(expr.pos, expr = expr)
    else:
        expr_list = [expr]
        while s.sy == '=':
            s.next()
            expr_list.append(p_expr(s))
        expr_list_list = []
        flatten_parallel_assignments(expr_list, expr_list_list)
        nodes = []
        for expr_list in expr_list_list:
            lhs_list = expr_list[:-1]
            rhs = expr_list[-1]
            if len(lhs_list) == 1:
                node = Nodes.SingleAssignmentNode(rhs.pos, 
                    lhs = lhs_list[0], rhs = rhs)
            else:
                node = Nodes.CascadedAssignmentNode(rhs.pos,
                    lhs_list = lhs_list, rhs = rhs)
            nodes.append(node)
        if len(nodes) == 1:
            return nodes[0]
        else:
            return Nodes.ParallelAssignmentNode(nodes[0].pos, stats = nodes)

def p_inplace_operation(s, lhs):
    pos = s.position()
    op = s.sy
    s.next()
    rhs = p_expr(s)
    return Nodes.AugmentedAssignmentNode(pos, lhs = lhs, operator = op, rhs = rhs)

def flatten_parallel_assignments(input, output):
    #  The input is a list of expression nodes, representing 
    #  the LHSs and RHS of one (possibly cascaded) assignment 
    #  statement. If they are all sequence constructors with 
    #  the same number of arguments, rearranges them into a
    #  list of equivalent assignments between the individual 
    #  elements. This transformation is applied recursively.
    size = find_parallel_assignment_size(input)
    if size >= 0:
        for i in range(size):
            new_exprs = [expr.args[i] for expr in input]
            flatten_parallel_assignments(new_exprs, output)
    else:
        output.append(input)

def find_parallel_assignment_size(input):
    #  The input is a list of expression nodes. If 
    #  they are all sequence constructors with the same number
    #  of arguments, return that number, else return -1.
    #  Produces an error message if they are all sequence
    #  constructors but not all the same size.
    for expr in input:
        if not expr.is_sequence_constructor:
            return -1
    rhs = input[-1]
    rhs_size = len(rhs.args)
    for lhs in input[:-1]:
        lhs_size = len(lhs.args)
        if lhs_size <> rhs_size:
            error(lhs.pos, "Unpacking sequence of wrong size (expected %d, got %d)"
                % (lhs_size, rhs_size))
            return -1
    return rhs_size

def p_print_statement(s):
    # s.sy == 'print'
    pos = s.position()
    s.next()
    if s.sy == '>>':
        s.error("'print >>' not yet implemented")
    args = []
    ewc = 0
    if s.sy not in ('NEWLINE', 'EOF'):
        args.append(p_simple_expr(s))
        while s.sy == ',':
            s.next()
            if s.sy in ('NEWLINE', 'EOF'):
                ewc = 1
                break
            args.append(p_simple_expr(s))
    return Nodes.PrintStatNode(pos, 
        args = args, ends_with_comma = ewc)

def p_del_statement(s):
    # s.sy == 'del'
    pos = s.position()
    s.next()
    args = p_simple_expr_list(s)
    return Nodes.DelStatNode(pos, args = args)

def p_pass_statement(s, with_newline = 0):
    pos = s.position()
    s.expect('pass')
    if with_newline:
        s.expect_newline("Expected a newline")
    return Nodes.PassStatNode(pos)

def p_break_statement(s):
    # s.sy == 'break'
    pos = s.position()
    s.next()
    return Nodes.BreakStatNode(pos)

def p_continue_statement(s):
    # s.sy == 'continue'
    pos = s.position()
    s.next()
    return Nodes.ContinueStatNode(pos)

def p_return_statement(s):
    # s.sy == 'return'
    pos = s.position()
    s.next()
    if s.sy not in statement_terminators:
        value = p_expr(s)
    else:
        value = None
    return Nodes.ReturnStatNode(pos, value = value)

def p_raise_statement(s):
    # s.sy == 'raise'
    pos = s.position()
    s.next()
    exc_type = None
    exc_value = None
    exc_tb = None
    if s.sy not in statement_terminators:
        exc_type = p_simple_expr(s)
        if s.sy == ',':
            s.next()
            exc_value = p_simple_expr(s)
            if s.sy == ',':
                s.next()
                exc_tb = p_simple_expr(s)
    if exc_type or exc_value or exc_tb:
        return Nodes.RaiseStatNode(pos, 
            exc_type = exc_type,
            exc_value = exc_value,
            exc_tb = exc_tb)
    else:
        return Nodes.ReraiseStatNode(pos)

def p_import_statement(s):
    # s.sy in ('import', 'cimport')
    pos = s.position()
    kind = s.sy
    s.next()
    items = [p_dotted_name(s, as_allowed = 1)]
    while s.sy == ',':
        s.next()
        items.append(p_dotted_name(s, as_allowed = 1))
    stats = []
    for pos, target_name, dotted_name, as_name in items:
        if kind == 'cimport':
            stat = Nodes.CImportStatNode(pos, 
                module_name = dotted_name,
                as_name = as_name)
        else:
            if as_name and "." in dotted_name:
                name_list = ExprNodes.ListNode(pos, args = [
                    ExprNodes.StringNode(pos, value = "*")])
            else:
                name_list = None
            stat = Nodes.SingleAssignmentNode(pos,
                lhs = ExprNodes.NameNode(pos, 
                    name = as_name or target_name),
                rhs = ExprNodes.ImportNode(pos, 
                    module_name = ExprNodes.StringNode(pos,
                        value = dotted_name),
                    name_list = name_list))
        stats.append(stat)
    return Nodes.StatListNode(pos, stats = stats)

def p_from_import_statement(s, ctx):
    # s.sy == 'from'
    pos = s.position()
    s.next()
    (dotted_name_pos, _, dotted_name, _) = \
        p_dotted_name(s, as_allowed = 0)
    if s.sy in ('import', 'cimport'):
        kind = s.sy
        s.next()
    else:
        s.error("Expected 'import' or 'cimport'")
    if kind == 'cimport' and ctx.level not in ('module', 'module_pxd'):
        s.error("cimport statement not allowed in this context")
    if s.sy == '*':
        s.error("'import *' not supported")
    is_cimport = kind == 'cimport'
    imported_names = [p_imported_name(s, is_cimport)]
    while s.sy == ',':
        s.next()
        imported_names.append(p_imported_name(s, is_cimport))
    if kind == 'cimport':
        for imp in imported_names:
            local_name = imp.as_name or imp.name
            s.add_type_name(local_name)
        return Nodes.FromCImportStatNode(pos,
            module_name = dotted_name,
            imported_names = imported_names)
    else:
        imported_name_strings = []
        items = []
        for imp in imported_names:
            imported_name_strings.append(
                ExprNodes.StringNode(imp.pos, value = imp.name))
            items.append(
                (imp.name,
                 ExprNodes.NameNode(imp.pos, 
                 	name = imp.as_name or imp.name)))
        import_list = ExprNodes.ListNode(
            imported_names[0].pos, args = imported_name_strings)
        return Nodes.FromImportStatNode(pos,
            module = ExprNodes.ImportNode(dotted_name_pos,
                module_name = ExprNodes.StringNode(dotted_name_pos,
                    value = dotted_name),
                name_list = import_list),
            items = items)

class ImportedName(object):
    #  pos
    #  name
    #  as_name
    #  kind     'class', 'struct', 'union', None
    
    def __init__(self, pos, name, as_name, kind):
        self.pos = pos
        self.name = name
        self.as_name = as_name
        self.kind = kind

imported_name_kinds = ('class', 'struct', 'union')

def p_imported_name(s, is_cimport):
    pos = s.position()
    kind = None
    if is_cimport and s.systring in imported_name_kinds:
        kind = s.systring
        s.next()
    name = p_ident(s)
    as_name = p_as_name(s)
    return ImportedName(pos, name, as_name, kind)

def p_dotted_name(s, as_allowed):
    pos = s.position()
    target_name = p_ident(s)
    as_name = None
    names = [target_name]
    while s.sy == '.':
        s.next()
        names.append(p_ident(s))
    if as_allowed:
        as_name = p_as_name(s)
    return (pos, target_name, join(names, "."), as_name)

def p_as_name(s):
    if s.sy == 'IDENT' and s.systring == 'as':
        s.next()
        return p_ident(s)
    else:
        return None

def p_assert_statement(s):
    # s.sy == 'assert'
    pos = s.position()
    s.next()
    cond = p_simple_expr(s)
    if s.sy == ',':
        s.next()
        value = p_simple_expr(s)
    else:
        value = None
    return Nodes.AssertStatNode(pos, cond = cond, value = value)

statement_terminators = (';', 'NEWLINE', 'EOF')

def p_if_statement(s):
    # s.sy == 'if'
    pos = s.position()
    s.next()
    if_clauses = [p_if_clause(s)]
    while s.sy == 'elif':
        s.next()
        if_clauses.append(p_if_clause(s))
    else_clause = p_else_clause(s)
    return Nodes.IfStatNode(pos,
        if_clauses = if_clauses, else_clause = else_clause)

def p_if_clause(s):
    pos = s.position()
    test = p_simple_expr(s)
    body = p_suite(s)
    return Nodes.IfClauseNode(pos,
        condition = test, body = body)

def p_else_clause(s):
    if s.sy == 'else':
        s.next()
        return p_suite(s)
    else:
        return None

def p_while_statement(s):
    # s.sy == 'while'
    pos = s.position()
    s.next()
    test = p_simple_expr(s)
    body = p_suite(s)
    else_clause = p_else_clause(s)
    return Nodes.WhileStatNode(pos, 
        condition = test, body = body, 
        else_clause = else_clause)

def p_for_statement(s):
    # s.sy == 'for'
    pos = s.position()
    s.next()
    expr = p_for_expr(s)
    if s.sy == 'in':
        return p_standard_for_statement(s, expr)
    elif s.sy in inequality_relations:
        return p_integer_for_statement(s, expr)
    elif s.sy == 'from':
        #warning(pos, "Old-style integer for-loop is deprecated, use 'for x < i < y' instead")
        return p_old_style_integer_for_statement(s, expr)
    else:
        s.error("Expected 'in' or an inequality relation")

def p_standard_for_statement(s, target):
    # s.sy == 'in'
    s.next()
    iterator = p_for_iterator(s)
    body = p_suite(s)
    else_clause = p_else_clause(s)
    return Nodes.ForInStatNode(target.pos, 
        target = target,
        iterator = iterator,
        body = body,
        else_clause = else_clause)

def p_integer_for_statement(s, bound1):
    rel1 = s.sy
    s.next()
    name_pos = s.position()
    target = p_name(s)
    rel2_pos = s.position()
    rel2 = p_inequality_relation(s)
    bound2 = p_bit_expr(s)
    if rel1[0] <> rel2[0]:
        error(rel2_pos,
            "Relation directions in integer for-loop do not match")
    body = p_suite(s)
    else_clause = p_else_clause(s)
    return Nodes.IntegerForStatNode(bound1.pos,
        bound1 = bound1,
        relation1 = rel1,
        target = target,
        relation2 = rel2,
        bound2 = bound2,
        body = body,
        else_clause = else_clause)

def p_old_style_integer_for_statement(s, target):
    # s.sy == 'for'
    s.next()
    bound1 = p_bit_expr(s)
    rel1 = p_inequality_relation(s)
    name2_pos = s.position()
    name2 = p_ident(s)
    rel2_pos = s.position()
    rel2 = p_inequality_relation(s)
    bound2 = p_bit_expr(s)
    if not target.is_name:
        error(target.pos, 
            "Target of for-from statement must be a variable name")
    elif name2 <> target.name:
        error(name2_pos,
            "Variable name in for-from range does not match target")
    if rel1[0] <> rel2[0]:
        error(rel2_pos,
            "Relation directions in for-from do not match")
    body = p_suite(s)
    else_clause = p_else_clause(s)
    return Nodes.IntegerForStatNode(bound1.pos,
        bound1 = bound1,
        relation1 = rel1,
        target = target,
        relation2 = rel2,
        bound2 = bound2,
        body = body,
        else_clause = else_clause)

def p_inequality_relation(s):
    if s.sy in inequality_relations:
        op = s.sy
        s.next()
        return op
    else:
        s.error("Expected one of '<', '<=', '>' '>='")

inequality_relations = ('<', '<=', '>', '>=')

def p_for_expr(s):
    # Target of standard for-statement or first bound of integer for-statement
    pos = s.position()
    expr = p_bit_expr(s)
    if s.sy == ',':
        s.next()
        exprs = [expr]
        while s.sy <> 'in':
            exprs.append(p_bit_expr(s))
            if s.sy <> ',':
                break
            s.next()
        return ExprNodes.TupleNode(pos, args = exprs)
    else:
        return expr

def p_for_iterator(s):
    pos = s.position()
    expr = p_expr(s)
    return ExprNodes.IteratorNode(pos, sequence = expr)

def p_try_statement(s):
    # s.sy == 'try'
    pos = s.position()
    s.next()
    body = p_suite(s)
    except_clauses = []
    else_clause = None
    if s.sy in ('except', 'else'):
        while s.sy == 'except':
            except_clauses.append(p_except_clause(s))
        if s.sy == 'else':
            s.next()
            else_clause = p_suite(s)
        return Nodes.TryExceptStatNode(pos,
            body = body, except_clauses = except_clauses,
            else_clause = else_clause)
    elif s.sy == 'finally':
        s.next()
        finally_clause = p_suite(s)
        return Nodes.TryFinallyStatNode(pos,
            body = body, finally_clause = finally_clause)
    else:
        s.error("Expected 'except' or 'finally'")

def p_except_clause(s):
    # s.sy == 'except'
    pos = s.position()
    s.next()
    exc_type = None
    exc_value = None
    tb_value = None
    if s.sy <> ':':
        exc_type = p_simple_expr(s)
        if s.sy == ',':
            s.next()
            exc_value = p_simple_expr(s)
            if s.sy == ',':
                s.next()
                tb_value = p_simple_expr(s)
    body = p_suite(s)
    return Nodes.ExceptClauseNode(pos,
        pattern = exc_type, exc_target = exc_value, tb_target = tb_value, body = body)

def p_include_statement(s, ctx):
    pos = s.position()
    s.next() # 'include'
    _, include_file_name = p_string_literal(s)
    s.expect_newline("Syntax error in include statement")
    if s.compile_time_eval:
        include_file_path = s.context.find_include_file(include_file_name, pos)
        if include_file_path:
            s.included_files.append(include_file_name)
            f = open(include_file_path, "rU")
            s2 = PyrexScanner(f, include_file_path, parent_scanner = s)
            try:
                tree = p_statement_list(s2, ctx)
            finally:
                f.close()
            return tree
        else:
            return None
    else:
        return Nodes.PassStatNode(pos)

def p_with_statement(s):
    pos = s.position()
    s.next() # 'with'
#	if s.sy == 'IDENT' and s.systring in ('gil', 'nogil'):
    if s.sy == 'IDENT' and s.systring == 'nogil':
        state = s.systring
        s.next()
        body = p_suite(s)
        return Nodes.GILStatNode(pos, state = state, body = body)
    else:
        s.error("Only 'with nogil' implemented")
    
def p_simple_statement(s, ctx):
    if s.sy == 'global':
        node = p_global_statement(s)
    elif s.sy == 'print':
        node = p_print_statement(s)
    elif s.sy == 'del':
        node = p_del_statement(s)
    elif s.sy == 'break':
        node = p_break_statement(s)
    elif s.sy == 'continue':
        node = p_continue_statement(s)
    elif s.sy == 'return':
        node = p_return_statement(s)
    elif s.sy == 'raise':
        node = p_raise_statement(s)
    elif s.sy == 'cimport':
        if ctx.level not in ('module', 'module_pxd'):
            s.error("cimport statement not allowed in this context")
        node = p_import_statement(s)
    elif s.sy == 'import':
        node = p_import_statement(s)
    elif s.sy == 'from':
        node = p_from_import_statement(s, ctx)
    elif s.sy == 'assert':
        node = p_assert_statement(s)
    elif s.sy == 'pass':
        node = p_pass_statement(s)
    else:
        node = p_expression_or_assignment(s)
    return node

def p_simple_statement_list(s, ctx):
    # Parse a series of simple statements on one line
    # separated by semicolons.
    stat = p_simple_statement(s, ctx)
    if s.sy == ';':
        stats = [stat]
        while s.sy == ';':
            s.next()
            if s.sy in ('NEWLINE', 'EOF'):
                break
            stats.append(p_simple_statement(s, ctx))
        stat = Nodes.StatListNode(stats[0].pos, stats = stats)
    s.expect_newline("Syntax error in simple statement list")
    return stat

def p_compile_time_expr(s):
    old = s.compile_time_expr
    s.compile_time_expr = 1
    expr = p_expr(s)
    s.compile_time_expr = old
    return expr

def p_DEF_statement(s):
    pos = s.position()
    denv = s.compile_time_env
    s.next() # 'DEF'
    name = p_ident(s)
    s.expect('=')
    expr = p_compile_time_expr(s)
    value = expr.compile_time_value(denv)
    #print "p_DEF_statement: %s = %r" % (name, value) ###
    denv.declare(name, value)
    s.expect_newline()
    return Nodes.PassStatNode(pos)

def p_IF_statement(s, ctx):
    pos = s.position()
    saved_eval = s.compile_time_eval
    current_eval = saved_eval
    denv = s.compile_time_env
    result = None
    while 1:
        s.next() # 'IF' or 'ELIF'
        expr = p_compile_time_expr(s)
        s.compile_time_eval = current_eval and bool(expr.compile_time_value(denv))
        body = p_suite(s, ctx)
        if s.compile_time_eval:
            result = body
            current_eval = 0
        if s.sy <> 'ELIF':
            break
    if s.sy == 'ELSE':
        s.next()
        s.compile_time_eval = current_eval
        body = p_suite(s, ctx)
        if current_eval:
            result = body
    if not result:
        result = Nodes.PassStatNode(pos)
    s.compile_time_eval = saved_eval
    return result

def p_statement(s, ctx):
    pos = s.position()
    cdef_flag = ctx.cdef_flag
    if s.sy == 'ctypedef':
        if ctx.level not in ('module', 'module_pxd'):
            s.error("ctypedef statement not allowed here")
        #if ctx.api:
        #	error(s.pos, "'api' not allowed with 'ctypedef'")
        return p_ctypedef_statement(s, ctx)
    elif s.sy == 'DEF':
        return p_DEF_statement(s)
    elif s.sy == 'IF':
        return p_IF_statement(s, ctx)
    else:
        if s.sy == 'cdef':
            cdef_flag = 1
            s.next()
            if s.sy == '+':
                ctx = ctx(cplus_flag = 1)
                s.next()
        if cdef_flag:
            if ctx.level not in ('module', 'module_pxd', 'function', 'c_class', 'c_class_pxd'):
                s.error('cdef statement not allowed here')
            return p_cdef_statement(s, ctx)
        else:
            if ctx.api:
                error(s.pos, "'api' not allowed with this statement")
            if s.sy == 'def':
                if ctx.level not in ('module', 'class', 'c_class', 'property'):
                    s.error('def statement not allowed here')
                return p_def_statement(s)
            elif s.sy == 'class':
                if ctx.level <> 'module':
                    s.error("class definition not allowed here")
                return p_class_statement(s)
            elif s.sy == 'include':
                #if ctx.level not in ('module', 'module_pxd'):
                #	s.error("include statement not allowed here")
                return p_include_statement(s, ctx)
            elif ctx.level == 'c_class' and s.sy == 'IDENT' and s.systring == 'property':
                return p_property_decl(s)
            elif s.sy == 'pass' and ctx.level <> 'property':
                return p_pass_statement(s, with_newline = 1)
            else:
                if ctx.level in ('c_class', 'c_class_pxd', 'property'):
                    s.error("Executable statement not allowed here")
                if s.sy == 'if':
                    return p_if_statement(s)
                elif s.sy == 'while':
                    return p_while_statement(s)
                elif s.sy == 'for':
                    return p_for_statement(s)
                elif s.sy == 'try':
                    return p_try_statement(s)
                elif s.sy == 'with':
                    return p_with_statement(s)
                else:
                    return p_simple_statement_list(s, ctx)

def p_statement_list(s, ctx):
    # Parse a series of statements separated by newlines.
    pos = s.position()
    stats = []
    while s.sy not in ('DEDENT', 'EOF'):
        stats.append(p_statement(s, ctx))
    if len(stats) == 1:
        return stats[0]
    else:
        return Nodes.StatListNode(pos, stats = stats)

def p_suite(s, ctx = Ctx(), with_doc = 0, with_pseudo_doc = 0):
    pos = s.position()
    s.expect(':')
    doc = None
    stmts = []
    if s.sy == 'NEWLINE':
        s.next()
        s.expect_indent()
        if with_doc or with_pseudo_doc:
            doc = p_doc_string(s)
        body = p_statement_list(s, ctx)
        s.expect_dedent()
    else:
        if ctx.api:
            error(s.pos, "'api' not allowed with this statement")
        if ctx.level in ('module', 'class', 'function', 'other'):
            body = p_simple_statement_list(s, ctx)
        else:
            body = p_pass_statement(s)
            s.expect_newline("Syntax error in declarations")
    if with_doc:
        return doc, body
    else:
        return body

def p_c_base_type(s, self_flag = 0):
    # If self_flag is true, this is the base type for the
    # self argument of a C method of an extension type.
    if s.sy == '(':
        return p_c_complex_base_type(s)
    else:
        return p_c_simple_base_type(s, self_flag)

def p_calling_convention(s):
    if s.sy == 'IDENT' and s.systring in calling_convention_words:
        result = s.systring
        s.next()
        return result
    else:
        return ""

calling_convention_words = ("__stdcall", "__cdecl", "__fastcall")

def p_c_complex_base_type(s):
    # s.sy == '('
    pos = s.position()
    s.next()
    base_type = p_c_base_type(s)
    declarator = p_c_declarator(s, empty = 1)
    s.expect(')')
    return Nodes.CComplexBaseTypeNode(pos, 
        base_type = base_type, declarator = declarator)

def p_c_simple_base_type(s, self_flag):
    #print "p_c_simple_base_type: self_flag =", self_flag
    is_basic = 0
    signed = 1
    longness = 0
    module_path = []
    pos = s.position()
    if looking_at_base_type(s):
        #print "p_c_simple_base_type: looking_at_base_type at", s.position()
        is_basic = 1
        signed, longness = p_sign_and_longness(s)
        if s.sy == 'IDENT' and s.systring in basic_c_type_names:
            name = s.systring
            s.next()
        else:
            name = 'int'
    elif s.looking_at_type_name() or looking_at_dotted_name(s):
        #print "p_c_simple_base_type: looking_at_type_name at", s.position()
        name = s.systring
        s.next()
        while s.sy == '.':
            module_path.append(name)
            s.next()
            name = p_ident(s)
    else:
        #print "p_c_simple_base_type: not looking at type at", s.position()
        name = None
    return Nodes.CSimpleBaseTypeNode(pos, 
        name = name, module_path = module_path,
        is_basic_c_type = is_basic, signed = signed,
        longness = longness, is_self_arg = self_flag)

def looking_at_type(s):
    return looking_at_base_type(s) or s.looking_at_type_name()

def looking_at_base_type(s):
    #print "looking_at_base_type?", s.sy, s.systring, s.position()
    return s.sy == 'IDENT' and s.systring in base_type_start_words

def looking_at_dotted_name(s):
    if s.sy == 'IDENT':
        name = s.systring
        s.next()
        result = s.sy == '.'
        s.put_back('IDENT', name)
        return result
    else:
        return 0

basic_c_type_names = ("void", "char", "int", "float", "double") #,
    #"size_t", "Py_ssize_t")

sign_and_longness_words = ("short", "long", "signed", "unsigned")

base_type_start_words = \
    basic_c_type_names + sign_and_longness_words

def p_sign_and_longness(s):
    signed = 1
    longness = 0
    while s.sy == 'IDENT' and s.systring in sign_and_longness_words:
        if s.systring == 'unsigned':
            signed = 0
        elif s.systring == 'signed':
            signed = 2
        elif s.systring == 'short':
            longness = -1
        elif s.systring == 'long':
            longness += 1
        s.next()
    return signed, longness

def p_opt_cname(s):
    literal = p_opt_string_literal(s)
    if literal:
        _, cname = literal
    else:
        cname = None
    return cname

def p_c_declarator(s, ctx = Ctx(), empty = 0, is_type = 0, cmethod_flag = 0, nonempty = 0,
        calling_convention_allowed = 0):
    # If empty is true, the declarator must be empty. If nonempty is true,
    # the declarator must be nonempty. Otherwise we don't care.
    # If cmethod_flag is true, then if this declarator declares
    # a function, it's a C method of an extension type.
    pos = s.position()
    if s.sy == '(':
        s.next()
        if s.sy == ')' or looking_at_type(s):
            base = Nodes.CNameDeclaratorNode(pos, name = "", cname = None)
            result = p_c_func_declarator(s, pos, ctx, base, cmethod_flag)
        else:
            result = p_c_declarator(s, ctx, empty = empty, is_type = is_type,
                cmethod_flag = cmethod_flag, nonempty = nonempty,
                calling_convention_allowed = 1)
            s.expect(')')
    else:
        result = p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag, nonempty)
    if not calling_convention_allowed and result.calling_convention and s.sy <> '(':
        error(s.position(), "%s on something that is not a function"
            % result.calling_convention)
    while s.sy in ('[', '('):
        pos = s.position()
        if s.sy == '[':
            result = p_c_array_declarator(s, result)
        else: # sy == '('
            s.next()
            result = p_c_func_declarator(s, pos, ctx, result, cmethod_flag)
        cmethod_flag = 0
    return result

def p_c_array_declarator(s, base):
    pos = s.position()
    s.next() # '['
    if s.sy <> ']':
        dim = p_expr(s)
    else:
        dim = None
    s.expect(']')
    return Nodes.CArrayDeclaratorNode(pos, base = base, dimension = dim)

def p_c_func_declarator(s, pos, ctx, base, cmethod_flag):
    #  Opening paren has already been skipped
    args = p_c_arg_list(s, ctx, cmethod_flag = cmethod_flag,
        nonempty_declarators = 0)
    ellipsis = p_optional_ellipsis(s)
    s.expect(')')
    nogil = p_nogil(s)
    exc_val, exc_check = p_exception_value_clause(s)
    with_gil = p_with_gil(s)
    return Nodes.CFuncDeclaratorNode(pos, 
        base = base, args = args, has_varargs = ellipsis,
        exception_value = exc_val, exception_check = exc_check,
        nogil = nogil or ctx.nogil or with_gil, with_gil = with_gil)

def p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag, nonempty):
    pos = s.position()
    calling_convention = p_calling_convention(s)
    if s.sy == '*':
        s.next()
        base = p_c_declarator(s, ctx, empty = empty, is_type = is_type,
            cmethod_flag = cmethod_flag, nonempty = nonempty)
        result = Nodes.CPtrDeclaratorNode(pos, 
            base = base)
    elif s.sy == '**': # scanner returns this as a single token
        s.next()
        base = p_c_declarator(s, ctx, empty = empty, is_type = is_type,
            cmethod_flag = cmethod_flag, nonempty = nonempty)
        result = Nodes.CPtrDeclaratorNode(pos,
            base = Nodes.CPtrDeclaratorNode(pos,
                base = base))
    else:
        if s.sy == 'IDENT':
            name = s.systring
            if is_type:
                s.add_type_name(name)
            if empty:
                error(s.position(), "Declarator should be empty")
            s.next()
            cname = p_opt_cname(s)
        else:
            if nonempty:
                error(s.position(), "Empty declarator")
            name = ""
            cname = None
        result = Nodes.CNameDeclaratorNode(pos,
            name = name, cname = cname)
    result.calling_convention = calling_convention
    return result

def p_nogil(s):
    if s.sy == 'IDENT' and s.systring == 'nogil':
        s.next()
        return 1
    else:
        return 0

def p_with_gil(s):
    if s.sy == 'with':
        s.next()
        s.expect_keyword('gil')
        return 1
    else:
        return 0

def p_exception_value_clause(s):
    exc_val = None
    exc_check = 0
    if s.sy == 'except':
        s.next()
        if s.sy == '*':
            exc_check = 1
            s.next()
        else:
            if s.sy == '?':
                exc_check = 1
                s.next()
            exc_val = p_simple_expr(s)
    return exc_val, exc_check

c_arg_list_terminators = ('*', '**', '.', ')')

def p_c_arg_list(s, ctx = Ctx(), in_pyfunc = 0, cmethod_flag = 0, nonempty_declarators = 0,
        kw_only = 0):
    #  Comma-separated list of C argument declarations, possibly empty.
    #  May have a trailing comma.
    args = []
    is_self_arg = cmethod_flag
    while s.sy not in c_arg_list_terminators:
        args.append(p_c_arg_decl(s, ctx, in_pyfunc, is_self_arg,
            nonempty = nonempty_declarators, kw_only = kw_only))
        if s.sy <> ',':
            break
        s.next()
        is_self_arg = 0
    return args

def p_optional_ellipsis(s):
    if s.sy == '.':
        expect_ellipsis(s)
        return 1
    else:
        return 0

def p_c_arg_decl(s, ctx, in_pyfunc, cmethod_flag = 0, nonempty = 0, kw_only = 0):
    pos = s.position()
    allow_none = None
    #not_none = 0
    default = None
    base_type = p_c_base_type(s, cmethod_flag)
    declarator = p_c_declarator(s, ctx, nonempty = nonempty)
    if s.sy in ('or', 'not'):
        or_not = s.sy
        s.next()
        if s.sy == 'IDENT' and s.systring == 'None':
            s.next()
        else:
            s.error("Expected 'None'")
        if not in_pyfunc:
            error(pos, "'%s None' only allowed in Python functions" % or_not)
        allow_none = or_not == 'or'
    if s.sy == '=':
        s.next()
        default = p_simple_expr(s)
    return Nodes.CArgDeclNode(pos,
        base_type = base_type,
        declarator = declarator,
        allow_none = allow_none,
        default = default,
        kw_only = kw_only)

def p_api(s):
    if s.sy == 'IDENT' and s.systring == 'api':
        s.next()
        return 1
    else:
        return 0

def p_cdef_statement(s, ctx):
    ctx = ctx(cdef_flag = 1)
    pos = s.position()
    ctx.visibility = p_visibility(s, ctx.visibility)
    ctx.api = ctx.api or p_api(s)
    if ctx.api:
        if ctx.visibility not in ('private', 'public'):
            error(pos, "Cannot combine 'api' with '%s'" % visibility)
    if ctx.visibility == 'extern' and s.sy == 'from':
        return p_cdef_extern_block(s, pos, ctx)
    if p_nogil(s):
        ctx.nogil = 1
    if s.sy == ':':
        return p_cdef_block(s, ctx)
    elif s.sy == 'class':
        if ctx.level not in ('module', 'module_pxd'):
            error(pos, "Extension type definition not allowed here")
        #if api:
        #	error(pos, "'api' not allowed with extension class")
        return p_c_class_definition(s, pos, ctx)
    elif s.sy == 'IDENT' and s.systring in struct_union_or_enum:
        if ctx.level not in ('module', 'module_pxd'):
            error(pos, "C struct/union/enum definition not allowed here")
        #if ctx.visibility == 'public':
        #	error(pos, "Public struct/union/enum definition not implemented")
        #if ctx.api:
        #	error(pos, "'api' not allowed with '%s'" % s.systring)
        if s.systring == "enum":
            return p_c_enum_definition(s, pos, ctx)
        else:
            return p_c_struct_or_union_definition(s, pos, ctx)
    elif s.sy == 'pass':
        node = p_pass_statement(s)
        s.expect_newline('Expected a newline')
        return node
    else:
        return p_c_func_or_var_declaration(s, pos, ctx)

def p_cdef_block(s, ctx):
    return p_suite(s, ctx(cdef_flag = 1))

def p_cdef_extern_block(s, pos, ctx):
    include_file = None
    s.expect('from')
    if s.sy == '*':
        s.next()
    else:
        _, include_file = p_string_literal(s)
    ctx = ctx(cdef_flag = 1, visibility = 'extern', extern_from = True)
    if p_nogil(s):
        ctx.nogil = 1
    body = p_suite(s, ctx)
    return Nodes.CDefExternNode(pos,
        include_file = include_file,
        body = body)

struct_union_or_enum = (
    "struct", "union", "enum"
)

def p_c_enum_definition(s, pos, ctx):
    # s.sy == ident 'enum'
    s.next()
    if s.sy == 'IDENT':
        name = s.systring
        s.next()
        s.add_type_name(name)
        cname = p_opt_cname(s)
    else:
        name = None
        cname = None
    items = None
    s.expect(':')
    items = []
    if s.sy <> 'NEWLINE':
        p_c_enum_line(s, items)
    else:
        s.next() # 'NEWLINE'
        s.expect_indent()
        while s.sy not in ('DEDENT', 'EOF'):
            p_c_enum_line(s, items)
        s.expect_dedent()
    return Nodes.CEnumDefNode(pos, name = name, cname = cname, items = items,
        typedef_flag = ctx.typedef_flag,
        visibility = ctx.visibility,
        in_pxd = ctx.level == 'module_pxd')

def p_c_enum_line(s, items):
    if s.sy <> 'pass':
        p_c_enum_item(s, items)
        while s.sy == ',':
            s.next()
            if s.sy in ('NEWLINE', 'EOF'):
                break
            p_c_enum_item(s, items)
    else:
        s.next()
    s.expect_newline("Syntax error in enum item list")

def p_c_enum_item(s, items):
    pos = s.position()
    name = p_ident(s)
    cname = p_opt_cname(s)
    value = None
    if s.sy == '=':
        s.next()
        value = p_simple_expr(s)
    items.append(Nodes.CEnumDefItemNode(pos, 
        name = name, cname = cname, value = value))

def p_c_struct_or_union_definition(s, pos, ctx):
    # s.sy == ident 'struct' or 'union'
    ctx.cplus_check(pos)
    kind = s.systring
    s.next()
    module_path, name = p_qualified_name(s)
    bases = []
    if s.sy == '(':
        s.next()
        while s.sy == 'IDENT':
            bases.append(p_qualified_name(s))
            if s.sy <> ',':
                break
            s.next()
        s.expect(')')
    if bases and not ctx.cplus_flag:
        error(s, "Only C++ struct may have bases")
    cname = p_opt_cname(s)
    s.add_type_name(name)
    attributes = None
    if s.sy == ':':
        s.next()
        s.expect('NEWLINE')
        s.expect_indent()
        attributes = []
        body_ctx = Ctx()
        while s.sy <> 'DEDENT':
            if s.sy <> 'pass':
                attributes.append(p_c_func_or_var_declaration(s,
                    s.position(), body_ctx))
            else:
                s.next()
                s.expect_newline("Expected a newline")
        s.expect_dedent()
    else:
        s.expect_newline("Syntax error in struct or union definition")
    return Nodes.CStructOrUnionDefNode(pos, 
        name = name, cname = cname, module_path = module_path,
        kind = kind, attributes = attributes,
        typedef_flag = ctx.typedef_flag,
        visibility = ctx.visibility,
        in_pxd = ctx.level == 'module_pxd',
        cplus_flag = ctx.cplus_flag,
        bases = bases)

def p_visibility(s, prev_visibility):
    pos = s.position()
    visibility = prev_visibility
    if s.sy == 'IDENT' and s.systring in ('extern', 'public', 'readonly'):
        visibility = s.systring
        if prev_visibility <> 'private' and visibility <> prev_visibility:
            s.error("Conflicting visibility options '%s' and '%s'"
                % (prev_visibility, visibility))
        s.next()
    return visibility

def p_c_func_or_var_declaration(s, pos, ctx):
    cmethod_flag = ctx.level in ('c_class', 'c_class_pxd')
    base_type = p_c_base_type(s)
    declarator = p_c_declarator(s, ctx, cmethod_flag = cmethod_flag, nonempty = 1)
    if s.sy == ':':
        if ctx.level not in ('module', 'c_class'):
            s.error("C function definition not allowed here")
        suite = p_suite(s, Ctx(level = 'function'), with_pseudo_doc = 1)
        result = Nodes.CFuncDefNode(pos,
            visibility = ctx.visibility,
            base_type = base_type,
            declarator = declarator, 
            body = suite,
            api = ctx.api)
    else:
        #if api:
        #	error(pos, "'api' not allowed with variable declaration")
        declarators = [declarator]
        while s.sy == ',':
            s.next()
            if s.sy == 'NEWLINE':
                break
            declarator = p_c_declarator(s, ctx, cmethod_flag = cmethod_flag, nonempty = 1)
            declarators.append(declarator)
        s.expect_newline("Syntax error in C variable declaration")
        result = Nodes.CVarDefNode(pos, 
            visibility = ctx.visibility,
            base_type = base_type, 
            declarators = declarators,
            in_pxd = ctx.level == 'module_pxd',
            api = ctx.api)
    return result

def p_ctypedef_statement(s, ctx):
    # s.sy == 'ctypedef'
    pos = s.position()
    s.next()
    visibility = p_visibility(s, ctx.visibility)
    api = p_api(s)
    ctx = ctx(typedef_flag = 1, visibility = visibility)
    if api:
        ctx.api = 1
    if s.sy == 'class':
        return p_c_class_definition(s, pos, ctx)
    elif s.sy == 'IDENT' and s.systring in ('struct', 'union', 'enum'):
        if s.systring == 'enum':
            return p_c_enum_definition(s, pos, ctx)
        else:
            return p_c_struct_or_union_definition(s, pos, ctx)
    else:
        base_type = p_c_base_type(s)
        declarator = p_c_declarator(s, ctx, is_type = 1, nonempty = 1)
        s.expect_newline("Syntax error in ctypedef statement")
        return Nodes.CTypeDefNode(pos,
            base_type = base_type, declarator = declarator,
            visibility = ctx.visibility,
            in_pxd = ctx.level == 'module_pxd')

def p_def_statement(s):
    # s.sy == 'def'
    pos = s.position()
    s.next()
    name = p_ident(s)
    #args = []
    s.expect('(');
    args = p_c_arg_list(s, in_pyfunc = 1, nonempty_declarators = 1)
    star_arg = None
    starstar_arg = None
    if s.sy == '*':
        s.next()
        if s.sy == 'IDENT':
            star_arg = p_py_arg_decl(s)
        if s.sy == ',':
            s.next()
            args.extend(p_c_arg_list(s, in_pyfunc = 1,
                nonempty_declarators = 1, kw_only = 1))
        elif s.sy <>')':
            s.error("Syntax error in Python function argument list")
    if s.sy == '**':
        s.next()
        starstar_arg = p_py_arg_decl(s)
    s.expect(')')
    if p_nogil(s):
        error(s.pos, "Python function cannot be declared nogil")
    doc, body = p_suite(s, Ctx(level = 'function'), with_doc = 1)
    return Nodes.DefNode(pos, name = name, args = args, 
        star_arg = star_arg, starstar_arg = starstar_arg,
        doc = doc, body = body)

def p_py_arg_decl(s):
    pos = s.position()
    name = p_ident(s)
    return Nodes.PyArgDeclNode(pos, name = name)

def p_class_statement(s):
    # s.sy == 'class'
    pos = s.position()
    s.next()
    class_name = p_ident(s)
    if s.sy == '(':
        s.next()
        base_list = p_simple_expr_list(s)
        s.expect(')')
    else:
        base_list = []
    doc, body = p_suite(s, Ctx(level = 'class'), with_doc = 1)
    return Nodes.PyClassDefNode(pos,
        name = class_name,
        bases = ExprNodes.TupleNode(pos, args = base_list),
        doc = doc, body = body)

def p_qualified_name(s):
    path = []
    name = p_ident(s)
    while s.sy == '.':
        s.next()
        path.append(name)
        name = p_ident(s)
    return path, name

class CClassOptions:

    objstruct_cname = None
    typeobj_cname = None
    no_gc = 0


def p_c_class_definition(s, pos, ctx):
    # s.sy == 'class'
    s.next()
    module_path, class_name = p_qualified_name(s)
    if module_path and s.sy == 'IDENT' and s.systring == 'as':
        s.next()
        as_name = p_ident(s)
    else:
        as_name = class_name
    s.add_type_name(as_name)
    options = CClassOptions()
    base_class_module = None
    base_class_name = None
    if s.sy == '(':
        s.next()
        base_class_path, base_class_name = p_qualified_name(s)
        if s.sy == ',':
            s.error("C class may only have one base class")
        s.expect(')')
        base_class_module = ".".join(base_class_path)
    if s.sy == '[':
        p_c_class_options(s, ctx, options)
    if s.sy == ':':
        if ctx.level == 'module_pxd':
            body_level = 'c_class_pxd'
        else:
            body_level = 'c_class'
        doc, body = p_suite(s, Ctx(level = body_level), with_doc = 1)
    else:
        s.expect_newline("Syntax error in C class definition")
        doc = None
        body = None
    if ctx.visibility == 'extern':
        if not module_path:
            error(pos, "Module name required for 'extern' C class")
        if options.typeobj_cname:
            error(pos, "Type object name specification not allowed for 'extern' C class")
    elif ctx.visibility == 'public':
        if not options.objstruct_cname:
            error(pos, "Object struct name specification required for 'public' C class")
        if not options.typeobj_cname:
            error(pos, "Type object name specification required for 'public' C class")
    else:
        if ctx.api:
            error(pos, "Only 'public' C class can be declared 'api'")
    return Nodes.CClassDefNode(pos,
        visibility = ctx.visibility,
        typedef_flag = ctx.typedef_flag,
        api = ctx.api,
        module_name = ".".join(module_path),
        class_name = class_name,
        as_name = as_name,
        base_class_module = base_class_module,
        base_class_name = base_class_name,
        options = options,
        in_pxd = ctx.level == 'module_pxd',
        doc = doc,
        body = body)

def p_c_class_options(s, ctx, options):
    s.expect('[')
    while 1:
        if s.sy <> 'IDENT':
            break
        if s.systring == 'object':
            if ctx.visibility not in ('public', 'extern'):
                error(s.position(), "Object name option only allowed for 'public' or 'extern' C class")
            s.next()
            options.objstruct_cname = p_ident(s)
        elif s.systring == 'type':
            if ctx.visibility not in ('public', 'extern'):
                error(s.position(), "Type name option only allowed for 'public' or 'extern' C class")
            s.next()
            options.typeobj_cname = p_ident(s)
        elif s.systring == 'nogc':
            s.next()
            options.no_gc = 1
        else:
            s.error("Unrecognised C class option '%s'" % s.systring)
        if s.sy <> ',':
            break
        s.next()
    s.expect(']', "Expected a C class option")

def p_property_decl(s):
    pos = s.position()
    s.next() # 'property'
    name = p_ident(s)
    doc, body = p_suite(s, Ctx(level = 'property'), with_doc = 1)
    return Nodes.PropertyNode(pos, name = name, doc = doc, body = body)

def p_doc_string(s):
    if s.sy == 'STRING' or s.sy == 'BEGIN_STRING':
        _, result = p_cat_string_literal(s)
        if s.sy <> 'EOF':
            s.expect_newline("Syntax error in doc string")
        return result
    else:
        return None

def p_module(s, pxd):
    s.add_type_name("object")
    pos = s.position()
    doc = p_doc_string(s)
    if pxd:
        level = 'module_pxd'
    else:
        level = 'module'
    body = p_statement_list(s, Ctx(level = level))
    if s.sy <> 'EOF':
        s.error("Syntax error in statement [%s,%s]" % (
            repr(s.sy), repr(s.systring)))
    return ModuleNode(pos, doc = doc, body = body)

#----------------------------------------------
#
#   Debugging
#
#----------------------------------------------

def print_parse_tree(f, node, level, key = None):	
    ind = "  " * level
    if node:
        f.write(ind)
        if key:
            f.write("%s: " % key)
        t = type(node)
        if t == TupleType:
            f.write("(%s @ %s\n" % (node[0], node[1]))
            for i in xrange(2, len(node)):
                print_parse_tree(f, node[i], level+1)
            f.write("%s)\n" % ind)
            return
        elif isinstance(node, Node):
            try:
                tag = node.tag
            except AttributeError:
                tag = node.__class__.__name__
            f.write("%s @ %s\n" % (tag, node.pos))
            for name, value in node.__dict__.items():
                if name <> 'tag' and name <> 'pos':
                    print_parse_tree(f, value, level+1, name)
            return
        elif t == ListType:
            f.write("[\n")
            for i in xrange(len(node)):
                print_parse_tree(f, node[i], level+1)
            f.write("%s]\n" % ind)
            return
    f.write("%s%s\n" % (ind, node))