diff options
Diffstat (limited to 'debian/pyrex/pyrex-0.9.9/Pyrex/Compiler/Scanning.py')
| -rw-r--r-- | debian/pyrex/pyrex-0.9.9/Pyrex/Compiler/Scanning.py | 390 |
1 files changed, 390 insertions, 0 deletions
diff --git a/debian/pyrex/pyrex-0.9.9/Pyrex/Compiler/Scanning.py b/debian/pyrex/pyrex-0.9.9/Pyrex/Compiler/Scanning.py new file mode 100644 index 00000000..1fc92f99 --- /dev/null +++ b/debian/pyrex/pyrex-0.9.9/Pyrex/Compiler/Scanning.py @@ -0,0 +1,390 @@ +# +# Pyrex Scanner +# + +#import pickle +import cPickle as pickle + +import os +import platform +import stat +import sys +from time import time + +from Pyrex import Plex +from Pyrex.Plex import Scanner +from Pyrex.Plex.Errors import UnrecognizedInput +from Errors import CompileError, error +from Lexicon import string_prefixes, make_lexicon + +plex_version = getattr(Plex, '_version', None) +#print "Plex version:", plex_version ### + +debug_scanner = 0 +trace_scanner = 0 +scanner_debug_flags = 0 +scanner_dump_file = None +binary_lexicon_pickle = 1 +notify_lexicon_unpickling = 0 +notify_lexicon_pickling = 1 + +lexicon = None + +#----------------------------------------------------------------- + +def hash_source_file(path): + # Try to calculate a hash code for the given source file. + # Returns an empty string if the file cannot be accessed. + #print "Hashing", path ### + try: + from hashlib import md5 + except ImportError: + from md5 import new as md5 + try: + try: + f = open(path, "rU") + text = f.read() + except IOError, e: + print "Unable to hash scanner source file (%s)" % e + return "" + finally: + f.close() + # Normalise spaces/tabs. We don't know what sort of + # space-tab substitution the file may have been + # through, so we replace all spans of spaces and + # tabs by a single space. + import re + text = re.sub("[ \t]+", " ", text) + hash = md5(text).hexdigest() + return hash + +def open_pickled_lexicon(expected_hash): + # Try to open pickled lexicon file and verify that + # it matches the source file. Returns the opened + # file if successful, otherwise None. ??? + f = None + result = None + if os.path.exists(lexicon_pickle): + try: + f = open(lexicon_pickle, "rb") + actual_hash = pickle.load(f) + if actual_hash == expected_hash: + result = f + f = None + else: + print "Lexicon hash mismatch:" ### + print " expected", expected_hash ### + print " got ", actual_hash ### + except IOError, e: + print "Warning: Unable to read pickled lexicon", lexicon_pickle + print e + if f: + f.close() + return result + +def try_to_unpickle_lexicon(): + global lexicon, lexicon_pickle, lexicon_hash + dir = os.path.dirname(__file__) + source_file = os.path.join(dir, "Lexicon.py") + lexicon_hash = hash_source_file(source_file) + lexicon_pickle = os.path.join(dir, "Lexicon.pickle") + f = open_pickled_lexicon(expected_hash = lexicon_hash) + if f: + if notify_lexicon_unpickling: + t0 = time() + print "Unpickling lexicon..." + lexicon = pickle.load(f) + f.close() + if notify_lexicon_unpickling: + t1 = time() + print "Done (%.2f seconds)" % (t1 - t0) + +def create_new_lexicon(): + global lexicon + t0 = time() + print "Creating lexicon..." + lexicon = make_lexicon() + t1 = time() + print "Done (%.2f seconds)" % (t1 - t0) + +def pickle_lexicon(): + f = None + try: + f = open(lexicon_pickle, "wb") + except IOError: + print "Warning: Unable to save pickled lexicon in", lexicon_pickle + if f: + if notify_lexicon_pickling: + t0 = time() + print "Pickling lexicon..." + pickle.dump(lexicon_hash, f, binary_lexicon_pickle) + pickle.dump(lexicon, f, binary_lexicon_pickle) + f.close() + if notify_lexicon_pickling: + t1 = time() + print "Done (%.2f seconds)" % (t1 - t0) + +def get_lexicon(): + global lexicon + if not lexicon and plex_version is None: + try_to_unpickle_lexicon() + if not lexicon: + create_new_lexicon() + if plex_version is None: + pickle_lexicon() + return lexicon + +#------------------------------------------------------------------ + +reserved_words = [ + "global", "include", "ctypedef", "cdef", "def", "class", + "print", "del", "pass", "break", "continue", "return", + "raise", "import", "exec", "try", "except", "finally", + "while", "if", "elif", "else", "for", "in", "assert", + "and", "or", "not", "is", "in", "lambda", "from", + "NULL", "cimport", "with", "DEF", "IF", "ELIF", "ELSE" +] + +class Method: + + def __init__(self, name): + self.name = name + self.__name__ = name # for Plex tracing + + def __call__(self, stream, text): + return getattr(stream, self.name)(text) + +#------------------------------------------------------------------ + +def build_resword_dict(): + d = {} + for word in reserved_words: + d[word] = 1 + return d + +#------------------------------------------------------------------ + +class CompileTimeScope(object): + + def __init__(self, outer = None): + self.entries = {} + self.outer = outer + + def declare(self, name, value): + self.entries[name] = value + + def lookup_here(self, name): + return self.entries[name] + + def lookup(self, name): + try: + return self.lookup_here(name) + except KeyError: + outer = self.outer + if outer: + return outer.lookup(name) + else: + raise + +def initial_compile_time_env(): + benv = CompileTimeScope() + names = ('UNAME_SYSNAME', 'UNAME_NODENAME', 'UNAME_RELEASE', + 'UNAME_VERSION', 'UNAME_MACHINE') + for name, value in zip(names, platform.uname()): + benv.declare(name, value) + import __builtin__ + names = ('False', 'True', + 'abs', 'bool', 'chr', 'cmp', 'complex', 'dict', 'divmod', 'enumerate', + 'float', 'hash', 'hex', 'int', 'len', 'list', 'long', 'map', 'max', 'min', + 'oct', 'ord', 'pow', 'range', 'reduce', 'repr', 'round', 'slice', 'str', + 'sum', 'tuple', 'xrange', 'zip') + for name in names: + benv.declare(name, getattr(__builtin__, name)) + denv = CompileTimeScope(benv) + return denv + +#------------------------------------------------------------------ + +class PyrexScanner(Scanner): + # context Context Compilation context + # type_names set Identifiers to be treated as type names + # included_files [string] Files included with 'include' statement + # compile_time_env dict Environment for conditional compilation + # compile_time_eval boolean In a true conditional compilation context + # compile_time_expr boolean In a compile-time expression context + + resword_dict = build_resword_dict() + + def __init__(self, file, filename, parent_scanner = None, + scope = None, context = None): + Scanner.__init__(self, get_lexicon(), file, filename) + if parent_scanner: + self.context = parent_scanner.context + self.type_names = parent_scanner.type_names + self.included_files = parent_scanner.included_files + self.compile_time_env = parent_scanner.compile_time_env + self.compile_time_eval = parent_scanner.compile_time_eval + self.compile_time_expr = parent_scanner.compile_time_expr + else: + self.context = context + self.type_names = scope.type_names + self.included_files = scope.pyrex_include_files + self.compile_time_env = initial_compile_time_env() + self.compile_time_eval = 1 + self.compile_time_expr = 0 + self.trace = trace_scanner + self.indentation_stack = [0] + self.indentation_char = None + self.bracket_nesting_level = 0 + self.begin('INDENT') + self.sy = '' + self.next() + + def current_level(self): + return self.indentation_stack[-1] + + def open_bracket_action(self, text): + self.bracket_nesting_level = self.bracket_nesting_level + 1 + return text + + def close_bracket_action(self, text): + self.bracket_nesting_level = self.bracket_nesting_level - 1 + return text + + def newline_action(self, text): + if self.bracket_nesting_level == 0: + self.begin('INDENT') + self.produce('NEWLINE', '') + + string_states = { + "'": 'SQ_STRING', + '"': 'DQ_STRING', + "'''": 'TSQ_STRING', + '"""': 'TDQ_STRING' + } + + def begin_string_action(self, text): + if text[:1] in string_prefixes: + text = text[1:] + self.begin(self.string_states[text]) + self.produce('BEGIN_STRING') + + def end_string_action(self, text): + self.begin('') + self.produce('END_STRING') + + def unclosed_string_action(self, text): + self.end_string_action(text) + self.error("Unclosed string literal") + + def indentation_action(self, text): + self.begin('') + # Indentation within brackets should be ignored. + #if self.bracket_nesting_level > 0: + # return + # Check that tabs and spaces are being used consistently. + if text: + c = text[0] + #print "Scanner.indentation_action: indent with", repr(c) ### + if self.indentation_char is None: + self.indentation_char = c + #print "Scanner.indentation_action: setting indent_char to", repr(c) + else: + if self.indentation_char <> c: + self.error("Mixed use of tabs and spaces") + if text.replace(c, "") <> "": + self.error("Mixed use of tabs and spaces") + # Figure out how many indents/dedents to do + current_level = self.current_level() + new_level = len(text) + #print "Changing indent level from", current_level, "to", new_level ### + if new_level == current_level: + return + elif new_level > current_level: + #print "...pushing level", new_level ### + self.indentation_stack.append(new_level) + self.produce('INDENT', '') + else: + while new_level < self.current_level(): + #print "...popping level", self.indentation_stack[-1] ### + self.indentation_stack.pop() + self.produce('DEDENT', '') + #print "...current level now", self.current_level() ### + if new_level <> self.current_level(): + self.error("Inconsistent indentation") + + def eof_action(self, text): + while len(self.indentation_stack) > 1: + self.produce('DEDENT', '') + self.indentation_stack.pop() + self.produce('EOF', '') + + def next(self): + try: + sy, systring = self.read() + except UnrecognizedInput: + self.error("Unrecognized character") + if sy == 'IDENT' and systring in self.resword_dict: + sy = systring + self.sy = sy + self.systring = systring + if debug_scanner: + _, line, col = self.position() + if not self.systring or self.sy == self.systring: + t = self.sy + else: + t = "%s %s" % (self.sy, self.systring) + print "--- %3d %2d %s" % (line, col, t) + + def put_back(self, sy, systring): + self.unread(self.sy, self.systring) + self.sy = sy + self.systring = systring + + def unread(self, token, value): + # This method should be added to Plex + self.queue.insert(0, (token, value)) + + def add_type_name(self, name): + self.type_names[name] = 1 + + def looking_at_type_name(self): + return self.sy == 'IDENT' and self.systring in self.type_names + + def error(self, message, pos = None): + if pos is None: + pos = self.position() + if self.sy == 'INDENT': + error(pos, "Possible inconsistent indentation") + raise error(pos, message) + + def expect(self, what, message = None): + if self.sy == what: + self.next() + else: + self.expected(what, message) + + def expect_keyword(self, what, message = None): + if self.sy == 'IDENT' and self.systring == what: + self.next() + else: + self.expected(what, message) + + def expected(self, what, message): + if message: + self.error(message) + else: + self.error("Expected '%s'" % what) + + def expect_indent(self): + self.expect('INDENT', + "Expected an increase in indentation level") + + def expect_dedent(self): + self.expect('DEDENT', + "Expected a decrease in indentation level") + + def expect_newline(self, message = "Expected a newline"): + # Expect either a newline or end of file + if self.sy <> 'EOF': + self.expect('NEWLINE', message) |