import re
from ppci import Token

def tokenize(tok_spec, txt):
    tok_re = '|'.join('(?P<{}>{})'.format(pair[0], pair[1]) for pair in tok_spec)
    gettok = re.compile(tok_re).match
    func_map = {pair[0]: pair[2] for pair in tok_spec}

    # Parse line:
    line = txt
    mo = gettok(line)
    pos = 0
    while mo:
        typ = mo.lastgroup
        val = mo.group(typ)
        func = func_map[typ]
        if func:
            typ, val = func(typ, val)
            yield Token(typ, val)
        pos = mo.end()
        mo = gettok(line, pos)
    if len(line) != pos:
        raise ParserException('Lex fault at {}'.format(line[pos:]))