Token tok;
    start_token(&tok);
    for (;;) {
        int c = look_next_char();
        if (('A' <= c && c <= 'Z') ||
            ('a' <= c && c <= 'z')) {  /* or whatever test */
            consume_char();
            add_to_token(tok, c);
        } else {
            break;
        }
    }
    end_token(tok);

Done. There's no point in going through a weird API.