2015-11-05 08:15:46 +08:00
|
|
|
/*
|
|
|
|
* Copyright (c) 2015 Andrew Kelley
|
|
|
|
*
|
|
|
|
* This file is part of zig, which is MIT licensed.
|
|
|
|
* See http://opensource.org/licenses/MIT
|
|
|
|
*/
|
|
|
|
|
2015-11-04 13:31:27 +08:00
|
|
|
#ifndef ZIG_TOKENIZER_HPP
|
|
|
|
#define ZIG_TOKENIZER_HPP
|
|
|
|
|
|
|
|
#include "buffer.hpp"
|
|
|
|
|
|
|
|
enum TokenId {
|
|
|
|
TokenIdEof,
|
|
|
|
TokenIdSymbol,
|
|
|
|
TokenIdKeywordFn,
|
|
|
|
TokenIdKeywordReturn,
|
|
|
|
TokenIdKeywordMut,
|
|
|
|
TokenIdKeywordConst,
|
2015-11-24 17:43:45 +08:00
|
|
|
TokenIdKeywordExtern,
|
2015-11-04 13:31:27 +08:00
|
|
|
TokenIdLParen,
|
|
|
|
TokenIdRParen,
|
|
|
|
TokenIdComma,
|
|
|
|
TokenIdStar,
|
|
|
|
TokenIdLBrace,
|
|
|
|
TokenIdRBrace,
|
|
|
|
TokenIdStringLiteral,
|
|
|
|
TokenIdSemicolon,
|
|
|
|
TokenIdNumberLiteral,
|
|
|
|
TokenIdPlus,
|
|
|
|
TokenIdColon,
|
|
|
|
TokenIdArrow,
|
|
|
|
TokenIdDash,
|
|
|
|
};
|
|
|
|
|
|
|
|
struct Token {
|
|
|
|
TokenId id;
|
|
|
|
int start_pos;
|
|
|
|
int end_pos;
|
|
|
|
int start_line;
|
|
|
|
int start_column;
|
|
|
|
};
|
|
|
|
|
|
|
|
enum TokenizeState {
|
|
|
|
TokenizeStateStart,
|
|
|
|
TokenizeStateSymbol,
|
|
|
|
TokenizeStateNumber,
|
|
|
|
TokenizeStateString,
|
|
|
|
TokenizeStateSawDash,
|
|
|
|
};
|
|
|
|
|
|
|
|
ZigList<Token> *tokenize(Buf *buf, Buf *cur_dir_path);
|
|
|
|
|
|
|
|
void print_tokens(Buf *buf, ZigList<Token> *tokens);
|
|
|
|
|
|
|
|
#endif
|