43 lines
671 B
C++
43 lines
671 B
C++
#pragma once
|
|
#ifndef TOKENIZER_HPP
|
|
#define TOKENIZER_HPP
|
|
|
|
#include <regex>
|
|
#include <unordered_map>
|
|
#include <utility>
|
|
#include <vector>
|
|
|
|
typedef enum {
|
|
PREPROC,
|
|
FUNCTION,
|
|
SUB,
|
|
END,
|
|
AS,
|
|
TYPE,
|
|
IDENTIFIER,
|
|
INTEGER,
|
|
STRING,
|
|
OPAREN,
|
|
CPAREN,
|
|
COMMA,
|
|
QUOTE,
|
|
EQUALS,
|
|
TOKEN_TYPE_COUNT
|
|
} PBTokenType;
|
|
|
|
typedef struct {
|
|
PBTokenType type;
|
|
std::string value;
|
|
} PBToken;
|
|
|
|
class Tokenizer {
|
|
std::vector<std::pair<std::string, std::regex>> tokentypes;
|
|
std::string code;
|
|
public:
|
|
Tokenizer(std::string);
|
|
std::vector<PBToken> tokenize();
|
|
PBToken tokenize_one_token();
|
|
std::string dump();
|
|
};
|
|
#endif
|