20 lines
393 B
C++
20 lines
393 B
C++
#include "tokenizer.hpp"
|
|
|
|
Tokenizer::Tokenizer(std::string code) {
|
|
this->code = code;
|
|
}
|
|
|
|
std::vector<PBToken> Tokenizer::tokenize() {
|
|
std::vector<PBToken> tokens;
|
|
tokens.push_back(tokenize_one_token());
|
|
return tokens;
|
|
}
|
|
|
|
PBToken Tokenizer::tokenize_one_token() {
|
|
return { .type = FUNCTION, .value = "Function" };
|
|
}
|
|
|
|
std::string Tokenizer::dump() {
|
|
return this->code;
|
|
}
|