AIR/cpp-peglib
1
0
Fork 0
mirror of https://github.com/yhirose/cpp-peglib.git synced 2024-12-22 11:55:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix #68

Browse Source
This commit is contained in:
yhirose 2019-08-27 02:44:04 -04:00
parent 69749237c9
commit d75ced2408
2 changed files with 79 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
peglib.h
View File

@ -1674,6 +1674,33 @@ struct AssignIDToDefinition : public Ope::Visitor
std::unordered_map<void*, size_t> ids;
};
struct IsLiteralToken : public Ope::Visitor
{
IsLiteralToken() : result_(false) {}
void visit(PrioritizedChoice& ope) override {
for (auto op: ope.opes_) {
if (!IsLiteralToken::check(*op)) {
return;
}
}
result_ = true;
}
void visit(LiteralString& /*ope*/) override {
result_ = true;
}
static bool check(Ope& ope) {
IsLiteralToken vis;
ope.accept(vis);
return vis.result_;
}
private:
bool result_;
};
struct TokenChecker : public Ope::Visitor
{
TokenChecker() : has_token_boundary_(false), has_rule_(false) {}
@ -1700,6 +1727,10 @@ struct TokenChecker : public Ope::Visitor
void visit(Whitespace& ope) override { ope.ope_->accept(*this); }
static bool is_token(Ope& ope) {
if (IsLiteralToken::check(ope)) {
return true;
}
TokenChecker vis;
ope.accept(vis);
return vis.has_token_boundary_ || !vis.has_rule_;
@ -1884,20 +1915,20 @@ private:
struct IsPrioritizedChoice : public Ope::Visitor
{
IsPrioritizedChoice() : is_prioritized_choice_(false) {}
IsPrioritizedChoice() : result_(false) {}
void visit(PrioritizedChoice& /*ope*/) override {
is_prioritized_choice_ = true;
result_ = true;
}
static bool is_prioritized_choice(Ope& ope) {
static bool check(Ope& ope) {
IsPrioritizedChoice vis;
ope.accept(vis);
return vis.is_prioritized_choice_;
return vis.result_;
}
private:
bool is_prioritized_choice_;
bool result_;
};
/*
@ -2218,7 +2249,7 @@ inline size_t Holder::parse(const char* s, size_t n, SemanticValues& sv, Context
chldsv.n_ = len;
chldsv.name_ = outer_->name;
if (!IsPrioritizedChoice::is_prioritized_choice(*ope_)) {
if (!IsPrioritizedChoice::check(*ope_)) {
chldsv.choice_count_ = 0;
chldsv.choice_ = 0;
}
@ -2885,6 +2916,14 @@ private:
// Automatic whitespace skipping
if (grammar.count(WHITESPACE_DEFINITION_NAME)) {
for (auto& x: grammar) {
auto& rule = x.second;
auto ope = rule.get_core_operator();
if (IsLiteralToken::check(*ope)) {
rule <= tok(ope);
}
}
auto& rule = (*data.grammar)[start];
rule.whitespaceOpe = wsp((*data.grammar)[WHITESPACE_DEFINITION_NAME].get_core_operator());
}

43
test/test.cc
View File

@ -155,15 +155,15 @@ TEST_CASE("Visit test", "[general]")
TEST_CASE("Token check test", "[general]")
{
parser parser(
" EXPRESSION <- _ TERM (TERM_OPERATOR TERM)* "
" TERM <- FACTOR (FACTOR_OPERATOR FACTOR)* "
" FACTOR <- NUMBER / '(' _ EXPRESSION ')' _ "
" TERM_OPERATOR <- < [-+] > _ "
" FACTOR_OPERATOR <- < [/*] > _ "
" NUMBER <- < [0-9]+ > _ "
" _ <- [ \t\r\n]* "
);
parser parser(R"(
EXPRESSION <- _ TERM (TERM_OPERATOR TERM)*
TERM <- FACTOR (FACTOR_OPERATOR FACTOR)*
FACTOR <- NUMBER / '(' _ EXPRESSION ')' _
TERM_OPERATOR <- < [-+] > _
FACTOR_OPERATOR <- < [/*] > _
NUMBER <- < [0-9]+ > _
_ <- [ \t\r\n]*
)");
REQUIRE(parser["EXPRESSION"].is_token() == false);
REQUIRE(parser["FACTOR"].is_token() == false);
@ -295,6 +295,31 @@ TEST_CASE("WHITESPACE test3", "[general]") {
REQUIRE(ret == true);
}
TEST_CASE("WHITESPACE test4", "[general]") {
peg::parser parser(R"(
ROOT <- HELLO OPE WORLD
HELLO <- 'hello'
OPE <- < [-+] >
WORLD <- 'world' / 'WORLD'
%whitespace <- [ \t\r\n]*
)");
parser["HELLO"] = [](const SemanticValues& sv) {
REQUIRE(sv.token() == "hello");
};
parser["OPE"] = [](const SemanticValues& sv) {
REQUIRE(sv.token() == "+");
};
parser["WORLD"] = [](const SemanticValues& sv) {
REQUIRE(sv.token() == "world");
};
auto ret = parser.parse(" hello + world ");
REQUIRE(ret == true);
}
TEST_CASE("Word expression test", "[general]") {
peg::parser parser(R"(
ROOT <- 'hello' ','? 'world'