diff --git a/peglib.h b/peglib.h index e3d5121..1ed6cef 100644 --- a/peglib.h +++ b/peglib.h @@ -1015,48 +1015,63 @@ struct AssignIDToDefinition : public Ope::Visitor op->accept(*this); } } - void visit(ZeroOrMore& ope) override { - ope.ope_->accept(*this); - } - void visit(OneOrMore& ope) override { - ope.ope_->accept(*this); - } - void visit(Option& ope) override { - ope.ope_->accept(*this); - } - void visit(AndPredicate& ope) override { - ope.ope_->accept(*this); - } - void visit(NotPredicate& ope) override { - ope.ope_->accept(*this); - } - void visit(LiteralString& ope) override { - } - void visit(CharacterClass& ope) override { - } - void visit(Character& ope) override { - } - void visit(AnyCharacter& ope) override { - } - void visit(Capture& ope) override { - ope.ope_->accept(*this); - } - void visit(Anchor& ope) override { - ope.ope_->accept(*this); - } - void visit(User& ope) override { - } - void visit(WeakHolder& ope) override { - ope.weak_.lock()->accept(*this); - } + void visit(ZeroOrMore& ope) override { ope.ope_->accept(*this); } + void visit(OneOrMore& ope) override { ope.ope_->accept(*this); } + void visit(Option& ope) override { ope.ope_->accept(*this); } + void visit(AndPredicate& ope) override { ope.ope_->accept(*this); } + void visit(NotPredicate& ope) override { ope.ope_->accept(*this); } + void visit(LiteralString& ope) override {} + void visit(CharacterClass& ope) override {} + void visit(Character& ope) override {} + void visit(AnyCharacter& ope) override {} + void visit(Capture& ope) override { ope.ope_->accept(*this); } + void visit(Anchor& ope) override { ope.ope_->accept(*this); } + void visit(User& ope) override {} + void visit(WeakHolder& ope) override { ope.weak_.lock()->accept(*this); } void visit(Holder& ope) override; - void visit(DefinitionReference& ope) override { - ope.get_rule()->accept(*this); - } + void visit(DefinitionReference& ope) override { ope.get_rule()->accept(*this); } std::unordered_map ids; }; +struct IsToken : public Ope::Visitor +{ + IsToken() : has_anchor(false), has_rule(false) {} + + void visit(Sequence& ope) override { + for (auto op: ope.opes_) { + op->accept(*this); + } + } + void visit(PrioritizedChoice& ope) override { + for (auto op: ope.opes_) { + op->accept(*this); + } + } + void visit(ZeroOrMore& ope) override { ope.ope_->accept(*this); } + void visit(OneOrMore& ope) override { ope.ope_->accept(*this); } + void visit(Option& ope) override { ope.ope_->accept(*this); } + void visit(AndPredicate& ope) override { ope.ope_->accept(*this); } + void visit(NotPredicate& ope) override { ope.ope_->accept(*this); } + void visit(LiteralString& ope) override {} + void visit(CharacterClass& ope) override {} + void visit(Character& ope) override {} + void visit(AnyCharacter& ope) override {} + void visit(Capture& ope) override { ope.ope_->accept(*this); } + void visit(Anchor& ope) override { has_anchor = true; } + void visit(User& ope) override {} + void visit(WeakHolder& ope) override { ope.weak_.lock()->accept(*this); } + void visit(Holder& ope) override {} + void visit(DefinitionReference& ope) override { has_rule = true; } + + bool is_token() const { + return has_anchor || !has_rule; + } + + bool has_anchor; + bool has_rule; +}; + /* * Definition */ @@ -1075,6 +1090,7 @@ public: : actions(1) , ignoreSemanticValue(false) , enablePackratParsing(false) + , is_token(false) , holder_(std::make_shared(this)) {} Definition(const Definition& rhs) @@ -1082,6 +1098,7 @@ public: , actions(1) , ignoreSemanticValue(false) , enablePackratParsing(false) + , is_token(false) , holder_(rhs.holder_) { holder_->outer_ = this; @@ -1092,6 +1109,7 @@ public: , actions(1) , ignoreSemanticValue(rhs.ignoreSemanticValue) , enablePackratParsing(rhs.enablePackratParsing) + , is_token(rhs.is_token) , holder_(std::move(rhs.holder_)) { holder_->outer_ = this; @@ -1101,9 +1119,10 @@ public: : actions(1) , ignoreSemanticValue(false) , enablePackratParsing(false) + , is_token(false) , holder_(std::make_shared(this)) { - holder_->ope_ = ope; + *this <= ope; } operator std::shared_ptr() { @@ -1111,7 +1130,12 @@ public: } Definition& operator<=(const std::shared_ptr& ope) { + IsToken isToken; + ope->accept(isToken); + is_token = isToken.is_token(); + holder_->ope_ = ope; + return *this; } @@ -1202,6 +1226,7 @@ public: std::function error_message; bool ignoreSemanticValue; bool enablePackratParsing; + bool is_token; private: friend class DefinitionReference; diff --git a/test/test.cc b/test/test.cc index 49d5973..c6d2f5a 100644 --- a/test/test.cc +++ b/test/test.cc @@ -178,6 +178,25 @@ TEST_CASE("Visit test", "[general]") REQUIRE(defIds.ids.size() == 4); } +TEST_CASE("Token check test", "[general]") +{ + peg parser( + " EXPRESSION <- _ TERM (TERM_OPERATOR TERM)* " + " TERM <- FACTOR (FACTOR_OPERATOR FACTOR)* " + " FACTOR <- NUMBER / '(' _ EXPRESSION ')' _ " + " TERM_OPERATOR <- < [-+] > _ " + " FACTOR_OPERATOR <- < [/*] > _ " + " NUMBER <- < [0-9]+ > _ " + " ~_ <- [ \t\r\n]* " + ); + + REQUIRE(parser["EXPRESSION"].is_token == false); + REQUIRE(parser["FACTOR"].is_token == false); + REQUIRE(parser["FACTOR_OPERATOR"].is_token == true); + REQUIRE(parser["NUMBER"].is_token == true); + REQUIRE(parser["_"].is_token == true); +} + TEST_CASE("Lambda action test", "[general]") { peg parser(