AIR
/
cpp-peglib
mirror of https://github.com/yhirose/cpp-peglib.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

C++17 support

Browse Source
pull/124/head
yhirose 4 years ago
parent b92da07bed
commit b26e2477ff
22 changed files with 1942 additions and 2295 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 49
      .github/workflows/cmake.yml
  2. 33
      .travis.yml
  3. 32
      CMakeLists.txt
  4. 95
      README.md
  5. 3
      appveyor.yml
  6. 2
      docs/build.sh
  7. 2
      docs/native.js
  8. BIN
      docs/native.wasm
  9. 8
      example/CMakeLists.txt
  10. 22
      example/calc.cc
  11. 26
      example/calc2.cc
  12. 9
      example/calc3.cc
  13. 16
      example/calc4.cc
  14. 9
      example/calc5.cc
  15. 10
      lint/CMakeLists.txt
  16. 9
      lint/peglint.cc
  17. 1326
      peglib.h
  18. 11
      pl0/CMakeLists.txt
  19. 56
      pl0/pl0.cc
  20. 13
      test/CMakeLists.txt
  21. 377
      test/test1.cc
  22. 271
      test/test2.cc

49
.github/workflows/cmake.yml
Unescape View File

@ -0,0 +1,49 @@
name: CMake
on: [push, pull_request]
env:
# Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.)
BUILD_TYPE: Release
jobs:
build:
# The CMake configure and build commands are platform agnostic and should work equally
# well on Windows or Mac. You can convert this to a matrix build if you need
# cross-platform coverage.
# See: https://docs.github.com/en/actions/configuring-and-managing-workflows/configuring-a-workflow#configuring-a-build-matrix
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ubuntu-20.04, macos-latest, windows-latest]
steps:
- uses: actions/checkout@v2
- name: Create Build Environment
# Some projects don't allow in-source building, so create a separate build directory
# We'll use this as our working directory for all subsequent commands
run: cmake -E make_directory ${{runner.workspace}}/build
- name: Configure CMake
# Use a bash shell so we can use the same syntax for environment variable
# access regardless of the host operating system
shell: bash
working-directory: ${{runner.workspace}}/build
# Note the current convention is to use the -S and -B options here to specify source
# and build directories, but this is only available with CMake 3.13 and higher.
# The CMake binaries on the Github Actions machines are (as of this writing) 3.12
run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE
- name: Build
working-directory: ${{runner.workspace}}/build
shell: bash
# Execute the build. You can specify a specific target with "--target <NAME>"
run: cmake --build . --config $BUILD_TYPE
- name: Test
working-directory: ${{runner.workspace}}/build
shell: bash
# Execute tests defined by the CMake configuration.
# See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail
run: ctest -C $BUILD_TYPE

33
.travis.yml
Unescape View File

@ -1,39 +1,20 @@
language: cpp
sudo: false
matrix:
include:
- compiler: gcc
- os: linux
addons:
apt:
sources:
- ubuntu-toolchain-r-test
- george-edison55-precise-backports
packages:
- cmake
- cmake-data
- gcc-4.9
- g++-4.9
env: COMPILER=g++-4.9
# - compiler: clang
# addons:
# apt:
# sources:
# - kubuntu-backports
# - ubuntu-toolchain-r-test
# - llvm-toolchain-precise-3.7
# packages:
# - cmake
# - clang-3.7
# env: COMPILER=clang++-3.7
- g++-8
env:
- MATRIX_EVAL="CC=gcc-8 && CXX=g++-8"
branches:
only:
- master
before_script:
- export CXX=$COMPILER
before_install:
- eval "${MATRIX_EVAL}"
script:
- mkdir build && cd build
- cmake .. && make && ctest -V
- cmake .. && make && ./test/test-main

32
CMakeLists.txt
Unescape View File

@ -1,33 +1,11 @@
cmake_minimum_required(VERSION 3.1.0)
project("cpp-peglib")
# Check if a supported compiler is used to setup the C++ standard to use:
get_property(known_features GLOBAL PROPERTY CMAKE_CXX_KNOWN_FEATURES)
list(FIND known_features "cxx_std_17" found)
if(NOT ${found} EQUAL -1)
# C++17 standard is supported
set(CMAKE_CXX_STANDARD 17)
else()
# Check for C++11 standard support
list(FIND known_features "cxx_std_11" found)
if(NOT ${found} EQUAL -1)
# C++11 standard is supported
set(CMAKE_CXX_STANDARD 11)
endif()
endif()
if(${found} EQUAL -1)
message(FATAL_ERROR "Your compiler is not supported.")
endif()
set(CMAKE_CXX_EXTENSIONS OFF)
if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang" OR CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Weverything -Wno-c++98-compat -Wno-padded -Wno-weak-vtables -Wno-exit-time-destructors -Wno-c++2a-compat -Wno-switch-enum -Wno-c++98-compat-pedantic")
elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wpedantic -Wextra -Woverloaded-virtual")
elseif(MSVC)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /W4 /wd4503 /wd4512 /utf-8")
if(MSVC)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zc:__cplusplus /utf-8")
endif()
set(THREADS_PREFER_PTHREAD_FLAG ON)
@ -37,10 +15,10 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
set(add_link_deps Threads::Threads)
endif()
enable_testing()
add_subdirectory(test)
add_subdirectory(example)
add_subdirectory(lint)
add_subdirectory(test)
enable_testing()
install(FILES peglib.h DESTINATION include)

95
README.md
Unescape View File

@ -1,10 +1,13 @@
cpp-peglib
==========
[![](https://github.com/yhirose/cpp-peglib/workflows/CMake/badge.svg)](https://github.com/yhirose/cpp-peglib/actions)
[![Build Status](https://travis-ci.org/yhirose/cpp-peglib.svg?branch=master)](https://travis-ci.org/yhirose/cpp-peglib)
[![Bulid Status](https://ci.appveyor.com/api/projects/status/github/yhirose/cpp-peglib?branch=master&svg=true)](https://ci.appveyor.com/project/yhirose/cpp-peglib)
C++11 header-only [PEG](http://en.wikipedia.org/wiki/Parsing_expression_grammar) (Parsing Expression Grammars) library. You can start using it right away just by including `peglib.h` in your project.
C++17 header-only [PEG](http://en.wikipedia.org/wiki/Parsing_expression_grammar) (Parsing Expression Grammars) library. You can start using it right away just by including `peglib.h` in your project.
Since this library only supports C++17 compilers, please make sure that compiler the option `-std=c++17` is enabled. (`/std:c++17 /Zc:__cplusplus` for MSVC)
You can also try the online version, PEG Playground at https://yhirose.github.io/cpp-peglib.
@ -54,29 +57,29 @@ int main(void) {
%whitespace <- [ \t]*
)");
assert((bool)parser == true);
assert(static_cast<bool>(parser) == true);
// (3) Setup actions
parser["Additive"] = [](const SemanticValues& sv) {
switch (sv.choice()) {
parser["Additive"] = [](const SemanticValues &vs) {
switch (vs.choice()) {
case 0: // "Multitive '+' Additive"
return any_cast<int>(sv[0]) + any_cast<int>(sv[1]);
return any_cast<int>(vs[0]) + any_cast<int>(vs[1]);
default: // "Multitive"
return any_cast<int>(sv[0]);
return any_cast<int>(vs[0]);
}
};
parser["Multitive"] = [](const SemanticValues& sv) {
switch (sv.choice()) {
parser["Multitive"] = [](const SemanticValues &vs) {
switch (vs.choice()) {
case 0: // "Primary '*' Multitive"
return any_cast<int>(sv[0]) * any_cast<int>(sv[1]);
return any_cast<int>(vs[0]) * any_cast<int>(vs[1]);
default: // "Primary"
return any_cast<int>(sv[0]);
return any_cast<int>(vs[0]);
}
};
parser["Number"] = [](const SemanticValues& sv) {
return stoi(sv.token(), nullptr, 10);
parser["Number"] = [](const SemanticValues &vs) {
return vs.token_to_number<int>();
};
// (4) Parse
@ -114,10 +117,10 @@ assert(ok);
There are four semantic actions available:
```cpp
[](const SemanticValues& sv, any& dt)
[](const SemanticValues& sv)
[](SemanticValues& sv, any& dt)
[](SemanticValues& sv)
[](const SemanticValues& vs, any& dt)
[](const SemanticValues& vs)
[](SemanticValues& vs, any& dt)
[](SemanticValues& vs)
```
`SemanticValues` value contains the following information:
@ -129,16 +132,7 @@ There are four semantic actions available:
`any& dt` is a 'read-write' context data which can be used for whatever purposes. The initial context data is set in `peg::parser::parse` method.
`peg::any` is a simpler implementatin of std::any. If the compiler in use supports C++17, by default `peg::any` is defined as an alias to `std::any`.
To force using the simpler `any` implementation that comes with `cpp-peglib`, define `PEGLIB_USE_STD_ANY` as 0 before including `peglib.h`:
```cpp
#define PEGLIB_USE_STD_ANY 0
#include <peglib.h>
[...]
```
A semantic action can return a value of arbitrary data type, which will be wrapped by `peg::any`. If a user returns nothing in a semantic action, the first semantic value in the `const SemanticValues& sv` argument will be returned. (Yacc parser has the same behavior.)
A semantic action can return a value of arbitrary data type, which will be wrapped by `peg::any`. If a user returns nothing in a semantic action, the first semantic value in the `const SemanticValues& vs` argument will be returned. (Yacc parser has the same behavior.)
Here shows the `SemanticValues` structure:
@ -150,21 +144,18 @@ struct SemanticValues : protected std::vector<any>
const char* ss;
// Matched string
std::string str() const; // Matched string
const char* c_str() const; // Matched string start
size_t length() const; // Matched string length
std::string_view sv() const { return sv_; }
// Line number and column at which the matched string is
std::pair<size_t, size_t> line_info() const;
// Tokens
std::vector<
std::pair<
const char*, // Token start
size_t>> // Token length
tokens;
std::vector<std::string_view> tokens;
std::string_view token(size_t id = 0) const;
std::string token(size_t id = 0) const;
// Token conversion
std::string token_to_string(size_t id = 0) const;
template <typename T> T token_to_number() const;
// Choice number (0 based index)
size_t choice() const;
@ -183,9 +174,9 @@ peg::parser parser(R"(
_ <- [ \t\r\n]*
)");
parser["TOKEN"] = [](const SemanticValues& sv) {
parser["TOKEN"] = [](const SemanticValues& vs) {
// 'token' doesn't include trailing whitespaces
auto token = sv.token();
auto token = vs.token();
};
auto ret = parser.parse(" token1, token2 ");
@ -200,8 +191,8 @@ peg::parser parser(R"(
~_ <- [ \t]*
)");
parser["ROOT"] = [&](const SemanticValues& sv) {
assert(sv.size() == 2); // should be 2 instead of 5.
parser["ROOT"] = [&](const SemanticValues& vs) {
assert(vs.size() == 2); // should be 2 instead of 5.
};
auto ret = parser.parse(" item1, item2 ");
@ -222,8 +213,8 @@ peg::parser parser(R"(
```cpp
peg::parser parser("NUMBER <- [0-9]+");
parser["NUMBER"] = [](const SemanticValues& sv) {
auto val = stol(sv.str(), nullptr, 10);
parser["NUMBER"] = [](const SemanticValues& vs) {
auto val = vs.token_to_number<long>();
if (val != 100) {
throw peg::parse_error("value error!!");
}
@ -246,7 +237,7 @@ parser["RULE"].enter = [](const char* s, size_t n, any& dt) {
std::cout << "enter" << std::endl;
};
parser["RULE"] = [](const SemanticValues& sv, any& dt) {
parser["RULE"] = [](const SemanticValues& vs, any& dt) {
std::cout << "action!" << std::endl;
};
@ -373,11 +364,11 @@ parser parser(R"(
}
)");
parser["INFIX_EXPRESSION"] = [](const SemanticValues& sv) -> long {
auto result = any_cast<long>(sv[0]);
if (sv.size() > 1) {
auto ope = any_cast<char>(sv[1]);
auto num = any_cast<long>(sv[2]);
parser["INFIX_EXPRESSION"] = [](const SemanticValues& vs) -> long {
auto result = any_cast<long>(vs[0]);
if (vs.size() > 1) {
auto ope = any_cast<char>(vs[1]);
auto num = any_cast<long>(vs[2]);
switch (ope) {
case '+': result += num; break;
case '-': result -= num; break;
@ -387,8 +378,8 @@ parser["INFIX_EXPRESSION"] = [](const SemanticValues& sv) -> long {
}
return result;
};
parser["OPERATOR"] = [](const SemanticValues& sv) { return *sv.c_str(); };
parser["NUMBER"] = [](const SemanticValues& sv) { return atol(sv.c_str()); };
parser["OPERATOR"] = [](const SemanticValues& vs) { return *vs.sv(); };
parser["NUMBER"] = [](const SemanticValues& vs) { return vs.token_to_number<long>(); };
long val;
parser.parse(" -1 + (1 + 2) * 3 - -1", val);
@ -446,8 +437,8 @@ vector<string> tags;
Definition ROOT, TAG_NAME, _;
ROOT <= seq(_, zom(seq(chr('['), TAG_NAME, chr(']'), _)));
TAG_NAME <= oom(seq(npd(chr(']')), dot())), [&](const SemanticValues& sv) {
tags.push_back(sv.str());
TAG_NAME <= oom(seq(npd(chr(']')), dot())), [&](const SemanticValues& vs) {
tags.push_back(vs.str());
};
_ <= zom(cls(" \t"));
@ -492,7 +483,7 @@ auto syntax = R"(
Rules additional_rules = {
{
"NAME", usr([](const char* s, size_t n, SemanticValues& sv, any& dt) -> size_t {
"NAME", usr([](const char* s, size_t n, SemanticValues& vs, any& dt) -> size_t {
static vector<string> names = { "PEG", "BNF" };
for (const auto& name: names) {
if (name.size() <= n && !name.compare(0, name.size(), s, name.size())) {

3
appveyor.yml
Unescape View File

@ -2,9 +2,6 @@ clone_depth: 5
environment:
matrix:
- JOB: Visual Studio 2015
APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
CMAKE_GENERATOR: "Visual Studio 14 2015"
- JOB: Visual Studio 2017
APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
CMAKE_GENERATOR: "Visual Studio 15 2017"

2
docs/build.sh
Unescape View File

@ -1,2 +1,2 @@
source ~/Projects/emsdk/emsdk_env.sh
emcc -std=c++11 -O3 --bind -o native.js native.cpp
emcc -std=c++17 -O3 --bind -o native.js native.cpp

2
docs/native.js
View File

File diff suppressed because one or more lines are too long

BIN
docs/native.wasm
View File

Binary file not shown.

8
example/CMakeLists.txt
Unescape View File

@ -1,13 +1,9 @@
cmake_minimum_required(VERSION 2.8)
cmake_minimum_required(VERSION 3.1)
project(example)
include_directories(..)
if(MSVC)
add_compile_options(${cxx11_options} /W3)
else()
add_compile_options(${cxx11_options})
endif()
set(CMAKE_CXX_STANDARD 17)
add_executable(calc calc.cc)
target_link_libraries(calc ${add_link_deps})

22
example/calc.cc
Unescape View File

@ -1,6 +1,6 @@
#include <peglib.h>
#include <assert.h>
#include <iostream>
#include <peglib.h>
using namespace peg;
using namespace std;
@ -19,26 +19,26 @@ int main(void) {
assert(static_cast<bool>(parser) == true);
// (3) Setup actions
parser["Additive"] = [](const SemanticValues& sv) {
switch (sv.choice()) {
parser["Additive"] = [](const SemanticValues &vs) {
switch (vs.choice()) {
case 0: // "Multitive '+' Additive"
return any_cast<int>(sv[0]) + any_cast<int>(sv[1]);
return any_cast<int>(vs[0]) + any_cast<int>(vs[1]);
default: // "Multitive"
return any_cast<int>(sv[0]);
return any_cast<int>(vs[0]);
}
};
parser["Multitive"] = [](const SemanticValues& sv) {
switch (sv.choice()) {
parser["Multitive"] = [](const SemanticValues &vs) {
switch (vs.choice()) {
case 0: // "Primary '*' Multitive"
return any_cast<int>(sv[0]) * any_cast<int>(sv[1]);
return any_cast<int>(vs[0]) * any_cast<int>(vs[1]);
default: // "Primary"
return any_cast<int>(sv[0]);
return any_cast<int>(vs[0]);
}
};
parser["Number"] = [](const SemanticValues& sv) {
return stoi(sv.token(), nullptr, 10);
parser["Number"] = [](const SemanticValues &vs) {
return vs.token_to_number<int>();
};
// (4) Parse

26
example/calc2.cc
Unescape View File

@ -5,9 +5,9 @@
// MIT License
//
#include <peglib.h>
#include <iostream>
#include <cstdlib>
#include <iostream>
#include <peglib.h>
using namespace peg;
@ -21,18 +21,17 @@ using namespace peg;
// FACTOR_OPERATOR <- [/*]
// NUMBER <- [0-9]+
//
int main(int argc, const char** argv)
{
int main(int argc, const char **argv) {
if (argc < 2 || std::string("--help") == argv[1]) {
std::cout << "usage: calc [formula]" << std::endl;
return 1;
}
auto reduce = [](const SemanticValues& sv) -> long {
auto result = any_cast<long>(sv[0]);
for (auto i = 1u; i < sv.size(); i += 2) {
auto num = any_cast<long>(sv[i + 1]);
auto ope = any_cast<char>(sv[i]);
auto reduce = [](const SemanticValues &vs) {
auto result = std::any_cast<long>(vs[0]);
for (auto i = 1u; i < vs.size(); i += 2) {
auto num = std::any_cast<long>(vs[i + 1]);
auto ope = std::any_cast<char>(vs[i]);
switch (ope) {
case '+': result += num; break;
case '-': result -= num; break;
@ -48,9 +47,12 @@ int main(int argc, const char** argv)
EXPRESSION <= seq(TERM, zom(seq(TERM_OPERATOR, TERM))), reduce;
TERM <= seq(FACTOR, zom(seq(FACTOR_OPERATOR, FACTOR))), reduce;
FACTOR <= cho(NUMBER, seq(chr('('), EXPRESSION, chr(')')));
TERM_OPERATOR <= cls("+-"), [](const SemanticValues& sv) { return static_cast<char>(*sv.c_str()); };
FACTOR_OPERATOR <= cls("*/"), [](const SemanticValues& sv) { return static_cast<char>(*sv.c_str()); };
NUMBER <= oom(cls("0-9")), [](const SemanticValues& sv) { return atol(sv.c_str()); };
TERM_OPERATOR <= cls("+-"),
[](const SemanticValues &vs) { return static_cast<char>(*vs.sv().data()); };
FACTOR_OPERATOR <= cls("*/"),
[](const SemanticValues &vs) { return static_cast<char>(*vs.sv().data()); };
NUMBER <= oom(cls("0-9")),
[](const SemanticValues &vs) { return vs.token_to_number<long>(); };
auto expr = argv[1];
long val = 0;

9
example/calc3.cc
Unescape View File

@ -5,14 +5,13 @@
// MIT License
//
#include <peglib.h>
#include <iostream>
#include <cstdlib>
#include <iostream>
#include <peglib.h>
using namespace peg;
int main(int argc, const char** argv)
{
int main(int argc, const char **argv) {
if (argc < 2 || std::string("--help") == argv[1]) {
std::cout << "usage: calc3 [formula]" << std::endl;
return 1;
@ -20,7 +19,7 @@ int main(int argc, const char** argv)
std::function<long(const Ast &)> eval = [&](const Ast &ast) {
if (ast.name == "NUMBER") {
return stol(ast.token);
return ast.token_to_number<long>();
} else {
const auto &nodes = ast.nodes;
auto result = eval(*nodes[0]);

16
example/calc4.cc
Unescape View File

@ -1,6 +1,6 @@
#include <peglib.h>
#include <assert.h>
#include <iostream>
#include <peglib.h>
using namespace peg;
using namespace std;
@ -18,11 +18,11 @@ int main(void) {
%whitespace <- [ \t\r\n]*
)");
parser["EXPRESSION"] = [](const SemanticValues& sv) -> long {
auto result = any_cast<long>(sv[0]);
if (sv.size() > 1) {
auto ope = any_cast<char>(sv[1]);
auto num = any_cast<long>(sv[2]);
parser["EXPRESSION"] = [](const SemanticValues &vs) {
auto result = any_cast<long>(vs[0]);
if (vs.size() > 1) {
auto ope = any_cast<char>(vs[1]);
auto num = any_cast<long>(vs[2]);
switch (ope) {
case '+': result += num; break;
case '-': result -= num; break;
@ -32,8 +32,8 @@ int main(void) {
}
return result;
};
parser["OPERATOR"] = [](const SemanticValues& sv) { return *sv.c_str(); };
parser["NUMBER"] = [](const SemanticValues& sv) { return atol(sv.c_str()); };
parser["OPERATOR"] = [](const SemanticValues &vs) { return *vs.sv().data(); };
parser["NUMBER"] = [](const SemanticValues &vs) { return atol(vs.sv().data()); };
long val;
parser.parse(" -1 + (1 + 2) * 3 - -1", val);

9
example/calc5.cc
Unescape View File

@ -5,14 +5,13 @@
// MIT License
//
#include <peglib.h>
#include <iostream>
#include <cstdlib>
#include <iostream>
#include <peglib.h>
using namespace peg;
int main(int argc, const char** argv)
{
int main(int argc, const char **argv) {
if (argc < 2 || std::string("--help") == argv[1]) {
std::cout << "usage: calc5 [formula]" << std::endl;
return 1;
@ -20,7 +19,7 @@ int main(int argc, const char** argv)
std::function<long(const Ast &)> eval = [&](const Ast &ast) {
if (ast.name == "NUMBER") {
return stol(ast.token);
return ast.token_to_number<long>();
} else {
const auto &nodes = ast.nodes;
auto result = eval(*nodes[0]);

10
lint/CMakeLists.txt
Unescape View File

@ -1,9 +1,9 @@
cmake_minimum_required(VERSION 2.8)
cmake_minimum_required(VERSION 3.1)
project(peglint)
include_directories(..)
add_definitions("-std=c++11")
add_executable(peglint peglint.cc)
if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
set(CMAKE_CXX_STANDARD 17)
add_executable(peglint peglint.cc)
target_link_libraries(peglint ${add_link_deps})
endif()

9
lint/peglint.cc
Unescape View File

@ -131,7 +131,7 @@ int main(int argc, const char **argv) {
parser.enable_trace(
[&](const char *name, const char *s, size_t /*n*/,
const peg::SemanticValues & /*sv*/, const peg::Context &c,
const peg::any & /*dt*/) {
const std::any & /*dt*/) {
auto pos = static_cast<size_t>(s - c.s);
auto backtrack = (pos < prev_pos ? "*" : "");
string indent;
@ -145,7 +145,7 @@ int main(int argc, const char **argv) {
},
[&](const char *name, const char *s, size_t /*n*/,
const peg::SemanticValues &sv, const peg::Context &c,
const peg::any & /*dt*/, size_t len) {
const std::any & /*dt*/, size_t len) {
auto pos = static_cast<size_t>(s - c.s);
if (len != static_cast<size_t>(-1)) { pos += len; }
string indent;
@ -160,8 +160,9 @@ int main(int argc, const char **argv) {
}
std::string token;
if (!sv.tokens.empty()) {
const auto &tok = sv.tokens[0];
token += " '" + std::string(tok.first, tok.second) + "'";
token += " '";
token += sv.tokens[0];
token +=+ "'";
}
std::cout << "L " << pos << "\t" << indent << ret << name << " #"
<< c.trace_ids.back() << choice.str() << token << std::endl;

1326
peglib.h
View File

File diff suppressed because it is too large Load Diff

11
pl0/CMakeLists.txt
Unescape View File

@ -1,14 +1,11 @@
cmake_minimum_required(VERSION 3.1)
project(tcp)
enable_language(CXX)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_CXX_STANDARD 11)
set(CMAKE_CXX_EXTENSIONS OFF)
project(pl0)
include_directories(..)
add_executable(pl0 pl0.cc)
set(CMAKE_CXX_STANDARD 17)
add_executable(pl0 pl0.cc)
find_package(LLVM REQUIRED CONFIG)
set(add_link_deps ${add_link_deps} LLVM)
target_include_directories(pl0 PUBLIC ${LLVM_INCLUDE_DIRS})

56
pl0/pl0.cc
Unescape View File

@ -182,11 +182,11 @@ struct SymbolTable {
// _)?
const auto& nodes = ast->nodes;
for (auto i = 0u; i < nodes.size(); i += 2) {
const auto& ident = nodes[i + 0]->token;
const auto& ident = nodes[i + 0]->token_to_string();
if (scope->has_symbol(ident)) {
throw_runtime_error(nodes[i], "'" + ident + "' is already defined...");
}
auto number = stoi(nodes[i + 1]->token);
auto number = nodes[i + 1]->token_to_number<int>();
scope->constants.emplace(ident, number);
}
}
@ -196,7 +196,7 @@ struct SymbolTable {
// var <- ('VAR' __ ident(',' _ ident)* ';' _) ?
const auto& nodes = ast->nodes;
for (auto i = 0u; i < nodes.size(); i += 1) {
const auto& ident = nodes[i]->token;
const auto& ident = nodes[i]->token_to_string();
if (scope->has_symbol(ident)) {
throw_runtime_error(nodes[i], "'" + ident + "' is already defined...");
}
@ -209,7 +209,7 @@ struct SymbolTable {
// procedure <- ('PROCEDURE' __ ident ';' _ block ';' _)*
const auto& nodes = ast->nodes;
for (auto i = 0u; i < nodes.size(); i += 2) {
const auto& ident = nodes[i + 0]->token;
const auto& ident = nodes[i + 0]->token_to_string();
auto block = nodes[i + 1];
scope->procedures[ident] = block;
build_on_ast(block, scope);
@ -219,7 +219,7 @@ struct SymbolTable {
static void assignment(const shared_ptr<AstPL0> ast,
shared_ptr<SymbolScope> scope) {
// assignment <- ident ':=' _ expression
const auto& ident = ast->nodes[0]->token;
const auto& ident = ast->nodes[0]->token_to_string();
if (scope->has_constant(ident)) {
throw_runtime_error(ast->nodes[0],
"cannot modify constant value '" + ident + "'...");
@ -238,7 +238,7 @@ struct SymbolTable {
static void call(const shared_ptr<AstPL0> ast,
shared_ptr<SymbolScope> scope) {
// call <- 'CALL' __ ident
const auto& ident = ast->nodes[0]->token;
const auto& ident = ast->nodes[0]->token_to_string();
if (!scope->has_procedure(ident)) {
throw_runtime_error(ast->nodes[0],
"undefined procedure '" + ident + "'...");
@ -256,7 +256,7 @@ struct SymbolTable {
static void ident(const shared_ptr<AstPL0> ast,
shared_ptr<SymbolScope> scope) {
const auto& ident = ast->token;
const auto& ident = ast->token_to_string();
if (!scope->has_symbol(ident)) {
throw_runtime_error(ast, "undefined variable '" + ident + "'...");
}
@ -360,13 +360,13 @@ struct Interpreter {
static void exec_assignment(const shared_ptr<AstPL0> ast,
shared_ptr<Environment> env) {
// assignment <- ident ':=' _ expression
env->set_variable(ast->nodes[0]->token, eval(ast->nodes[1], env));
env->set_variable(ast->nodes[0]->token_to_string(), eval(ast->nodes[1], env));
}
static void exec_call(const shared_ptr<AstPL0> ast,
shared_ptr<Environment> env) {
// call <- 'CALL' __ ident
exec_block(env->get_procedure(ast->nodes[0]->token), env);
exec_block(env->get_procedure(ast->nodes[0]->token_to_string()), env);
}
static void exec_statements(const shared_ptr<AstPL0> ast,
@ -406,7 +406,7 @@ struct Interpreter {
// in <- ('in' __ / 'read' __ / '?' _) ident
int val;
cin >> val;
env->set_variable(ast->nodes[0]->token, val);
env->set_variable(ast->nodes[0]->token_to_string(), val);
}
static bool eval_condition(const shared_ptr<AstPL0> ast,
@ -434,7 +434,7 @@ struct Interpreter {
// compare <- expression compare_op expression
const auto& nodes = ast->nodes;
auto lval = eval_expression(nodes[0], env);
auto op = peg::str2tag(nodes[1]->token.c_str());
auto op = peg::str2tag(nodes[1]->token_to_string().c_str());
auto rval = eval_expression(nodes[2], env);
switch (op) {
case "="_:
@ -473,11 +473,11 @@ struct Interpreter {
shared_ptr<Environment> env) {
// expression <- sign term (term_op term)*
const auto& nodes = ast->nodes;
auto sign = nodes[0]->token;
auto sign = nodes[0]->token_to_string();
auto sign_val = (sign.empty() || sign == "+") ? 1 : -1;
auto val = eval(nodes[1], env) * sign_val;
for (auto i = 2u; i < nodes.size(); i += 2) {
auto ope = nodes[i + 0]->token[0];
auto ope = nodes[i + 0]->token_to_string()[0];
auto rval = eval(nodes[i + 1], env);
switch (ope) {
case '+':
@ -497,7 +497,7 @@ struct Interpreter {
const auto& nodes = ast->nodes;
auto val = eval(nodes[0], env);
for (auto i = 1u; i < nodes.size(); i += 2) {
auto ope = nodes[i + 0]->token[0];
auto ope = nodes[i + 0]->token_to_string()[0];
auto rval = eval(nodes[i + 1], env);
switch (ope) {
case '*':
@ -516,12 +516,12 @@ struct Interpreter {
static int eval_ident(const shared_ptr<AstPL0> ast,
shared_ptr<Environment> env) {
return env->get_value(ast, ast->token);
return env->get_value(ast, ast->token_to_string());
}
static int eval_number(const shared_ptr<AstPL0> ast,
shared_ptr<Environment> env) {
return stol(ast->token);
return stol(ast->token_to_string());
}
};
@ -653,8 +653,8 @@ struct LLVM {
void compile_const(const shared_ptr<AstPL0> ast) {
for (auto i = 0u; i < ast->nodes.size(); i += 2) {
auto ident = ast->nodes[i]->token;
auto number = stoi(ast->nodes[i + 1]->token);
auto ident = ast->nodes[i]->token_to_string();
auto number = stoi(ast->nodes[i + 1]->token_to_string());
auto alloca =
builder_.CreateAlloca(builder_.getInt32Ty(), nullptr, ident);
@ -664,14 +664,14 @@ struct LLVM {
void compile_var(const shared_ptr<AstPL0> ast) {
for (const auto node : ast->nodes) {
auto ident = node->token;
auto ident = node->token_to_string();
builder_.CreateAlloca(builder_.getInt32Ty(), nullptr, ident);
}
}
void compile_procedure(const shared_ptr<AstPL0> ast) {
for (auto i = 0u; i < ast->nodes.size(); i += 2) {
auto ident = ast->nodes[i]->token;
auto ident = ast->nodes[i]->token_to_string();
auto block = ast->nodes[i + 1];
std::vector<Type*> pt(block->scope->free_variables.size(),
@ -712,7 +712,7 @@ struct LLVM {
}
void compile_assignment(const shared_ptr<AstPL0> ast) {
auto ident = ast->nodes[0]->token;
auto ident = ast->nodes[0]->token_to_string();
auto fn = builder_.GetInsertBlock()->getParent();
auto tbl = fn->getValueSymbolTable();
@ -726,7 +726,7 @@ struct LLVM {
}
void compile_call(const shared_ptr<AstPL0> ast) {
auto ident = ast->nodes[0]->token;
auto ident = ast->nodes[0]->token_to_string();
auto scope = get_closest_scope(ast);
auto block = scope->get_procedure(ident);
@ -805,7 +805,7 @@ struct LLVM {
auto lhs = compile_expression(ast->nodes[0]);
auto rhs = compile_expression(ast->nodes[2]);
const auto& ope = ast->nodes[1]->token;
const auto& ope = ast->nodes[1]->token_to_string();
switch (ope[0]) {
case '=':
return builder_.CreateICmpEQ(lhs, rhs, "icmpeq");
@ -836,7 +836,7 @@ struct LLVM {
Value* compile_expression(const shared_ptr<AstPL0> ast) {
const auto& nodes = ast->nodes;
auto sign = nodes[0]->token;
auto sign = nodes[0]->token_to_string();
auto negative = !(sign.empty() || sign == "+");
auto val = compile_term(nodes[1]);
@ -845,7 +845,7 @@ struct LLVM {
}
for (auto i = 2u; i < nodes.size(); i += 2) {
auto ope = nodes[i + 0]->token[0];
auto ope = nodes[i + 0]->token_to_string()[0];
auto rval = compile_term(nodes[i + 1]);
switch (ope) {
case '+':
@ -863,7 +863,7 @@ struct LLVM {
const auto& nodes = ast->nodes;
auto val = compile_factor(nodes[0]);
for (auto i = 1u; i < nodes.size(); i += 2) {
auto ope = nodes[i + 0]->token[0];
auto ope = nodes[i + 0]->token_to_string()[0];
auto rval = compile_switch_value(nodes[i + 1]);
switch (ope) {
case '*':
@ -889,7 +889,7 @@ struct LLVM {
}
Value* compile_ident(const shared_ptr<AstPL0> ast) {
auto ident = ast->token;
auto ident = ast->token_to_string();
auto fn = builder_.GetInsertBlock()->getParent();
auto tbl = fn->getValueSymbolTable();
@ -903,7 +903,7 @@ struct LLVM {
Value* compile_number(const shared_ptr<AstPL0> ast) {
return ConstantInt::getIntegerValue(builder_.getInt32Ty(),
APInt(32, ast->token, 10));
APInt(32, ast->token_to_string(), 10));
}
};

13
test/CMakeLists.txt
Unescape View File

@ -1,13 +1,14 @@
cmake_minimum_required(VERSION 3.1)
project(test)
enable_language(CXX)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_CXX_STANDARD 11)
set(CMAKE_CXX_EXTENSIONS OFF)
include_directories(..)
set(CMAKE_CXX_STANDARD 17)
add_executable(test-main test-main.cc test1.cc test2.cc test3.cc)
target_link_libraries(test-main ${add_link_deps})
add_test(TestMain test-main)
add_test(
NAME TestMain
COMMAND test-main
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})

377
test/test1.cc
Unescape View File

@ -4,20 +4,16 @@
using namespace peg;
#if !defined(PEGLIB_NO_UNICODE_CHARS)
TEST_CASE("Simple syntax test (with unicode)", "[general]")
{
parser parser(
u8" ROOT ← _ "
" _ <- ' ' "
);
TEST_CASE("Simple syntax test (with unicode)", "[general]") {
parser parser(u8" ROOT ← _ "
" _ <- ' ' ");
bool ret = parser;
REQUIRE(ret == true);
}
#endif
TEST_CASE("Simple syntax test", "[general]")
{
TEST_CASE("Simple syntax test", "[general]") {
parser parser(R"(
ROOT <- _
_ <- ' '
@ -27,15 +23,13 @@ TEST_CASE("Simple syntax test", "[general]")
REQUIRE(ret == true);
}
TEST_CASE("Empty syntax test", "[general]")
{
TEST_CASE("Empty syntax test", "[general]") {
parser parser("");
bool ret = parser;
REQUIRE(ret == false);
}
TEST_CASE("Start rule with ignore operator test", "[general]")
{
TEST_CASE("Start rule with ignore operator test", "[general]") {
parser parser(R"(
~ROOT <- _
_ <- ' '
@ -45,19 +39,17 @@ TEST_CASE("Start rule with ignore operator test", "[general]")
REQUIRE(ret == false);
}
TEST_CASE("Invalid UTF-8 text test", "[general]")
{
TEST_CASE("Invalid UTF-8 text test", "[general]") {
std::string s = "a <- '";
s += static_cast<char>(0xe8); // Make invalid utf8 text...
parser parser(s.c_str());
parser parser(s.data());
bool ret = parser;
REQUIRE(ret == false);
}
TEST_CASE("Backslash escape sequence test", "[general]")
{
TEST_CASE("Backslash escape sequence test", "[general]") {
parser parser(R"(
ROOT <- _
_ <- '\\'
@ -67,8 +59,7 @@ TEST_CASE("Backslash escape sequence test", "[general]")
REQUIRE(ret == true);
}
TEST_CASE("Invalid escape sequence test", "[general]")
{
TEST_CASE("Invalid escape sequence test", "[general]") {
parser parser(R"(
ROOT <- _
_ <- '\'
@ -78,22 +69,19 @@ TEST_CASE("Invalid escape sequence test", "[general]")
REQUIRE(ret == false);
}
TEST_CASE("Action taking non const Semantic Values parameter", "[general]")
{
TEST_CASE("Action taking non const Semantic Values parameter", "[general]") {
parser parser(R"(
ROOT <- TEXT
TEXT <- [a-zA-Z]+
)");
parser["ROOT"] = [&](SemanticValues& sv) {
auto s = any_cast<std::string>(sv[0]);
parser["ROOT"] = [&](SemanticValues &vs) {
auto s = std::string(std::any_cast<std::string_view>(vs[0]));
s[0] = 'H'; // mutate
return std::string(std::move(s)); // move
return s; // move
};
parser["TEXT"] = [&](SemanticValues& sv) {
return sv.token();
};
parser["TEXT"] = [&](SemanticValues &vs) { return vs.token(); };
std::string val;
auto ret = parser.parse("hello", val);
@ -101,18 +89,17 @@ TEST_CASE("Action taking non const Semantic Values parameter", "[general]")
REQUIRE(val == "Hello");
}
TEST_CASE("String capture test", "[general]")
{
TEST_CASE("String capture test", "[general]") {
parser parser(R"(
ROOT <- _ ('[' TAG_NAME ']' _)*
TAG_NAME <- (!']' .)+
_ <- [ \t]*
)");
std::vector<std::string> tags;
std::vector<std::string_view> tags;
parser["TAG_NAME"] = [&](const SemanticValues& sv) {
tags.push_back(sv.str());
parser["TAG_NAME"] = [&](const SemanticValues &vs) {
tags.push_back(vs.sv());
};
auto ret = parser.parse(" [tag1] [tag:2] [tag-3] ");
@ -126,14 +113,14 @@ TEST_CASE("String capture test", "[general]")
using namespace peg;
TEST_CASE("String capture test2", "[general]")
{
std::vector<std::string> tags;
TEST_CASE("String capture test2", "[general]") {
std::vector<std::string_view> tags;
Definition ROOT, TAG, TAG_NAME, WS;
ROOT <= seq(WS, zom(TAG));
TAG <= seq(chr('['), TAG_NAME, chr(']'), WS);
TAG_NAME <= oom(seq(npd(chr(']')), dot())), [&](const SemanticValues& sv) { tags.push_back(sv.str()); };
TAG_NAME <= oom(seq(npd(chr(']')), dot())),
[&](const SemanticValues &vs) { tags.push_back(vs.sv()); };
WS <= zom(cls(" \t"));
auto r = ROOT.parse(" [tag1] [tag:2] [tag-3] ");
@ -145,20 +132,16 @@ TEST_CASE("String capture test2", "[general]")
REQUIRE(tags[2] == "tag-3");
}
TEST_CASE("String capture test3", "[general]")
{
TEST_CASE("String capture test3", "[general]") {
parser pg(R"(
ROOT <- _ TOKEN*
TOKEN <- '[' < (!']' .)+ > ']' _
_ <- [ \t\r\n]*
)");
std::vector<std::string_view> tags;
std::vector<std::string> tags;
pg["TOKEN"] = [&](const SemanticValues& sv) {
tags.push_back(sv.token());
};
pg["TOKEN"] = [&](const SemanticValues &vs) { tags.push_back(vs.token()); };
auto ret = pg.parse(" [tag1] [tag:2] [tag-3] ");
@ -169,8 +152,7 @@ TEST_CASE("String capture test3", "[general]")
REQUIRE(tags[2] == "tag-3");
}
TEST_CASE("Cyclic grammer test", "[general]")
{
TEST_CASE("Cyclic grammer test", "[general]") {
Definition PARENT;
Definition CHILD;
@ -178,8 +160,7 @@ TEST_CASE("Cyclic grammer test", "[general]")
CHILD <= seq(PARENT);
}
TEST_CASE("Visit test", "[general]")
{
TEST_CASE("Visit test", "[general]") {
Definition ROOT, TAG, TAG_NAME, WS;
ROOT <= seq(WS, zom(TAG));
@ -193,8 +174,7 @@ TEST_CASE("Visit test", "[general]")
REQUIRE(defIds.ids.size() == 4);
}
TEST_CASE("Token check test", "[general]")
{
TEST_CASE("Token check test", "[general]") {
parser parser(R"(
EXPRESSION <- _ TERM (TERM_OPERATOR TERM)*
TERM <- FACTOR (FACTOR_OPERATOR FACTOR)*
@ -212,25 +192,21 @@ TEST_CASE("Token check test", "[general]")
REQUIRE(parser["_"].is_token() == true);
}
TEST_CASE("Lambda action test", "[general]")
{
TEST_CASE("Lambda action test", "[general]") {
parser parser(R"(
START <- (CHAR)*
CHAR <- .
)");
std::string ss;
parser["CHAR"] = [&](const SemanticValues& sv) {
ss += *sv.c_str();
};
parser["CHAR"] = [&](const SemanticValues &vs) { ss += *vs.sv().data(); };
bool ret = parser.parse("hello");
REQUIRE(ret == true);
REQUIRE(ss == "hello");
}
TEST_CASE("enter/leave handlers test", "[general]")
{
TEST_CASE("enter/leave handlers test", "[general]") {
parser parser(R"(
START <- LTOKEN '=' RTOKEN
LTOKEN <- TOKEN
@ -238,21 +214,22 @@ TEST_CASE("enter/leave handlers test", "[general]")
TOKEN <- [A-Za-z]+
)");
parser["LTOKEN"].enter = [&](const char*, size_t, any& dt) {
auto& require_upper_case = *any_cast<bool*>(dt);
parser["LTOKEN"].enter = [&](const char *, size_t, std::any &dt) {
auto &require_upper_case = *std::any_cast<bool *>(dt);
require_upper_case = false;
};
parser["LTOKEN"].leave = [&](const char*, size_t, size_t, any&, any& dt) {
auto& require_upper_case = *any_cast<bool*>(dt);
parser["LTOKEN"].leave = [&](const char *, size_t, size_t, std::any &,
std::any &dt) {
auto &require_upper_case = *std::any_cast<bool *>(dt);
require_upper_case = true;
};
auto message = "should be upper case string...";
parser["TOKEN"] = [&](const SemanticValues& sv, any& dt) {
auto& require_upper_case = *any_cast<bool*>(dt);
parser["TOKEN"] = [&](const SemanticValues &vs, std::any &dt) {
auto &require_upper_case = *std::any_cast<bool *>(dt);
if (require_upper_case) {
const auto& s = sv.str();
const auto &s = vs.sv();
if (!std::all_of(s.begin(), s.end(), ::isupper)) {
throw parse_error(message);
}
@ -260,7 +237,7 @@ TEST_CASE("enter/leave handlers test", "[general]")
};
bool require_upper_case = false;
any dt = &require_upper_case;
std::any dt = &require_upper_case;
REQUIRE(parser.parse("hello=world", dt) == false);
REQUIRE(parser.parse("HELLO=world", dt) == false);
REQUIRE(parser.parse("hello=WORLD", dt) == true);
@ -274,8 +251,7 @@ TEST_CASE("enter/leave handlers test", "[general]")
parser.parse("hello=world", dt);
}
TEST_CASE("WHITESPACE test", "[general]")
{
TEST_CASE("WHITESPACE test", "[general]") {
parser parser(R"(
# Rules
ROOT <- ITEM (',' ITEM)*
@ -293,8 +269,7 @@ TEST_CASE("WHITESPACE test", "[general]")
REQUIRE(ret == true);
}
TEST_CASE("WHITESPACE test2", "[general]")
{
TEST_CASE("WHITESPACE test2", "[general]") {
parser parser(R"(
# Rules
ROOT <- ITEM (',' ITEM)*
@ -305,9 +280,9 @@ TEST_CASE("WHITESPACE test2", "[general]")
TAB <- '\t'
)");
std::vector<std::string> items;
parser["ITEM"] = [&](const SemanticValues& sv) {
items.push_back(sv.token());
std::vector<std::string_view> items;
parser["ITEM"] = [&](const SemanticValues &vs) {
items.push_back(vs.token());
};
auto ret = parser.parse(R"([one], [two] ,[three] )");
@ -328,8 +303,8 @@ TEST_CASE("WHITESPACE test3", "[general]") {
%whitespace <- [ \t]*
)");
parser["StrQuot"] = [](const SemanticValues& sv) {
REQUIRE(sv.token() == R"( aaa \" bbb )");
parser["StrQuot"] = [](const SemanticValues &vs) {
REQUIRE(vs.token() == R"( aaa \" bbb )");
};
auto ret = parser.parse(R"( " aaa \" bbb " )");
@ -345,16 +320,14 @@ TEST_CASE("WHITESPACE test4", "[general]") {
%whitespace <- [ \t\r\n]*
)");
parser["HELLO"] = [](const SemanticValues& sv) {
REQUIRE(sv.token() == "hello");
parser["HELLO"] = [](const SemanticValues &vs) {
REQUIRE(vs.token() == "hello");
};
parser["OPE"] = [](const SemanticValues& sv) {
REQUIRE(sv.token() == "+");
};
parser["OPE"] = [](const SemanticValues &vs) { REQUIRE(vs.token() == "+"); };
parser["WORLD"] = [](const SemanticValues& sv) {
REQUIRE(sv.token() == "world");
parser["WORLD"] = [](const SemanticValues &vs) {
REQUIRE(vs.token() == "world");
};
auto ret = parser.parse(" hello + world ");
@ -375,42 +348,33 @@ TEST_CASE("Word expression test", "[general]") {
REQUIRE(parser.parse("hello , world") == true);
}
TEST_CASE("Skip token test", "[general]")
{
parser parser(
" ROOT <- _ ITEM (',' _ ITEM _)* "
TEST_CASE("Skip token test", "[general]") {
parser parser(" ROOT <- _ ITEM (',' _ ITEM _)* "
" ITEM <- ([a-z0-9])+ "
" ~_ <- [ \t]* "
);
" ~_ <- [ \t]* ");
parser["ROOT"] = [&](const SemanticValues& sv) {
REQUIRE(sv.size() == 2);
};
parser["ROOT"] = [&](const SemanticValues &vs) { REQUIRE(vs.size() == 2); };
auto ret = parser.parse(" item1, item2 ");
REQUIRE(ret == true);
}
TEST_CASE("Skip token test2", "[general]")
{
TEST_CASE("Skip token test2", "[general]") {
parser parser(R"(
ROOT <- ITEM (',' ITEM)*
ITEM <- < ([a-z0-9])+ >
%whitespace <- [ \t]*
)");
parser["ROOT"] = [&](const SemanticValues& sv) {
REQUIRE(sv.size() == 2);
};
parser["ROOT"] = [&](const SemanticValues &vs) { REQUIRE(vs.size() == 2); };
auto ret = parser.parse(" item1, item2 ");
REQUIRE(ret == true);
}
TEST_CASE("Custom AST test", "[general]")
{
TEST_CASE("Custom AST test", "[general]") {
struct CustomType {};
using CustomAst = AstBase<CustomType>;
@ -427,8 +391,7 @@ TEST_CASE("Custom AST test", "[general]")
REQUIRE(ast->nodes.size() == 4);
}
TEST_CASE("Backtracking test", "[general]")
{
TEST_CASE("Backtracking test", "[general]") {
parser parser(R"(
START <- PAT1 / PAT2
PAT1 <- HELLO ' One'
@ -437,9 +400,7 @@ TEST_CASE("Backtracking test", "[general]")
)");
size_t count = 0;
parser["HELLO"] = [&](const SemanticValues& /*sv*/) {
count++;
};
parser["HELLO"] = [&](const SemanticValues & /*vs*/) { count++; };
parser.enable_packrat_parsing();
@ -448,8 +409,7 @@ TEST_CASE("Backtracking test", "[general]")
REQUIRE(count == 1); // Skip second time
}
TEST_CASE("Backtracking with AST", "[general]")
{
TEST_CASE("Backtracking with AST", "[general]") {
parser parser(R"(
S <- A? B (A B)* A
A <- 'a'
@ -463,11 +423,8 @@ TEST_CASE("Backtracking with AST", "[general]")
REQUIRE(ast->nodes.size() == 2);
}
TEST_CASE("Octal/Hex/Unicode value test", "[general]")
{
parser parser(
R"( ROOT <- '\132\x7a\u30f3' )"
);
TEST_CASE("Octal/Hex/Unicode value test", "[general]") {
parser parser(R"( ROOT <- '\132\x7a\u30f3' )");
auto ret = parser.parse("Zzン");
@ -482,32 +439,30 @@ TEST_CASE("Ignore case test", "[general]") {
%whitespace <- [ \t\r\n]*
)");
parser["HELLO"] = [](const SemanticValues& sv) {
REQUIRE(sv.token() == "Hello");
parser["HELLO"] = [](const SemanticValues &vs) {
REQUIRE(vs.token() == "Hello");
};
parser["WORLD"] = [](const SemanticValues& sv) {
REQUIRE(sv.token() == "World");
parser["WORLD"] = [](const SemanticValues &vs) {
REQUIRE(vs.token() == "World");
};
auto ret = parser.parse(" Hello World ");
REQUIRE(ret == true);
}
TEST_CASE("mutable lambda test", "[general]")
{
std::vector<std::string> vec;
TEST_CASE("mutable lambda test", "[general]") {
std::vector<std::string_view> vec;
parser pg("ROOT <- 'mutable lambda test'");
// This test makes sure if the following code can be compiled.
pg["TOKEN"] = [=](const SemanticValues& sv) mutable {
vec.push_back(sv.str());
pg["TOKEN"] = [=](const SemanticValues &vs) mutable {
vec.push_back(vs.sv());
};
}
TEST_CASE("Simple calculator test", "[general]")
{
TEST_CASE("Simple calculator test", "[general]") {
parser parser(R"(
Additive <- Multitive '+' Additive / Multitive
Multitive <- Primary '*' Multitive / Primary
@ -515,27 +470,21 @@ TEST_CASE("Simple calculator test", "[general]")
Number <- [0-9]+
)");
parser["Additive"] = [](const SemanticValues& sv) {
switch (sv.choice()) {
case 0:
return any_cast<int>(sv[0]) + any_cast<int>(sv[1]);
default:
return any_cast<int>(sv[0]);
parser["Additive"] = [](const SemanticValues &vs) {
switch (vs.choice()) {
case 0: return std::any_cast<int>(vs[0]) + std::any_cast<int>(vs[1]);
default: return std::any_cast<int>(vs[0]);
}
};
parser["Multitive"] = [](const SemanticValues& sv) {
switch (sv.choice()) {
case 0:
return any_cast<int>(sv[0]) * any_cast<int>(sv[1]);
default:
return any_cast<int>(sv[0]);
parser["Multitive"] = [](const SemanticValues &vs) {
switch (vs.choice()) {
case 0: return std::any_cast<int>(vs[0]) * std::any_cast<int>(vs[1]);
default: return std::any_cast<int>(vs[0]);
}
};
parser["Number"] = [](const SemanticValues& sv) {
return atoi(sv.c_str());
};
parser["Number"] = [](const SemanticValues &vs) { return vs.token_to_number<int>(); };
int val;
parser.parse("(1+2)*3", val);
@ -543,8 +492,7 @@ TEST_CASE("Simple calculator test", "[general]")
REQUIRE(val == 9);
}
TEST_CASE("Calculator test", "[general]")
{
TEST_CASE("Calculator test", "[general]") {
// Construct grammer
Definition EXPRESSION, TERM, FACTOR, TERM_OPERATOR, FACTOR_OPERATOR, NUMBER;
@ -556,11 +504,11 @@ TEST_CASE("Calculator test", "[general]")
NUMBER <= oom(cls("0-9"));
// Setup actions
auto reduce = [](const SemanticValues& sv) -> long {
long ret = any_cast<long>(sv[0]);
for (auto i = 1u; i < sv.size(); i += 2) {
auto num = any_cast<long>(sv[i + 1]);
switch (any_cast<char>(sv[i])) {
auto reduce = [](const SemanticValues &vs) -> long {
long ret = std::any_cast<long>(vs[0]);
for (auto i = 1u; i < vs.size(); i += 2) {
auto num = std::any_cast<long>(vs[i + 1]);
switch (std::any_cast<char>(vs[i])) {
case '+': ret += num; break;
case '-': ret -= num; break;
case '*': ret *= num; break;
@ -572,9 +520,9 @@ TEST_CASE("Calculator test", "[general]")
EXPRESSION = reduce;
TERM = reduce;
TERM_OPERATOR = [](const SemanticValues& sv) { return *sv.c_str(); };
FACTOR_OPERATOR = [](const SemanticValues& sv) { return *sv.c_str(); };
NUMBER = [](const SemanticValues& sv) { return stol(sv.str(), nullptr, 10); };
TERM_OPERATOR = [](const SemanticValues &vs) { return *vs.sv().data(); };
FACTOR_OPERATOR = [](const SemanticValues &vs) { return *vs.sv().data(); };
NUMBER = [](const SemanticValues &vs) { return vs.token_to_number<long>(); };
// Parse
long val;
@ -584,8 +532,7 @@ TEST_CASE("Calculator test", "[general]")
REQUIRE(val == -3);
}
TEST_CASE("Calculator test2", "[general]")
{
TEST_CASE("Calculator test2", "[general]") {
// Parse syntax
auto syntax = R"(
# Grammar for Calculator...
@ -602,11 +549,11 @@ TEST_CASE("Calculator test2", "[general]")
auto &g = *grammar;
// Setup actions
auto reduce = [](const SemanticValues& sv) -> long {
long ret = any_cast<long>(sv[0]);
for (auto i = 1u; i < sv.size(); i += 2) {
auto num = any_cast<long>(sv[i + 1]);
switch (any_cast<char>(sv[i])) {
auto reduce = [](const SemanticValues &vs) -> long {
long ret = std::any_cast<long>(vs[0]);
for (auto i = 1u; i < vs.size(); i += 2) {
auto num = std::any_cast<long>(vs[i + 1]);
switch (std::any_cast<char>(vs[i])) {
case '+': ret += num; break;
case '-': ret -= num; break;
case '*': ret *= num; break;
@ -618,9 +565,9 @@ TEST_CASE("Calculator test2", "[general]")
g["EXPRESSION"] = reduce;
g["TERM"] = reduce;
g["TERM_OPERATOR"] = [](const SemanticValues& sv) { return *sv.c_str(); };
g["FACTOR_OPERATOR"] = [](const SemanticValues& sv) { return *sv.c_str(); };
g["NUMBER"] = [](const SemanticValues& sv) { return stol(sv.str(), nullptr, 10); };
g["TERM_OPERATOR"] = [](const SemanticValues &vs) { return *vs.sv().data(); };
g["FACTOR_OPERATOR"] = [](const SemanticValues &vs) { return *vs.sv().data(); };
g["NUMBER"] = [](const SemanticValues &vs) { return vs.token_to_number<long>(); };
// Parse
long val;
@ -630,8 +577,7 @@ TEST_CASE("Calculator test2", "[general]")
REQUIRE(val == -3);
}
TEST_CASE("Calculator test3", "[general]")
{
TEST_CASE("Calculator test3", "[general]") {
// Parse syntax
parser parser(R"(
# Grammar for Calculator...
@ -643,11 +589,11 @@ TEST_CASE("Calculator test3", "[general]")
NUMBER <- [0-9]+
)");
auto reduce = [](const SemanticValues& sv) -> long {
long ret = any_cast<long>(sv[0]);
for (auto i = 1u; i < sv.size(); i += 2) {
auto num = any_cast<long>(sv[i + 1]);
switch (any_cast<char>(sv[i])) {
auto reduce = [](const SemanticValues &vs) -> long {
long ret = std::any_cast<long>(vs[0]);
for (auto i = 1u; i < vs.size(); i += 2) {
auto num = std::any_cast<long>(vs[i + 1]);
switch (std::any_cast<char>(vs[i])) {
case '+': ret += num; break;
case '-': ret -= num; break;
case '*': ret *= num; break;
@ -660,9 +606,13 @@ TEST_CASE("Calculator test3", "[general]")
// Setup actions
parser["EXPRESSION"] = reduce;
parser["TERM"] = reduce;
parser["TERM_OPERATOR"] = [](const SemanticValues& sv) { return static_cast<char>(*sv.c_str()); };
parser["FACTOR_OPERATOR"] = [](const SemanticValues& sv) { return static_cast<char>(*sv.c_str()); };
parser["NUMBER"] = [](const SemanticValues& sv) { return stol(sv.str(), nullptr, 10); };
parser["TERM_OPERATOR"] = [](const SemanticValues &vs) {
return static_cast<char>(*vs.sv().data());
};
parser["FACTOR_OPERATOR"] = [](const SemanticValues &vs) {
return static_cast<char>(*vs.sv().data());
};
parser["NUMBER"] = [](const SemanticValues &vs) { return vs.token_to_number<long>(); };
// Parse
long val;
@ -672,8 +622,7 @@ TEST_CASE("Calculator test3", "[general]")
REQUIRE(val == -3);
}
TEST_CASE("Calculator test with AST", "[general]")
{
TEST_CASE("Calculator test with AST", "[general]") {
parser parser(R"(
EXPRESSION <- _ TERM (TERM_OPERATOR TERM)*
TERM <- FACTOR (FACTOR_OPERATOR FACTOR)*
@ -688,7 +637,7 @@ TEST_CASE("Calculator test with AST", "[general]")
std::function<long(const Ast &)> eval = [&](const Ast &ast) {
if (ast.name == "NUMBER") {
return stol(ast.token);
return ast.token_to_number<long>();
} else {
const auto &nodes = ast.nodes;
auto result = eval(*nodes[0]);
@ -717,7 +666,8 @@ TEST_CASE("Calculator test with AST", "[general]")
TEST_CASE("Calculator test with combinators and AST", "[general]") {
// Construct grammer
AST_DEFINITIONS(EXPRESSION, TERM, FACTOR, TERM_OPERATOR, FACTOR_OPERATOR, NUMBER);
AST_DEFINITIONS(EXPRESSION, TERM, FACTOR, TERM_OPERATOR, FACTOR_OPERATOR,
NUMBER);
EXPRESSION <= seq(TERM, zom(seq(TERM_OPERATOR, TERM)));
TERM <= seq(FACTOR, zom(seq(FACTOR_OPERATOR, FACTOR)));
@ -728,7 +678,7 @@ TEST_CASE("Calculator test with combinators and AST", "[general]") {
std::function<long(const Ast &)> eval = [&](const Ast &ast) {
if (ast.name == "NUMBER") {
return stol(ast.token);
return ast.token_to_number<long>();
} else {
const auto &nodes = ast.nodes;
auto result = eval(*nodes[0]);
@ -755,8 +705,7 @@ TEST_CASE("Calculator test with combinators and AST", "[general]") {
REQUIRE(val == -3);
}
TEST_CASE("Ignore semantic value test", "[general]")
{
TEST_CASE("Ignore semantic value test", "[general]") {
parser parser(R"(
START <- ~HELLO WORLD
HELLO <- 'Hello' _
@ -774,8 +723,7 @@ TEST_CASE("Ignore semantic value test", "[general]")
REQUIRE(ast->nodes[0]->name == "WORLD");
}
TEST_CASE("Ignore semantic value of 'or' predicate test", "[general]")
{
TEST_CASE("Ignore semantic value of 'or' predicate test", "[general]") {
parser parser(R"(
START <- _ !DUMMY HELLO_WORLD '.'
HELLO_WORLD <- HELLO 'World' _
@ -794,8 +742,7 @@ TEST_CASE("Ignore semantic value of 'or' predicate test", "[general]")
REQUIRE(ast->nodes[0]->name == "HELLO_WORLD");
}
TEST_CASE("Ignore semantic value of 'and' predicate test", "[general]")
{
TEST_CASE("Ignore semantic value of 'and' predicate test", "[general]") {
parser parser(R"(
START <- _ &HELLO HELLO_WORLD '.'
HELLO_WORLD <- HELLO 'World' _
@ -813,8 +760,7 @@ TEST_CASE("Ignore semantic value of 'and' predicate test", "[general]")
REQUIRE(ast->nodes[0]->name == "HELLO_WORLD");
}
TEST_CASE("Literal token on AST test1", "[general]")
{
TEST_CASE("Literal token on AST test1", "[general]") {
parser parser(R"(
STRING_LITERAL <- '"' (('\\"' / '\\t' / '\\n') / (!["] .))* '"'
)");
@ -829,8 +775,7 @@ TEST_CASE("Literal token on AST test1", "[general]")
REQUIRE(ast->nodes.empty());
}
TEST_CASE("Literal token on AST test2", "[general]")
{
TEST_CASE("Literal token on AST test2", "[general]") {
parser parser(R"(
STRING_LITERAL <- '"' (ESC / CHAR)* '"'
ESC <- ('\\"' / '\\t' / '\\n')
@ -847,8 +792,7 @@ TEST_CASE("Literal token on AST test2", "[general]")
REQUIRE(ast->nodes.size() == 3);
}
TEST_CASE("Literal token on AST test3", "[general]")
{
TEST_CASE("Literal token on AST test3", "[general]") {
parser parser(R"(
STRING_LITERAL <- < '"' (ESC / CHAR)* '"' >
ESC <- ('\\"' / '\\t' / '\\n')
@ -865,8 +809,7 @@ TEST_CASE("Literal token on AST test3", "[general]")
REQUIRE(ast->nodes.empty());
}
TEST_CASE("Missing missing definitions test", "[general]")
{
TEST_CASE("Missing missing definitions test", "[general]") {
parser parser(R"(
A <- B C
)");
@ -874,8 +817,7 @@ TEST_CASE("Missing missing definitions test", "[general]")
REQUIRE(!parser);
}
TEST_CASE("Definition duplicates test", "[general]")
{
TEST_CASE("Definition duplicates test", "[general]") {
parser parser(R"(
A <- ''
A <- ''
@ -884,8 +826,7 @@ TEST_CASE("Definition duplicates test", "[general]")
REQUIRE(!parser);
}
TEST_CASE("Semantic values test", "[general]")
{
TEST_CASE("Semantic values test", "[general]") {
parser parser(R"(
term <- ( a b c x )? a b c
a <- 'a'
@ -895,14 +836,14 @@ TEST_CASE("Semantic values test", "[general]")
)");
for (const auto &rule : parser.get_rule_names()) {
parser[rule.c_str()] = [rule](const SemanticValues& sv, any&) {
parser[rule.data()] = [rule](const SemanticValues &vs, std::any &) {
if (rule == "term") {
REQUIRE(any_cast<std::string>(sv[0]) == "a at 0");
REQUIRE(any_cast<std::string>(sv[1]) == "b at 1");
REQUIRE(any_cast<std::string>(sv[2]) == "c at 2");
REQUIRE(std::any_cast<std::string>(vs[0]) == "a at 0");
REQUIRE(std::any_cast<std::string>(vs[1]) == "b at 1");
REQUIRE(std::any_cast<std::string>(vs[2]) == "c at 2");
return std::string();
} else {
return rule + " at " + std::to_string(sv.c_str() - sv.ss);
return rule + " at " + std::to_string(vs.sv().data() - vs.ss);
}
};
}
@ -910,36 +851,33 @@ TEST_CASE("Semantic values test", "[general]")
REQUIRE(parser.parse("abc"));
}
TEST_CASE("Ordered choice count", "[general]")
{
TEST_CASE("Ordered choice count", "[general]") {
parser parser(R"(
S <- 'a' / 'b'
)");
parser["S"] = [](const SemanticValues& sv) {
REQUIRE(sv.choice() == 1);
REQUIRE(sv.choice_count() == 2);
parser["S"] = [](const SemanticValues &vs) {
REQUIRE(vs.choice() == 1);
REQUIRE(vs.choice_count() == 2);
};
parser.parse("b");
}
TEST_CASE("Ordered choice count 2", "[general]")
{
TEST_CASE("Ordered choice count 2", "[general]") {
parser parser(R"(
S <- ('a' / 'b')*
)");
parser["S"] = [](const SemanticValues& sv) {
REQUIRE(sv.choice() == 0);
REQUIRE(sv.choice_count() == 0);
parser["S"] = [](const SemanticValues &vs) {
REQUIRE(vs.choice() == 0);
REQUIRE(vs.choice_count() == 0);
};
parser.parse("b");
}
TEST_CASE("Semantic value tag", "[general]")
{
TEST_CASE("Semantic value tag", "[general]") {
parser parser(R"(
S <- A? B* C?
A <- 'a'
@ -949,10 +887,10 @@ TEST_CASE("Semantic value tag", "[general]")
{
using namespace udl;
parser["S"] = [](const SemanticValues& sv) {
REQUIRE(sv.size() == 1);
REQUIRE(sv.tags.size() == 1);
REQUIRE(sv.tags[0] == "C"_);
parser["S"] = [](const SemanticValues &vs) {
REQUIRE(vs.size() == 1);
REQUIRE(vs.tags.size() == 1);
REQUIRE(vs.tags[0] == "C"_);
};
auto ret = parser.parse("c");
REQUIRE(ret == true);
@ -960,11 +898,11 @@ TEST_CASE("Semantic value tag", "[general]")
{
using namespace udl;
parser["S"] = [](const SemanticValues& sv) {
REQUIRE(sv.size() == 2);
REQUIRE(sv.tags.size() == 2);
REQUIRE(sv.tags[0] == "B"_);
REQUIRE(sv.tags[1] == "B"_);
parser["S"] = [](const SemanticValues &vs) {
REQUIRE(vs.size() == 2);
REQUIRE(vs.tags.size() == 2);
REQUIRE(vs.tags[0] == "B"_);
REQUIRE(vs.tags[1] == "B"_);
};
auto ret = parser.parse("bb");
REQUIRE(ret == true);
@ -972,19 +910,18 @@ TEST_CASE("Semantic value tag", "[general]")
{
using namespace udl;
parser["S"] = [](const SemanticValues& sv) {
REQUIRE(sv.size() == 2);
REQUIRE(sv.tags.size() == 2);
REQUIRE(sv.tags[0] == "A"_);
REQUIRE(sv.tags[1] == "C"_);
parser["S"] = [](const SemanticValues &vs) {
REQUIRE(vs.size() == 2);
REQUIRE(vs.tags.size() == 2);
REQUIRE(vs.tags[0] == "A"_);
REQUIRE(vs.tags[1] == "C"_);
};
auto ret = parser.parse("ac");
REQUIRE(ret == true);
}
}
TEST_CASE("Negated Class test", "[general]")
{
TEST_CASE("Negated Class test", "[general]") {
parser parser(R"(
ROOT <- [^a-z_]+
)");

271
test/test2.cc
Unescape View File

@ -101,8 +101,7 @@ TEST_CASE("Infinite loop 1", "[infinite loop]")
REQUIRE(!pg);
}
TEST_CASE("Infinite loop 2", "[infinite loop]")
{
TEST_CASE("Infinite loop 2", "[infinite loop]") {
parser pg(R"(
ROOT <- WH TOKEN+ WH
TOKEN <- [a-z0-9]*
@ -112,8 +111,7 @@ TEST_CASE("Infinite loop 2", "[infinite loop]")
REQUIRE(!pg);
}
TEST_CASE("Infinite loop 3", "[infinite loop]")
{
TEST_CASE("Infinite loop 3", "[infinite loop]") {
parser pg(R"(
ROOT <- WH TOKEN* WH
TOKEN <- !'word1'
@ -123,8 +121,7 @@ TEST_CASE("Infinite loop 3", "[infinite loop]")
REQUIRE(!pg);
}
TEST_CASE("Infinite loop 4", "[infinite loop]")
{
TEST_CASE("Infinite loop 4", "[infinite loop]") {
parser pg(R"(
ROOT <- WH TOKEN* WH
TOKEN <- &'word1'
@ -134,8 +131,7 @@ TEST_CASE("Infinite loop 4", "[infinite loop]")
REQUIRE(!pg);
}
TEST_CASE("Infinite loop 5", "[infinite loop]")
{
TEST_CASE("Infinite loop 5", "[infinite loop]") {
parser pg(R"(
Numbers <- Number*
Number <- [0-9]+ / Spacing
@ -146,8 +142,7 @@ TEST_CASE("Infinite loop 5", "[infinite loop]")
REQUIRE(!pg);
}
TEST_CASE("Infinite loop 6", "[infinite loop]")
{
TEST_CASE("Infinite loop 6", "[infinite loop]") {
parser pg(R"(
S <- ''*
)");
@ -155,8 +150,7 @@ TEST_CASE("Infinite loop 6", "[infinite loop]")
REQUIRE(!pg);
}
TEST_CASE("Infinite loop 7", "[infinite loop]")
{
TEST_CASE("Infinite loop 7", "[infinite loop]") {
parser pg(R"(
S <- A*
A <- ''
@ -165,8 +159,7 @@ TEST_CASE("Infinite loop 7", "[infinite loop]")
REQUIRE(!pg);
}
TEST_CASE("Not infinite 1", "[infinite loop]")
{
TEST_CASE("Not infinite 1", "[infinite loop]") {
parser pg(R"(
Numbers <- Number* EOF
Number <- [0-9]+ / Spacing
@ -177,8 +170,7 @@ TEST_CASE("Not infinite 1", "[infinite loop]")
REQUIRE(!!pg); // OK
}
TEST_CASE("Not infinite 2", "[infinite loop]")
{
TEST_CASE("Not infinite 2", "[infinite loop]") {
parser pg(R"(
ROOT <- _ ('[' TAG_NAME ']' _)*
# In a sequence operator, if there is at least one non-empty element, we can treat it as non-empty
@ -189,8 +181,7 @@ TEST_CASE("Not infinite 2", "[infinite loop]")
REQUIRE(!!pg); // OK
}
TEST_CASE("Not infinite 3", "[infinite loop]")
{
TEST_CASE("Not infinite 3", "[infinite loop]") {
parser pg(R"(
EXPRESSION <- _ TERM (TERM_OPERATOR TERM)*
TERM <- FACTOR (FACTOR_OPERATOR FACTOR)*
@ -204,8 +195,7 @@ TEST_CASE("Not infinite 3", "[infinite loop]")
REQUIRE(!!pg); // OK
}
TEST_CASE("Precedence climbing", "[precedence]")
{
TEST_CASE("Precedence climbing", "[precedence]") {
parser parser(R"(
START <- _ EXPRESSION
EXPRESSION <- ATOM (OPERATOR ATOM)* {
@ -223,11 +213,11 @@ TEST_CASE("Precedence climbing", "[precedence]")
parser.enable_packrat_parsing();
// Setup actions
parser["EXPRESSION"] = [](const SemanticValues& sv) -> long {
auto result = any_cast<long>(sv[0]);
if (sv.size() > 1) {
auto ope = any_cast<char>(sv[1]);
auto num = any_cast<long>(sv[2]);
parser["EXPRESSION"] = [](const SemanticValues &vs) -> long {
auto result = std::any_cast<long>(vs[0]);
if (vs.size() > 1) {
auto ope = std::any_cast<char>(vs[1]);
auto num = std::any_cast<long>(vs[2]);
switch (ope) {
case '+': result += num; break;
case '-': result -= num; break;
@ -237,8 +227,8 @@ TEST_CASE("Precedence climbing", "[precedence]")
}
return result;
};
parser["OPERATOR"] = [](const SemanticValues& sv) { return *sv.c_str(); };
parser["NUMBER"] = [](const SemanticValues& sv) { return atol(sv.c_str()); };
parser["OPERATOR"] = [](const SemanticValues &vs) { return *vs.sv().data(); };
parser["NUMBER"] = [](const SemanticValues &vs) { return vs.token_to_number<long>(); };
bool ret = parser;
REQUIRE(ret == true);
@ -262,8 +252,7 @@ TEST_CASE("Precedence climbing", "[precedence]")
}
}
TEST_CASE("Precedence climbing with macro", "[precedence]")
{
TEST_CASE("Precedence climbing with macro", "[precedence]") {
// Create a PEG parser
parser parser(R"(
EXPRESSION <- INFIX_EXPRESSION(ATOM, OPERATOR)
@ -284,11 +273,11 @@ TEST_CASE("Precedence climbing with macro", "[precedence]")
REQUIRE(ret == true);
// Setup actions
parser["INFIX_EXPRESSION"] = [](const SemanticValues& sv) -> long {
auto result = any_cast<long>(sv[0]);
if (sv.size() > 1) {
auto ope = any_cast<char>(sv[1]);
auto num = any_cast<long>(sv[2]);
parser["INFIX_EXPRESSION"] = [](const SemanticValues &vs) -> long {
auto result = std::any_cast<long>(vs[0]);
if (vs.size() > 1) {
auto ope = std::any_cast<char>(vs[1]);
auto num = std::any_cast<long>(vs[2]);
switch (ope) {
case '+': result += num; break;
case '-': result -= num; break;
@ -298,8 +287,8 @@ TEST_CASE("Precedence climbing with macro", "[precedence]")
}
return result;
};
parser["OPERATOR"] = [](const SemanticValues& sv) { return *sv.c_str(); };
parser["NUMBER"] = [](const SemanticValues& sv) { return atol(sv.c_str()); };
parser["OPERATOR"] = [](const SemanticValues &vs) { return *vs.sv().data(); };
parser["NUMBER"] = [](const SemanticValues &vs) { return vs.token_to_number<long>(); };
{
auto expr = " 1 + 2 * 3 * (4 - 5 + 6) / 7 - 8 ";
@ -320,8 +309,7 @@ TEST_CASE("Precedence climbing with macro", "[precedence]")
}
}
TEST_CASE("Precedence climbing error1", "[precedence]")
{
TEST_CASE("Precedence climbing error1", "[precedence]") {
parser parser(R"(
START <- _ EXPRESSION
EXPRESSION <- ATOM (OPERATOR ATOM1)* {
@ -341,8 +329,7 @@ TEST_CASE("Precedence climbing error1", "[precedence]")
REQUIRE(ret == false);
}
TEST_CASE("Precedence climbing error2", "[precedence]")
{
TEST_CASE("Precedence climbing error2", "[precedence]") {
parser parser(R"(
START <- _ EXPRESSION
EXPRESSION <- ATOM OPERATOR ATOM {
@ -392,8 +379,7 @@ TEST_CASE("Packrat parser test with %whitespace%", "[packrat]") {
REQUIRE(ret == true);
}
TEST_CASE("Packrat parser test with macro", "[packrat]")
{
TEST_CASE("Packrat parser test with macro", "[packrat]") {
parser parser(R"(
EXPRESSION <- _ LIST(TERM, TERM_OPERATOR)
TERM <- LIST(FACTOR, FACTOR_OPERATOR)
@ -412,7 +398,8 @@ TEST_CASE("Packrat parser test with macro", "[packrat]")
REQUIRE(ret == true);
}
TEST_CASE("Packrat parser test with precedence expression parser", "[packrat]") {
TEST_CASE("Packrat parser test with precedence expression parser",
"[packrat]") {
peg::parser parser(R"(
Expression <- Atom (Operator Atom)* { precedence L + - L * / }
Atom <- _? Number _?
@ -430,8 +417,7 @@ TEST_CASE("Packrat parser test with precedence expression parser", "[packrat]")
REQUIRE(ret == true);
}
TEST_CASE("Backreference test", "[backreference]")
{
TEST_CASE("Backreference test", "[backreference]") {
parser parser(R"(
START <- _ LQUOTE < (!RQUOTE .)* > RQUOTE _
LQUOTE <- 'R"' $delm< [a-zA-Z]* > '('
@ -440,9 +426,7 @@ TEST_CASE("Backreference test", "[backreference]")
)");
std::string token;
parser["START"] = [&](const SemanticValues& sv) {
token = sv.token();
};
parser["START"] = [&](const SemanticValues &vs) { token = vs.token(); };
{
token.clear();
@ -485,8 +469,7 @@ TEST_CASE("Backreference test", "[backreference]")
}
}
TEST_CASE("Invalid backreference test", "[backreference]")
{
TEST_CASE("Invalid backreference test", "[backreference]") {
parser parser(R"(
START <- _ LQUOTE (!RQUOTE .)* RQUOTE _
LQUOTE <- 'R"' $delm< [a-zA-Z]* > '('
@ -494,16 +477,13 @@ TEST_CASE("Invalid backreference test", "[backreference]")
~_ <- [ \t\r\n]*
)");
REQUIRE_THROWS_AS(
parser.parse(R"delm(
REQUIRE_THROWS_AS(parser.parse(R"delm(
R"foo("(hello world)")foo"
)delm"),
std::runtime_error);
}
TEST_CASE("Nested capture test", "[backreference]")
{
TEST_CASE("Nested capture test", "[backreference]") {
parser parser(R"(
ROOT <- CONTENT
CONTENT <- (ELEMENT / TEXT)*
@ -521,8 +501,7 @@ TEST_CASE("Nested capture test", "[backreference]")
REQUIRE(!parser.parse("This is a <u>test</u> text</b>."));
}
TEST_CASE("Backreference with Prioritized Choice test", "[backreference]")
{
TEST_CASE("Backreference with Prioritized Choice test", "[backreference]") {
parser parser(R"(
TREE <- WRONG_BRANCH / CORRECT_BRANCH
WRONG_BRANCH <- BRANCH THAT IS_capture WRONG
@ -538,8 +517,7 @@ TEST_CASE("Backreference with Prioritized Choice test", "[backreference]")
REQUIRE_THROWS_AS(parser.parse("branchthatiscorrect"), std::runtime_error);
}
TEST_CASE("Backreference with Zero or More test", "[backreference]")
{
TEST_CASE("Backreference with Zero or More test", "[backreference]") {
parser parser(R"(
TREE <- WRONG_BRANCH* CORRECT_BRANCH
WRONG_BRANCH <- BRANCH THAT IS_capture WRONG
@ -554,14 +532,16 @@ TEST_CASE("Backreference with Zero or More test", "[backreference]")
REQUIRE(parser.parse("branchthatiswrongbranchthatiscorrect"));
REQUIRE(!parser.parse("branchthatiswrongbranchthatIscorrect"));
REQUIRE(!parser.parse("branchthatiswrongbranchthatIswrongbranchthatiscorrect"));
REQUIRE(parser.parse("branchthatiswrongbranchthatIswrongbranchthatIscorrect"));
REQUIRE(
!parser.parse("branchthatiswrongbranchthatIswrongbranchthatiscorrect"));
REQUIRE(
parser.parse("branchthatiswrongbranchthatIswrongbranchthatIscorrect"));
REQUIRE_THROWS_AS(parser.parse("branchthatiscorrect"), std::runtime_error);
REQUIRE_THROWS_AS(parser.parse("branchthatiswron_branchthatiscorrect"), std::runtime_error);
REQUIRE_THROWS_AS(parser.parse("branchthatiswron_branchthatiscorrect"),
std::runtime_error);
}
TEST_CASE("Backreference with One or More test", "[backreference]")
{
TEST_CASE("Backreference with One or More test", "[backreference]") {
parser parser(R"(
TREE <- WRONG_BRANCH+ CORRECT_BRANCH
WRONG_BRANCH <- BRANCH THAT IS_capture WRONG
@ -576,14 +556,15 @@ TEST_CASE("Backreference with One or More test", "[backreference]")
REQUIRE(parser.parse("branchthatiswrongbranchthatiscorrect"));
REQUIRE(!parser.parse("branchthatiswrongbranchthatIscorrect"));
REQUIRE(!parser.parse("branchthatiswrongbranchthatIswrongbranchthatiscorrect"));
REQUIRE(parser.parse("branchthatiswrongbranchthatIswrongbranchthatIscorrect"));
REQUIRE(
!parser.parse("branchthatiswrongbranchthatIswrongbranchthatiscorrect"));
REQUIRE(
parser.parse("branchthatiswrongbranchthatIswrongbranchthatIscorrect"));
REQUIRE(!parser.parse("branchthatiscorrect"));
REQUIRE(!parser.parse("branchthatiswron_branchthatiscorrect"));
}
TEST_CASE("Backreference with Option test", "[backreference]")
{
TEST_CASE("Backreference with Option test", "[backreference]") {
parser parser(R"(
TREE <- WRONG_BRANCH? CORRECT_BRANCH
WRONG_BRANCH <- BRANCH THAT IS_capture WRONG
@ -598,14 +579,16 @@ TEST_CASE("Backreference with Option test", "[backreference]")
REQUIRE(parser.parse("branchthatiswrongbranchthatiscorrect"));
REQUIRE(!parser.parse("branchthatiswrongbranchthatIscorrect"));
REQUIRE(!parser.parse("branchthatiswrongbranchthatIswrongbranchthatiscorrect"));
REQUIRE(!parser.parse("branchthatiswrongbranchthatIswrongbranchthatIscorrect"));
REQUIRE(
!parser.parse("branchthatiswrongbranchthatIswrongbranchthatiscorrect"));
REQUIRE(
!parser.parse("branchthatiswrongbranchthatIswrongbranchthatIscorrect"));
REQUIRE_THROWS_AS(parser.parse("branchthatiscorrect"), std::runtime_error);
REQUIRE_THROWS_AS(parser.parse("branchthatiswron_branchthatiscorrect"), std::runtime_error);
REQUIRE_THROWS_AS(parser.parse("branchthatiswron_branchthatiscorrect"),
std::runtime_error);
}
TEST_CASE("Repetition {0}", "[repetition]")
{
TEST_CASE("Repetition {0}", "[repetition]") {
parser parser(R"(
START <- '(' DIGIT{3} ') ' DIGIT{3} '-' DIGIT{4}
DIGIT <- [0-9]
@ -616,8 +599,7 @@ TEST_CASE("Repetition {0}", "[repetition]")
REQUIRE(!parser.parse("(123) 45-7a90"));
}
TEST_CASE("Repetition {2,4}", "[repetition]")
{
TEST_CASE("Repetition {2,4}", "[repetition]") {
parser parser(R"(
START <- DIGIT{2,4}
DIGIT <- [0-9]
@ -629,8 +611,7 @@ TEST_CASE("Repetition {2,4}", "[repetition]")
REQUIRE(!parser.parse("12345"));
}
TEST_CASE("Repetition {2,1}", "[repetition]")
{
TEST_CASE("Repetition {2,1}", "[repetition]") {
parser parser(R"(
START <- DIGIT{2,1} # invalid range
DIGIT <- [0-9]
@ -640,8 +621,7 @@ TEST_CASE("Repetition {2,1}", "[repetition]")
REQUIRE(!parser.parse("123"));
}
TEST_CASE("Repetition {2,}", "[repetition]")
{
TEST_CASE("Repetition {2,}", "[repetition]") {
parser parser(R"(
START <- DIGIT{2,}
DIGIT <- [0-9]
@ -652,8 +632,7 @@ TEST_CASE("Repetition {2,}", "[repetition]")
REQUIRE(parser.parse("1234"));
}
TEST_CASE("Repetition {,2}", "[repetition]")
{
TEST_CASE("Repetition {,2}", "[repetition]") {
parser parser(R"(
START <- DIGIT{,2}
DIGIT <- [0-9]
@ -664,8 +643,7 @@ TEST_CASE("Repetition {,2}", "[repetition]")
REQUIRE(!parser.parse("1234"));
}
TEST_CASE("Left recursive test", "[left recursive]")
{
TEST_CASE("Left recursive test", "[left recursive]") {
parser parser(R"(
A <- A 'a'
B <- A 'a'
@ -674,8 +652,7 @@ TEST_CASE("Left recursive test", "[left recursive]")
REQUIRE(!parser);
}
TEST_CASE("Left recursive with option test", "[left recursive]")
{
TEST_CASE("Left recursive with option test", "[left recursive]") {
parser parser(R"(
A <- 'a' / 'b'? B 'c'
B <- A
@ -684,8 +661,7 @@ TEST_CASE("Left recursive with option test", "[left recursive]")
REQUIRE(!parser);
}
TEST_CASE("Left recursive with zom test", "[left recursive]")
{
TEST_CASE("Left recursive with zom test", "[left recursive]") {
parser parser(R"(
A <- 'a'* A*
)");
@ -693,8 +669,7 @@ TEST_CASE("Left recursive with zom test", "[left recursive]")
REQUIRE(!parser);
}
TEST_CASE("Left recursive with a ZOM content rule", "[left recursive]")
{
TEST_CASE("Left recursive with a ZOM content rule", "[left recursive]") {
parser parser(R"(
A <- B
B <- _ A
@ -704,49 +679,38 @@ TEST_CASE("Left recursive with a ZOM content rule", "[left recursive]")
REQUIRE(!parser);
}
TEST_CASE("Left recursive with empty string test", "[left recursive]")
{
parser parser(
" A <- '' A"
);
TEST_CASE("Left recursive with empty string test", "[left recursive]") {
parser parser(" A <- '' A");
REQUIRE(!parser);
}
TEST_CASE("User defined rule test", "[user rule]")
{
TEST_CASE("User defined rule test", "[user rule]") {
auto g = parser(R"(
ROOT <- _ 'Hello' _ NAME '!' _
)",
{
{
"NAME", usr([](const char* s, size_t n, SemanticValues& /*sv*/, any& /*dt*/) -> size_t {
{{"NAME", usr([](const char *s, size_t n, SemanticValues &,
std::any &) -> size_t {
static std::vector<std::string> names = {"PEG", "BNF"};
for (const auto &name : names) {
if (name.size() <= n && !name.compare(0, name.size(), s, name.size())) {
if (name.size() <= n &&
!name.compare(0, name.size(), s, name.size())) {
return name.size();
}
}
return static_cast<size_t>(-1);
})
},
{
"~_", zom(cls(" \t\r\n"))
}
});
})},
{"~_", zom(cls(" \t\r\n"))}});
REQUIRE(g.parse(" Hello BNF! ") == true);
}
TEST_CASE("Semantic predicate test", "[predicate]")
{
TEST_CASE("Semantic predicate test", "[predicate]") {
parser parser("NUMBER <- [0-9]+");
parser["NUMBER"] = [](const SemanticValues& sv) {
auto val = stol(sv.token(), nullptr, 10);
if (val != 100) {
throw parse_error("value error!!");
}
parser["NUMBER"] = [](const SemanticValues &vs) {
auto val = vs.token_to_number<long>();
if (val != 100) { throw parse_error("value error!!"); }
return val;
};
@ -762,8 +726,7 @@ TEST_CASE("Semantic predicate test", "[predicate]")
REQUIRE(!parser.parse("200", val));
}
TEST_CASE("Japanese character", "[unicode]")
{
TEST_CASE("Japanese character", "[unicode]") {
peg::parser parser(u8R"(
文 <- 修飾語? 主語 述語 '。'
主語 <- 名詞 助詞
@ -781,20 +744,17 @@ TEST_CASE("Japanese character", "[unicode]")
REQUIRE(parser.parse(u8R"(サーバーを復旧します。)"));
}
TEST_CASE("dot with a code", "[unicode]")
{
TEST_CASE("dot with a code", "[unicode]") {
peg::parser parser(" S <- 'a' . 'b' ");
REQUIRE(parser.parse(u8R"(aあb)"));
}
TEST_CASE("dot with a char", "[unicode]")
{
TEST_CASE("dot with a char", "[unicode]") {
peg::parser parser(" S <- 'a' . 'b' ");
REQUIRE(parser.parse(u8R"(aåb)"));
}
TEST_CASE("character class", "[unicode]")
{
TEST_CASE("character class", "[unicode]") {
peg::parser parser(R"(
S <- 'a' [い-おAさC-Eた-とは] 'b'
)");
@ -828,8 +788,7 @@ TEST_CASE("dot with a grapheme", "[unicode]")
}
#endif
TEST_CASE("Macro simple test", "[macro]")
{
TEST_CASE("Macro simple test", "[macro]") {
parser parser(R"(
S <- HELLO WORLD
HELLO <- T('hello')
@ -840,8 +799,7 @@ TEST_CASE("Macro simple test", "[macro]")
REQUIRE(parser.parse("hello \tworld "));
}
TEST_CASE("Macro two parameters", "[macro]")
{
TEST_CASE("Macro two parameters", "[macro]") {
parser parser(R"(
S <- HELLO_WORLD
HELLO_WORLD <- T('hello', 'world')
@ -851,8 +809,7 @@ TEST_CASE("Macro two parameters", "[macro]")
REQUIRE(parser.parse("hello \tworld "));
}
TEST_CASE("Macro syntax error", "[macro]")
{
TEST_CASE("Macro syntax error", "[macro]") {
parser parser(R"(
S <- T('hello')
T (a) <- a [ \t]*
@ -862,8 +819,7 @@ TEST_CASE("Macro syntax error", "[macro]")
REQUIRE(ret == false);
}
TEST_CASE("Macro missing argument", "[macro]")
{
TEST_CASE("Macro missing argument", "[macro]") {
parser parser(R"(
S <- T ('hello')
T(a, b) <- a [ \t]* b
@ -873,8 +829,7 @@ TEST_CASE("Macro missing argument", "[macro]")
REQUIRE(ret == false);
}
TEST_CASE("Macro reference syntax error", "[macro]")
{
TEST_CASE("Macro reference syntax error", "[macro]") {
parser parser(R"(
S <- T ('hello')
T(a) <- a [ \t]*
@ -884,8 +839,7 @@ TEST_CASE("Macro reference syntax error", "[macro]")
REQUIRE(ret == false);
}
TEST_CASE("Macro invalid macro reference error", "[macro]")
{
TEST_CASE("Macro invalid macro reference error", "[macro]") {
parser parser(R"(
S <- T('hello')
T <- 'world'
@ -895,8 +849,7 @@ TEST_CASE("Macro invalid macro reference error", "[macro]")
REQUIRE(ret == false);
}
TEST_CASE("Macro calculator", "[macro]")
{
TEST_CASE("Macro calculator", "[macro]") {
// Create a PEG parser
parser parser(R"(
# Grammar for simple calculator...
@ -912,11 +865,11 @@ TEST_CASE("Macro calculator", "[macro]")
)");
// Setup actions
auto reduce = [](const SemanticValues& sv) -> long {
auto result = any_cast<long>(sv[0]);
for (auto i = 1u; i < sv.size(); i += 2) {
auto num = any_cast<long>(sv[i + 1]);
auto ope = any_cast<char>(sv[i]);
auto reduce = [](const SemanticValues &vs) {
auto result = std::any_cast<long>(vs[0]);
for (auto i = 1u; i < vs.size(); i += 2) {
auto num = std::any_cast<long>(vs[i + 1]);
auto ope = std::any_cast<char>(vs[i]);
switch (ope) {
case '+': result += num; break;
case '-': result -= num; break;
@ -929,9 +882,13 @@ TEST_CASE("Macro calculator", "[macro]")
parser["EXPRESSION"] = reduce;
parser["TERM"] = reduce;
parser["TERM_OPERATOR"] = [](const SemanticValues& sv) { return static_cast<char>(*sv.c_str()); };
parser["FACTOR_OPERATOR"] = [](const SemanticValues& sv) { return static_cast<char>(*sv.c_str()); };
parser["NUMBER"] = [](const SemanticValues& sv) { return atol(sv.c_str()); };
parser["TERM_OPERATOR"] = [](const SemanticValues &vs) {
return static_cast<char>(*vs.sv().data());
};
parser["FACTOR_OPERATOR"] = [](const SemanticValues &vs) {
return static_cast<char>(*vs.sv().data());
};
parser["NUMBER"] = [](const SemanticValues &vs) { return vs.token_to_number<long>(); };
bool ret = parser;
REQUIRE(ret == true);
@ -944,8 +901,7 @@ TEST_CASE("Macro calculator", "[macro]")
REQUIRE(val == -3);
}
TEST_CASE("Macro expression arguments", "[macro]")
{
TEST_CASE("Macro expression arguments", "[macro]") {
parser parser(R"(
S <- M('hello' / 'Hello', 'world' / 'World')
M(arg0, arg1) <- arg0 [ \t]+ arg1
@ -954,8 +910,7 @@ TEST_CASE("Macro expression arguments", "[macro]")
REQUIRE(parser.parse("Hello world"));
}
TEST_CASE("Macro recursive", "[macro]")
{
TEST_CASE("Macro recursive", "[macro]") {
parser parser(R"(
S <- M('abc')
M(s) <- !s / s ' ' M(s / '123') / s
@ -967,8 +922,7 @@ TEST_CASE("Macro recursive", "[macro]")
REQUIRE(parser.parse("abc 123 abc"));
}
TEST_CASE("Macro recursive2", "[macro]")
{
TEST_CASE("Macro recursive2", "[macro]") {
auto syntaxes = std::vector<const char *>{
"S <- M('abc') M(s) <- !s / s ' ' M(s* '-' '123') / s",
"S <- M('abc') M(s) <- !s / s ' ' M(s+ '-' '123') / s",
@ -985,8 +939,7 @@ TEST_CASE("Macro recursive2", "[macro]")
}
}
TEST_CASE("Macro exclusive modifiers", "[macro]")
{
TEST_CASE("Macro exclusive modifiers", "[macro]") {
parser parser(R"(
S <- Modifiers(!"") _
Modifiers(Appeared) <- (!Appeared) (
@ -1006,8 +959,7 @@ TEST_CASE("Macro exclusive modifiers", "[macro]")
REQUIRE(!parser.parse("public static public"));
}
TEST_CASE("Macro token check test", "[macro]")
{
TEST_CASE("Macro token check test", "[macro]") {
parser parser(R"(
# Grammar for simple calculator...
EXPRESSION <- _ LIST(TERM, TERM_OPERATOR)
@ -1053,8 +1005,7 @@ TEST_CASE("Nested macro call", "[macro]") {
REQUIRE(parser.parse("val"));
}
TEST_CASE("Nested macro call2", "[macro]")
{
TEST_CASE("Nested macro call2", "[macro]") {
parser parser(R"(
START <- A('TestVal1', 'TestVal2')+
A(Aarg1, Aarg2) <- B(Aarg1) '#End'
@ -1072,8 +1023,8 @@ TEST_CASE("Line information test", "[line information]") {
)");
std::vector<std::pair<size_t, size_t>> locations;
parser["WORD"] = [&](const peg::SemanticValues& sv) {
locations.push_back(sv.line_info());
parser["WORD"] = [&](const peg::SemanticValues &vs) {
locations.push_back(vs.line_info());
};
bool ret = parser;
@ -1091,8 +1042,7 @@ TEST_CASE("Line information test", "[line information]") {
REQUIRE(locations[6] == std::make_pair<size_t, size_t>(3, 1));
}
TEST_CASE("Dictionary", "[dic]")
{
TEST_CASE("Dictionary", "[dic]") {
parser parser(R"(
START <- 'This month is ' MONTH '.'
MONTH <- 'Jan' | 'January' | 'Feb' | 'February'
@ -1104,8 +1054,7 @@ TEST_CASE("Dictionary", "[dic]")
REQUIRE_FALSE(parser.parse("This month is ."));
}
TEST_CASE("Dictionary invalid", "[dic]")
{
TEST_CASE("Dictionary invalid", "[dic]") {
parser parser(R"(
START <- 'This month is ' MONTH '.'
MONTH <- 'Jan' | 'January' | [a-z]+ | 'Feb' | 'February'

Write Preview
Loading…
Cancel
Save