AIR/cpp-peglib
1
0
Fork 0
mirror of https://github.com/yhirose/cpp-peglib.git synced 2024-12-22 11:55:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #124 from yhirose/cpp17

Browse Source 
C++17 support
This commit is contained in:
yhirose 2020-12-19 19:36:33 -05:00 committed by GitHub
commit 78eb383fa1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 1913 additions and 2266 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
.github/workflows/cmake.yml vendored Normal file
View File

@ -0,0 +1,49 @@
name: CMake
on: [push, pull_request]
env:
# Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.)
BUILD_TYPE: Release
jobs:
build:
# The CMake configure and build commands are platform agnostic and should work equally
# well on Windows or Mac. You can convert this to a matrix build if you need
# cross-platform coverage.
# See: https://docs.github.com/en/actions/configuring-and-managing-workflows/configuring-a-workflow#configuring-a-build-matrix
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ubuntu-20.04, macos-latest, windows-latest]
steps:
- uses: actions/checkout@v2
- name: Create Build Environment
# Some projects don't allow in-source building, so create a separate build directory
# We'll use this as our working directory for all subsequent commands
run: cmake -E make_directory ${{runner.workspace}}/build
- name: Configure CMake
# Use a bash shell so we can use the same syntax for environment variable
# access regardless of the host operating system
shell: bash
working-directory: ${{runner.workspace}}/build
# Note the current convention is to use the -S and -B options here to specify source
# and build directories, but this is only available with CMake 3.13 and higher.
# The CMake binaries on the Github Actions machines are (as of this writing) 3.12
run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE
- name: Build
working-directory: ${{runner.workspace}}/build
shell: bash
# Execute the build. You can specify a specific target with "--target <NAME>"
run: cmake --build . --config $BUILD_TYPE
- name: Test
working-directory: ${{runner.workspace}}/build
shell: bash
# Execute tests defined by the CMake configuration.
# See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail
run: ctest -C $BUILD_TYPE

33
.travis.yml
View File

@ -1,39 +1,20 @@
language: cpp
sudo: false
matrix:
include:
- compiler: gcc
- os: linux
addons:
apt:
sources:
- ubuntu-toolchain-r-test
- george-edison55-precise-backports
packages:
- cmake
- cmake-data
- gcc-4.9
- g++-4.9
env: COMPILER=g++-4.9
# - compiler: clang
# addons:
# apt:
# sources:
# - kubuntu-backports
# - ubuntu-toolchain-r-test
# - llvm-toolchain-precise-3.7
# packages:
# - cmake
# - clang-3.7
# env: COMPILER=clang++-3.7
- g++-8
env:
- MATRIX_EVAL="CC=gcc-8 && CXX=g++-8"
branches:
only:
- master
before_script:
- export CXX=$COMPILER
before_install:
- eval "${MATRIX_EVAL}"
script:
- mkdir build && cd build
- cmake .. && make && ctest -V
- cmake .. && make && ./test/test-main

34
CMakeLists.txt
View File

@ -1,33 +1,11 @@
cmake_minimum_required(VERSION 3.1.0)
project("cpp-peglib")
# Check if a supported compiler is used to setup the C++ standard to use:
get_property(known_features GLOBAL PROPERTY CMAKE_CXX_KNOWN_FEATURES)
list(FIND known_features "cxx_std_17" found)
if(NOT ${found} EQUAL -1)
# C++17 standard is supported
set(CMAKE_CXX_STANDARD 17)
else()
# Check for C++11 standard support
list(FIND known_features "cxx_std_11" found)
if(NOT ${found} EQUAL -1)
# C++11 standard is supported
set(CMAKE_CXX_STANDARD 11)
endif()
endif()
if(${found} EQUAL -1)
message(FATAL_ERROR "Your compiler is not supported.")
endif()
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_EXTENSIONS OFF)
if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang" OR CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Weverything -Wno-c++98-compat -Wno-padded -Wno-weak-vtables -Wno-exit-time-destructors -Wno-c++2a-compat -Wno-switch-enum -Wno-c++98-compat-pedantic")
elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wpedantic -Wextra -Woverloaded-virtual")
elseif(MSVC)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /W4 /wd4503 /wd4512 /utf-8")
if(MSVC)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zc:__cplusplus /utf-8")
endif()
set(THREADS_PREFER_PTHREAD_FLAG ON)
@ -37,10 +15,10 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
set(add_link_deps Threads::Threads)
endif()
enable_testing()
add_subdirectory(test)
add_subdirectory(example)
add_subdirectory(lint)
add_subdirectory(test)
enable_testing()
install(FILES peglib.h DESTINATION include)

95
README.md
View File

@ -1,10 +1,13 @@
cpp-peglib
==========
[![](https://github.com/yhirose/cpp-peglib/workflows/CMake/badge.svg)](https://github.com/yhirose/cpp-peglib/actions)
[![Build Status](https://travis-ci.org/yhirose/cpp-peglib.svg?branch=master)](https://travis-ci.org/yhirose/cpp-peglib)
[![Bulid Status](https://ci.appveyor.com/api/projects/status/github/yhirose/cpp-peglib?branch=master&svg=true)](https://ci.appveyor.com/project/yhirose/cpp-peglib)
C++11 header-only [PEG](http://en.wikipedia.org/wiki/Parsing_expression_grammar) (Parsing Expression Grammars) library. You can start using it right away just by including `peglib.h` in your project.
C++17 header-only [PEG](http://en.wikipedia.org/wiki/Parsing_expression_grammar) (Parsing Expression Grammars) library. You can start using it right away just by including `peglib.h` in your project.
Since this library only supports C++17 compilers, please make sure that compiler the option `-std=c++17` is enabled. (`/std:c++17 /Zc:__cplusplus` for MSVC)
You can also try the online version, PEG Playground at https://yhirose.github.io/cpp-peglib.
@ -54,29 +57,29 @@ int main(void) {
%whitespace <- [ \t]*
)");
assert((bool)parser == true);
assert(static_cast<bool>(parser) == true);
// (3) Setup actions
parser["Additive"] = [](const SemanticValues& sv) {
switch (sv.choice()) {
parser["Additive"] = [](const SemanticValues &vs) {
switch (vs.choice()) {
case 0: // "Multitive '+' Additive"
return any_cast<int>(sv[0]) + any_cast<int>(sv[1]);
return any_cast<int>(vs[0]) + any_cast<int>(vs[1]);
default: // "Multitive"
return any_cast<int>(sv[0]);
return any_cast<int>(vs[0]);
}
};
parser["Multitive"] = [](const SemanticValues& sv) {
switch (sv.choice()) {
parser["Multitive"] = [](const SemanticValues &vs) {
switch (vs.choice()) {
case 0: // "Primary '*' Multitive"
return any_cast<int>(sv[0]) * any_cast<int>(sv[1]);
return any_cast<int>(vs[0]) * any_cast<int>(vs[1]);
default: // "Primary"
return any_cast<int>(sv[0]);
return any_cast<int>(vs[0]);
}
};
parser["Number"] = [](const SemanticValues& sv) {
return stoi(sv.token(), nullptr, 10);
parser["Number"] = [](const SemanticValues &vs) {
return vs.token_to_number<int>();
};
// (4) Parse
@ -114,10 +117,10 @@ assert(ok);
There are four semantic actions available:
```cpp
[](const SemanticValues& sv, any& dt)
[](const SemanticValues& sv)
[](SemanticValues& sv, any& dt)
[](SemanticValues& sv)
[](const SemanticValues& vs, any& dt)
[](const SemanticValues& vs)
[](SemanticValues& vs, any& dt)
[](SemanticValues& vs)
```
`SemanticValues` value contains the following information:
@ -129,16 +132,7 @@ There are four semantic actions available:
`any& dt` is a 'read-write' context data which can be used for whatever purposes. The initial context data is set in `peg::parser::parse` method.
`peg::any` is a simpler implementatin of std::any. If the compiler in use supports C++17, by default `peg::any` is defined as an alias to `std::any`.
To force using the simpler `any` implementation that comes with `cpp-peglib`, define `PEGLIB_USE_STD_ANY` as 0 before including `peglib.h`:
```cpp
#define PEGLIB_USE_STD_ANY 0
#include <peglib.h>
[...]
```
A semantic action can return a value of arbitrary data type, which will be wrapped by `peg::any`. If a user returns nothing in a semantic action, the first semantic value in the `const SemanticValues& sv` argument will be returned. (Yacc parser has the same behavior.)
A semantic action can return a value of arbitrary data type, which will be wrapped by `peg::any`. If a user returns nothing in a semantic action, the first semantic value in the `const SemanticValues& vs` argument will be returned. (Yacc parser has the same behavior.)
Here shows the `SemanticValues` structure:
@ -150,21 +144,18 @@ struct SemanticValues : protected std::vector<any>
const char* ss;
// Matched string
std::string str() const; // Matched string
const char* c_str() const; // Matched string start
size_t length() const; // Matched string length
std::string_view sv() const { return sv_; }
// Line number and column at which the matched string is
std::pair<size_t, size_t> line_info() const;
// Tokens
std::vector<
std::pair<
const char*, // Token start
size_t>> // Token length
tokens;
std::vector<std::string_view> tokens;
std::string_view token(size_t id = 0) const;
std::string token(size_t id = 0) const;
// Token conversion
std::string token_to_string(size_t id = 0) const;
template <typename T> T token_to_number() const;
// Choice number (0 based index)
size_t choice() const;
@ -183,9 +174,9 @@ peg::parser parser(R"(
_ <- [ \t\r\n]*
)");
parser["TOKEN"] = [](const SemanticValues& sv) {
parser["TOKEN"] = [](const SemanticValues& vs) {
// 'token' doesn't include trailing whitespaces
auto token = sv.token();
auto token = vs.token();
};
auto ret = parser.parse(" token1, token2 ");
@ -200,8 +191,8 @@ peg::parser parser(R"(
~_ <- [ \t]*
)");
parser["ROOT"] = [&](const SemanticValues& sv) {
assert(sv.size() == 2); // should be 2 instead of 5.
parser["ROOT"] = [&](const SemanticValues& vs) {
assert(vs.size() == 2); // should be 2 instead of 5.
};
auto ret = parser.parse(" item1, item2 ");
@ -222,8 +213,8 @@ peg::parser parser(R"(
```cpp
peg::parser parser("NUMBER <- [0-9]+");
parser["NUMBER"] = [](const SemanticValues& sv) {
auto val = stol(sv.str(), nullptr, 10);
parser["NUMBER"] = [](const SemanticValues& vs) {
auto val = vs.token_to_number<long>();
if (val != 100) {
throw peg::parse_error("value error!!");
}
@ -246,7 +237,7 @@ parser["RULE"].enter = [](const char* s, size_t n, any& dt) {
std::cout << "enter" << std::endl;
};
parser["RULE"] = [](const SemanticValues& sv, any& dt) {
parser["RULE"] = [](const SemanticValues& vs, any& dt) {
std::cout << "action!" << std::endl;
};
@ -373,11 +364,11 @@ parser parser(R"(
}
)");
parser["INFIX_EXPRESSION"] = [](const SemanticValues& sv) -> long {
auto result = any_cast<long>(sv[0]);
if (sv.size() > 1) {
auto ope = any_cast<char>(sv[1]);
auto num = any_cast<long>(sv[2]);
parser["INFIX_EXPRESSION"] = [](const SemanticValues& vs) -> long {
auto result = any_cast<long>(vs[0]);
if (vs.size() > 1) {
auto ope = any_cast<char>(vs[1]);
auto num = any_cast<long>(vs[2]);
switch (ope) {
case '+': result += num; break;
case '-': result -= num; break;
@ -387,8 +378,8 @@ parser["INFIX_EXPRESSION"] = [](const SemanticValues& sv) -> long {
}
return result;
};
parser["OPERATOR"] = [](const SemanticValues& sv) { return *sv.c_str(); };
parser["NUMBER"] = [](const SemanticValues& sv) { return atol(sv.c_str()); };
parser["OPERATOR"] = [](const SemanticValues& vs) { return *vs.sv(); };
parser["NUMBER"] = [](const SemanticValues& vs) { return vs.token_to_number<long>(); };
long val;
parser.parse(" -1 + (1 + 2) * 3 - -1", val);
@ -446,8 +437,8 @@ vector<string> tags;
Definition ROOT, TAG_NAME, _;
ROOT <= seq(_, zom(seq(chr('['), TAG_NAME, chr(']'), _)));
TAG_NAME <= oom(seq(npd(chr(']')), dot())), [&](const SemanticValues& sv) {
tags.push_back(sv.str());
TAG_NAME <= oom(seq(npd(chr(']')), dot())), [&](const SemanticValues& vs) {
tags.push_back(vs.str());
};
_ <= zom(cls(" \t"));
@ -492,7 +483,7 @@ auto syntax = R"(
Rules additional_rules = {
{
"NAME", usr([](const char* s, size_t n, SemanticValues& sv, any& dt) -> size_t {
"NAME", usr([](const char* s, size_t n, SemanticValues& vs, any& dt) -> size_t {
static vector<string> names = { "PEG", "BNF" };
for (const auto& name: names) {
if (name.size() <= n && !name.compare(0, name.size(), s, name.size())) {

3
appveyor.yml
View File

@ -2,9 +2,6 @@ clone_depth: 5
environment:
matrix:
- JOB: Visual Studio 2015
APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
CMAKE_GENERATOR: "Visual Studio 14 2015"
- JOB: Visual Studio 2017
APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
CMAKE_GENERATOR: "Visual Studio 15 2017"

2
docs/build.sh
View File

@ -1,2 +1,2 @@
source ~/Projects/emsdk/emsdk_env.sh
emcc -std=c++11 -O3 --bind -o native.js native.cpp
emcc -std=c++17 -O3 --bind -o native.js native.cpp

2
docs/native.js
View File

File diff suppressed because one or more lines are too long

BIN
docs/native.wasm
View File

Binary file not shown.

8
example/CMakeLists.txt
View File

@ -1,13 +1,9 @@
cmake_minimum_required(VERSION 2.8)
cmake_minimum_required(VERSION 3.1)
project(example)
include_directories(..)
if(MSVC)
add_compile_options(${cxx11_options} /W3)
else()
add_compile_options(${cxx11_options})
endif()
set(CMAKE_CXX_STANDARD 17)
add_executable(calc calc.cc)
target_link_libraries(calc ${add_link_deps})

22
example/calc.cc
View File

@ -1,6 +1,6 @@
#include <peglib.h>
#include <assert.h>
#include <iostream>
#include <peglib.h>
using namespace peg;
using namespace std;
@ -19,26 +19,26 @@ int main(void) {
assert(static_cast<bool>(parser) == true);
// (3) Setup actions
parser["Additive"] = [](const SemanticValues& sv) {
switch (sv.choice()) {
parser["Additive"] = [](const SemanticValues &vs) {
switch (vs.choice()) {
case 0: // "Multitive '+' Additive"
return any_cast<int>(sv[0]) + any_cast<int>(sv[1]);
return any_cast<int>(vs[0]) + any_cast<int>(vs[1]);
default: // "Multitive"
return any_cast<int>(sv[0]);
return any_cast<int>(vs[0]);
}
};
parser["Multitive"] = [](const SemanticValues& sv) {
switch (sv.choice()) {
parser["Multitive"] = [](const SemanticValues &vs) {
switch (vs.choice()) {
case 0: // "Primary '*' Multitive"
return any_cast<int>(sv[0]) * any_cast<int>(sv[1]);
return any_cast<int>(vs[0]) * any_cast<int>(vs[1]);
default: // "Primary"
return any_cast<int>(sv[0]);
return any_cast<int>(vs[0]);
}
};
parser["Number"] = [](const SemanticValues& sv) {
return stoi(sv.token(), nullptr, 10);
parser["Number"] = [](const SemanticValues &vs) {
return vs.token_to_number<int>();
};
// (4) Parse

26
example/calc2.cc
View File

@ -5,9 +5,9 @@
// MIT License
//
#include <peglib.h>
#include <iostream>
#include <cstdlib>
#include <iostream>
#include <peglib.h>
using namespace peg;
@ -21,18 +21,17 @@ using namespace peg;
// FACTOR_OPERATOR <- [/*]
// NUMBER <- [0-9]+
//
int main(int argc, const char** argv)
{
int main(int argc, const char **argv) {
if (argc < 2 || std::string("--help") == argv[1]) {
std::cout << "usage: calc [formula]" << std::endl;
return 1;
}
auto reduce = [](const SemanticValues& sv) -> long {
auto result = any_cast<long>(sv[0]);
for (auto i = 1u; i < sv.size(); i += 2) {
auto num = any_cast<long>(sv[i + 1]);
auto ope = any_cast<char>(sv[i]);
auto reduce = [](const SemanticValues &vs) {
auto result = std::any_cast<long>(vs[0]);
for (auto i = 1u; i < vs.size(); i += 2) {
auto num = std::any_cast<long>(vs[i + 1]);
auto ope = std::any_cast<char>(vs[i]);
switch (ope) {
case '+': result += num; break;
case '-': result -= num; break;
@ -48,9 +47,12 @@ int main(int argc, const char** argv)
EXPRESSION <= seq(TERM, zom(seq(TERM_OPERATOR, TERM))), reduce;
TERM <= seq(FACTOR, zom(seq(FACTOR_OPERATOR, FACTOR))), reduce;
FACTOR <= cho(NUMBER, seq(chr('('), EXPRESSION, chr(')')));
TERM_OPERATOR <= cls("+-"), [](const SemanticValues& sv) { return static_cast<char>(*sv.c_str()); };
FACTOR_OPERATOR <= cls("*/"), [](const SemanticValues& sv) { return static_cast<char>(*sv.c_str()); };
NUMBER <= oom(cls("0-9")), [](const SemanticValues& sv) { return atol(sv.c_str()); };
TERM_OPERATOR <= cls("+-"),
[](const SemanticValues &vs) { return static_cast<char>(*vs.sv().data()); };
FACTOR_OPERATOR <= cls("*/"),
[](const SemanticValues &vs) { return static_cast<char>(*vs.sv().data()); };
NUMBER <= oom(cls("0-9")),
[](const SemanticValues &vs) { return vs.token_to_number<long>(); };
auto expr = argv[1];
long val = 0;

13
example/calc3.cc
View File

@ -5,24 +5,23 @@
// MIT License
//
#include <peglib.h>
#include <iostream>
#include <cstdlib>
#include <iostream>
#include <peglib.h>
using namespace peg;
int main(int argc, const char** argv)
{
int main(int argc, const char **argv) {
if (argc < 2 || std::string("--help") == argv[1]) {
std::cout << "usage: calc3 [formula]" << std::endl;
return 1;
}
std::function<long (const Ast&)> eval = [&](const Ast& ast) {
std::function<long(const Ast &)> eval = [&](const Ast &ast) {
if (ast.name == "NUMBER") {
return stol(ast.token);
return ast.token_to_number<long>();
} else {
const auto& nodes = ast.nodes;
const auto &nodes = ast.nodes;
auto result = eval(*nodes[0]);
for (auto i = 1u; i < nodes.size(); i += 2) {
auto num = eval(*nodes[i + 1]);

16
example/calc4.cc
View File

@ -1,6 +1,6 @@
#include <peglib.h>
#include <assert.h>
#include <iostream>
#include <peglib.h>
using namespace peg;
using namespace std;
@ -18,11 +18,11 @@ int main(void) {
%whitespace <- [ \t\r\n]*
)");
parser["EXPRESSION"] = [](const SemanticValues& sv) -> long {
auto result = any_cast<long>(sv[0]);
if (sv.size() > 1) {
auto ope = any_cast<char>(sv[1]);
auto num = any_cast<long>(sv[2]);
parser["EXPRESSION"] = [](const SemanticValues &vs) {
auto result = any_cast<long>(vs[0]);
if (vs.size() > 1) {
auto ope = any_cast<char>(vs[1]);
auto num = any_cast<long>(vs[2]);
switch (ope) {
case '+': result += num; break;
case '-': result -= num; break;
@ -32,8 +32,8 @@ int main(void) {
}
return result;
};
parser["OPERATOR"] = [](const SemanticValues& sv) { return *sv.c_str(); };
parser["NUMBER"] = [](const SemanticValues& sv) { return atol(sv.c_str()); };
parser["OPERATOR"] = [](const SemanticValues &vs) { return *vs.sv().data(); };
parser["NUMBER"] = [](const SemanticValues &vs) { return atol(vs.sv().data()); };
long val;
parser.parse(" -1 + (1 + 2) * 3 - -1", val);

13
example/calc5.cc
View File

@ -5,24 +5,23 @@
// MIT License
//
#include <peglib.h>
#include <iostream>
#include <cstdlib>
#include <iostream>
#include <peglib.h>
using namespace peg;
int main(int argc, const char** argv)
{
int main(int argc, const char **argv) {
if (argc < 2 || std::string("--help") == argv[1]) {
std::cout << "usage: calc5 [formula]" << std::endl;
return 1;
}
std::function<long (const Ast&)> eval = [&](const Ast& ast) {
std::function<long(const Ast &)> eval = [&](const Ast &ast) {
if (ast.name == "NUMBER") {
return stol(ast.token);
return ast.token_to_number<long>();
} else {
const auto& nodes = ast.nodes;
const auto &nodes = ast.nodes;
auto result = eval(*nodes[0]);
if (nodes.size() > 1) {
auto ope = nodes[1]->token[0];

14
lint/CMakeLists.txt
View File

@ -1,9 +1,9 @@
cmake_minimum_required(VERSION 2.8)
cmake_minimum_required(VERSION 3.1)
project(peglint)
include_directories(..)
add_definitions("-std=c++11")
add_executable(peglint peglint.cc)
if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
target_link_libraries(peglint ${add_link_deps})
endif()
include_directories(..)
set(CMAKE_CXX_STANDARD 17)
add_executable(peglint peglint.cc)
target_link_libraries(peglint ${add_link_deps})

9
lint/peglint.cc
View File

@ -131,7 +131,7 @@ int main(int argc, const char **argv) {
parser.enable_trace(
[&](const char *name, const char *s, size_t /*n*/,
const peg::SemanticValues & /*sv*/, const peg::Context &c,
const peg::any & /*dt*/) {
const std::any & /*dt*/) {
auto pos = static_cast<size_t>(s - c.s);
auto backtrack = (pos < prev_pos ? "*" : "");
string indent;
@ -145,7 +145,7 @@ int main(int argc, const char **argv) {
},
[&](const char *name, const char *s, size_t /*n*/,
const peg::SemanticValues &sv, const peg::Context &c,
const peg::any & /*dt*/, size_t len) {
const std::any & /*dt*/, size_t len) {
auto pos = static_cast<size_t>(s - c.s);
if (len != static_cast<size_t>(-1)) { pos += len; }
string indent;
@ -160,8 +160,9 @@ int main(int argc, const char **argv) {
}
std::string token;
if (!sv.tokens.empty()) {
const auto &tok = sv.tokens[0];
token += " '" + std::string(tok.first, tok.second) + "'";
token += " '";
token += sv.tokens[0];
token +=+ "'";
}
std::cout << "L " << pos << "\t" << indent << ret << name << " #"
<< c.trace_ids.back() << choice.str() << token << std::endl;

1332
peglib.h
View File

File diff suppressed because it is too large Load Diff

11
pl0/CMakeLists.txt
View File

@ -1,14 +1,11 @@
cmake_minimum_required(VERSION 3.1)
project(tcp)
enable_language(CXX)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_CXX_STANDARD 11)
set(CMAKE_CXX_EXTENSIONS OFF)
project(pl0)
include_directories(..)
add_executable(pl0 pl0.cc)
set(CMAKE_CXX_STANDARD 17)
add_executable(pl0 pl0.cc)
find_package(LLVM REQUIRED CONFIG)
set(add_link_deps ${add_link_deps} LLVM)
target_include_directories(pl0 PUBLIC ${LLVM_INCLUDE_DIRS})

56
pl0/pl0.cc
View File

@ -182,11 +182,11 @@ struct SymbolTable {
// _)?
const auto& nodes = ast->nodes;
for (auto i = 0u; i < nodes.size(); i += 2) {
const auto& ident = nodes[i + 0]->token;
const auto& ident = nodes[i + 0]->token_to_string();
if (scope->has_symbol(ident)) {
throw_runtime_error(nodes[i], "'" + ident + "' is already defined...");
}
auto number = stoi(nodes[i + 1]->token);
auto number = nodes[i + 1]->token_to_number<int>();
scope->constants.emplace(ident, number);
}
}
@ -196,7 +196,7 @@ struct SymbolTable {
// var <- ('VAR' __ ident(',' _ ident)* ';' _) ?
const auto& nodes = ast->nodes;
for (auto i = 0u; i < nodes.size(); i += 1) {
const auto& ident = nodes[i]->token;
const auto& ident = nodes[i]->token_to_string();
if (scope->has_symbol(ident)) {
throw_runtime_error(nodes[i], "'" + ident + "' is already defined...");
}
@ -209,7 +209,7 @@ struct SymbolTable {
// procedure <- ('PROCEDURE' __ ident ';' _ block ';' _)*
const auto& nodes = ast->nodes;
for (auto i = 0u; i < nodes.size(); i += 2) {
const auto& ident = nodes[i + 0]->token;
const auto& ident = nodes[i + 0]->token_to_string();
auto block = nodes[i + 1];
scope->procedures[ident] = block;
build_on_ast(block, scope);
@ -219,7 +219,7 @@ struct SymbolTable {
static void assignment(const shared_ptr<AstPL0> ast,
shared_ptr<SymbolScope> scope) {
// assignment <- ident ':=' _ expression
const auto& ident = ast->nodes[0]->token;
const auto& ident = ast->nodes[0]->token_to_string();
if (scope->has_constant(ident)) {
throw_runtime_error(ast->nodes[0],
"cannot modify constant value '" + ident + "'...");
@ -238,7 +238,7 @@ struct SymbolTable {
static void call(const shared_ptr<AstPL0> ast,
shared_ptr<SymbolScope> scope) {
// call <- 'CALL' __ ident
const auto& ident = ast->nodes[0]->token;
const auto& ident = ast->nodes[0]->token_to_string();
if (!scope->has_procedure(ident)) {
throw_runtime_error(ast->nodes[0],
"undefined procedure '" + ident + "'...");
@ -256,7 +256,7 @@ struct SymbolTable {
static void ident(const shared_ptr<AstPL0> ast,
shared_ptr<SymbolScope> scope) {
const auto& ident = ast->token;
const auto& ident = ast->token_to_string();
if (!scope->has_symbol(ident)) {
throw_runtime_error(ast, "undefined variable '" + ident + "'...");
}
@ -360,13 +360,13 @@ struct Interpreter {
static void exec_assignment(const shared_ptr<AstPL0> ast,
shared_ptr<Environment> env) {
// assignment <- ident ':=' _ expression
env->set_variable(ast->nodes[0]->token, eval(ast->nodes[1], env));
env->set_variable(ast->nodes[0]->token_to_string(), eval(ast->nodes[1], env));
}
static void exec_call(const shared_ptr<AstPL0> ast,
shared_ptr<Environment> env) {
// call <- 'CALL' __ ident
exec_block(env->get_procedure(ast->nodes[0]->token), env);
exec_block(env->get_procedure(ast->nodes[0]->token_to_string()), env);
}
static void exec_statements(const shared_ptr<AstPL0> ast,
@ -406,7 +406,7 @@ struct Interpreter {
// in <- ('in' __ / 'read' __ / '?' _) ident
int val;
cin >> val;
env->set_variable(ast->nodes[0]->token, val);
env->set_variable(ast->nodes[0]->token_to_string(), val);
}
static bool eval_condition(const shared_ptr<AstPL0> ast,
@ -434,7 +434,7 @@ struct Interpreter {
// compare <- expression compare_op expression
const auto& nodes = ast->nodes;
auto lval = eval_expression(nodes[0], env);
auto op = peg::str2tag(nodes[1]->token.c_str());
auto op = peg::str2tag(nodes[1]->token_to_string().c_str());
auto rval = eval_expression(nodes[2], env);
switch (op) {
case "="_:
@ -473,11 +473,11 @@ struct Interpreter {
shared_ptr<Environment> env) {
// expression <- sign term (term_op term)*
const auto& nodes = ast->nodes;
auto sign = nodes[0]->token;
auto sign = nodes[0]->token_to_string();
auto sign_val = (sign.empty() || sign == "+") ? 1 : -1;
auto val = eval(nodes[1], env) * sign_val;
for (auto i = 2u; i < nodes.size(); i += 2) {
auto ope = nodes[i + 0]->token[0];
auto ope = nodes[i + 0]->token_to_string()[0];
auto rval = eval(nodes[i + 1], env);
switch (ope) {
case '+':
@ -497,7 +497,7 @@ struct Interpreter {
const auto& nodes = ast->nodes;
auto val = eval(nodes[0], env);
for (auto i = 1u; i < nodes.size(); i += 2) {
auto ope = nodes[i + 0]->token[0];
auto ope = nodes[i + 0]->token_to_string()[0];
auto rval = eval(nodes[i + 1], env);
switch (ope) {
case '*':
@ -516,12 +516,12 @@ struct Interpreter {
static int eval_ident(const shared_ptr<AstPL0> ast,
shared_ptr<Environment> env) {
return env->get_value(ast, ast->token);
return env->get_value(ast, ast->token_to_string());
}
static int eval_number(const shared_ptr<AstPL0> ast,
shared_ptr<Environment> env) {
return stol(ast->token);
return stol(ast->token_to_string());
}
};
@ -653,8 +653,8 @@ struct LLVM {
void compile_const(const shared_ptr<AstPL0> ast) {
for (auto i = 0u; i < ast->nodes.size(); i += 2) {
auto ident = ast->nodes[i]->token;
auto number = stoi(ast->nodes[i + 1]->token);
auto ident = ast->nodes[i]->token_to_string();
auto number = stoi(ast->nodes[i + 1]->token_to_string());
auto alloca =
builder_.CreateAlloca(builder_.getInt32Ty(), nullptr, ident);
@ -664,14 +664,14 @@ struct LLVM {
void compile_var(const shared_ptr<AstPL0> ast) {
for (const auto node : ast->nodes) {
auto ident = node->token;
auto ident = node->token_to_string();
builder_.CreateAlloca(builder_.getInt32Ty(), nullptr, ident);
}
}
void compile_procedure(const shared_ptr<AstPL0> ast) {
for (auto i = 0u; i < ast->nodes.size(); i += 2) {
auto ident = ast->nodes[i]->token;
auto ident = ast->nodes[i]->token_to_string();
auto block = ast->nodes[i + 1];
std::vector<Type*> pt(block->scope->free_variables.size(),
@ -712,7 +712,7 @@ struct LLVM {
}
void compile_assignment(const shared_ptr<AstPL0> ast) {
auto ident = ast->nodes[0]->token;
auto ident = ast->nodes[0]->token_to_string();
auto fn = builder_.GetInsertBlock()->getParent();
auto tbl = fn->getValueSymbolTable();
@ -726,7 +726,7 @@ struct LLVM {
}
void compile_call(const shared_ptr<AstPL0> ast) {
auto ident = ast->nodes[0]->token;
auto ident = ast->nodes[0]->token_to_string();
auto scope = get_closest_scope(ast);
auto block = scope->get_procedure(ident);
@ -805,7 +805,7 @@ struct LLVM {
auto lhs = compile_expression(ast->nodes[0]);
auto rhs = compile_expression(ast->nodes[2]);
const auto& ope = ast->nodes[1]->token;
const auto& ope = ast->nodes[1]->token_to_string();
switch (ope[0]) {
case '=':
return builder_.CreateICmpEQ(lhs, rhs, "icmpeq");
@ -836,7 +836,7 @@ struct LLVM {
Value* compile_expression(const shared_ptr<AstPL0> ast) {
const auto& nodes = ast->nodes;
auto sign = nodes[0]->token;
auto sign = nodes[0]->token_to_string();
auto negative = !(sign.empty() || sign == "+");
auto val = compile_term(nodes[1]);
@ -845,7 +845,7 @@ struct LLVM {
}
for (auto i = 2u; i < nodes.size(); i += 2) {
auto ope = nodes[i + 0]->token[0];
auto ope = nodes[i + 0]->token_to_string()[0];
auto rval = compile_term(nodes[i + 1]);
switch (ope) {
case '+':
@ -863,7 +863,7 @@ struct LLVM {
const auto& nodes = ast->nodes;
auto val = compile_factor(nodes[0]);
for (auto i = 1u; i < nodes.size(); i += 2) {
auto ope = nodes[i + 0]->token[0];
auto ope = nodes[i + 0]->token_to_string()[0];
auto rval = compile_switch_value(nodes[i + 1]);
switch (ope) {
case '*':
@ -889,7 +889,7 @@ struct LLVM {
}
Value* compile_ident(const shared_ptr<AstPL0> ast) {
auto ident = ast->token;
auto ident = ast->token_to_string();
auto fn = builder_.GetInsertBlock()->getParent();
auto tbl = fn->getValueSymbolTable();
@ -903,7 +903,7 @@ struct LLVM {
Value* compile_number(const shared_ptr<AstPL0> ast) {
return ConstantInt::getIntegerValue(builder_.getInt32Ty(),
APInt(32, ast->token, 10));
APInt(32, ast->token_to_string(), 10));
}
};

13
test/CMakeLists.txt
View File

@ -1,13 +1,14 @@
cmake_minimum_required(VERSION 3.1)
project(test)
enable_language(CXX)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_CXX_STANDARD 11)
set(CMAKE_CXX_EXTENSIONS OFF)
include_directories(..)
set(CMAKE_CXX_STANDARD 17)
add_executable(test-main test-main.cc test1.cc test2.cc test3.cc)
target_link_libraries(test-main ${add_link_deps})
add_test(TestMain test-main)
add_test(
NAME TestMain
COMMAND test-main
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})

387
test/test1.cc
View File

@ -4,20 +4,16 @@
using namespace peg;
#if !defined(PEGLIB_NO_UNICODE_CHARS)
TEST_CASE("Simple syntax test (with unicode)", "[general]")
{
parser parser(
u8" ROOT ← _ "
" _ <- ' ' "
);
TEST_CASE("Simple syntax test (with unicode)", "[general]") {
parser parser(u8" ROOT ← _ "
" _ <- ' ' ");
bool ret = parser;
REQUIRE(ret == true);
}
#endif
TEST_CASE("Simple syntax test", "[general]")
{
TEST_CASE("Simple syntax test", "[general]") {
parser parser(R"(
ROOT <- _
_ <- ' '
@ -27,15 +23,13 @@ TEST_CASE("Simple syntax test", "[general]")
REQUIRE(ret == true);
}
TEST_CASE("Empty syntax test", "[general]")
{
TEST_CASE("Empty syntax test", "[general]") {
parser parser("");
bool ret = parser;
REQUIRE(ret == false);
}
TEST_CASE("Start rule with ignore operator test", "[general]")
{
TEST_CASE("Start rule with ignore operator test", "[general]") {
parser parser(R"(
~ROOT <- _
_ <- ' '
@ -45,19 +39,17 @@ TEST_CASE("Start rule with ignore operator test", "[general]")
REQUIRE(ret == false);
}
TEST_CASE("Invalid UTF-8 text test", "[general]")
{
TEST_CASE("Invalid UTF-8 text test", "[general]") {
std::string s = "a <- '";
s += static_cast<char>(0xe8); // Make invalid utf8 text...
parser parser(s.c_str());
parser parser(s.data());
bool ret = parser;
REQUIRE(ret == false);
}
TEST_CASE("Backslash escape sequence test", "[general]")
{
TEST_CASE("Backslash escape sequence test", "[general]") {
parser parser(R"(
ROOT <- _
_ <- '\\'
@ -67,8 +59,7 @@ TEST_CASE("Backslash escape sequence test", "[general]")
REQUIRE(ret == true);
}
TEST_CASE("Invalid escape sequence test", "[general]")
{
TEST_CASE("Invalid escape sequence test", "[general]") {
parser parser(R"(
ROOT <- _
_ <- '\'
@ -78,22 +69,19 @@ TEST_CASE("Invalid escape sequence test", "[general]")
REQUIRE(ret == false);
}
TEST_CASE("Action taking non const Semantic Values parameter", "[general]")
{
TEST_CASE("Action taking non const Semantic Values parameter", "[general]") {
parser parser(R"(
ROOT <- TEXT
TEXT <- [a-zA-Z]+
)");
parser["ROOT"] = [&](SemanticValues& sv) {
auto s = any_cast<std::string>(sv[0]);
parser["ROOT"] = [&](SemanticValues &vs) {
auto s = std::string(std::any_cast<std::string_view>(vs[0]));
s[0] = 'H'; // mutate
return std::string(std::move(s)); // move
return s; // move
};
parser["TEXT"] = [&](SemanticValues& sv) {
return sv.token();
};
parser["TEXT"] = [&](SemanticValues &vs) { return vs.token(); };
std::string val;
auto ret = parser.parse("hello", val);
@ -101,18 +89,17 @@ TEST_CASE("Action taking non const Semantic Values parameter", "[general]")
REQUIRE(val == "Hello");
}
TEST_CASE("String capture test", "[general]")
{
TEST_CASE("String capture test", "[general]") {
parser parser(R"(
ROOT <- _ ('[' TAG_NAME ']' _)*
TAG_NAME <- (!']' .)+
_ <- [ \t]*
)");
std::vector<std::string> tags;
std::vector<std::string_view> tags;
parser["TAG_NAME"] = [&](const SemanticValues& sv) {
tags.push_back(sv.str());
parser["TAG_NAME"] = [&](const SemanticValues &vs) {
tags.push_back(vs.sv());
};
auto ret = parser.parse(" [tag1] [tag:2] [tag-3] ");
@ -126,14 +113,14 @@ TEST_CASE("String capture test", "[general]")
using namespace peg;
TEST_CASE("String capture test2", "[general]")
{
std::vector<std::string> tags;
TEST_CASE("String capture test2", "[general]") {
std::vector<std::string_view> tags;
Definition ROOT, TAG, TAG_NAME, WS;
ROOT <= seq(WS, zom(TAG));
TAG <= seq(chr('['), TAG_NAME, chr(']'), WS);
TAG_NAME <= oom(seq(npd(chr(']')), dot())), [&](const SemanticValues& sv) { tags.push_back(sv.str()); };
TAG_NAME <= oom(seq(npd(chr(']')), dot())),
[&](const SemanticValues &vs) { tags.push_back(vs.sv()); };
WS <= zom(cls(" \t"));
auto r = ROOT.parse(" [tag1] [tag:2] [tag-3] ");
@ -145,20 +132,16 @@ TEST_CASE("String capture test2", "[general]")
REQUIRE(tags[2] == "tag-3");
}
TEST_CASE("String capture test3", "[general]")
{
TEST_CASE("String capture test3", "[general]") {
parser pg(R"(
ROOT <- _ TOKEN*
TOKEN <- '[' < (!']' .)+ > ']' _
_ <- [ \t\r\n]*
)");
std::vector<std::string_view> tags;
std::vector<std::string> tags;
pg["TOKEN"] = [&](const SemanticValues& sv) {
tags.push_back(sv.token());
};
pg["TOKEN"] = [&](const SemanticValues &vs) { tags.push_back(vs.token()); };
auto ret = pg.parse(" [tag1] [tag:2] [tag-3] ");
@ -169,8 +152,7 @@ TEST_CASE("String capture test3", "[general]")
REQUIRE(tags[2] == "tag-3");
}
TEST_CASE("Cyclic grammer test", "[general]")
{
TEST_CASE("Cyclic grammer test", "[general]") {
Definition PARENT;
Definition CHILD;
@ -178,8 +160,7 @@ TEST_CASE("Cyclic grammer test", "[general]")
CHILD <= seq(PARENT);
}
TEST_CASE("Visit test", "[general]")
{
TEST_CASE("Visit test", "[general]") {
Definition ROOT, TAG, TAG_NAME, WS;
ROOT <= seq(WS, zom(TAG));
@ -193,8 +174,7 @@ TEST_CASE("Visit test", "[general]")
REQUIRE(defIds.ids.size() == 4);
}
TEST_CASE("Token check test", "[general]")
{
TEST_CASE("Token check test", "[general]") {
parser parser(R"(
EXPRESSION <- _ TERM (TERM_OPERATOR TERM)*
TERM <- FACTOR (FACTOR_OPERATOR FACTOR)*
@ -212,25 +192,21 @@ TEST_CASE("Token check test", "[general]")
REQUIRE(parser["_"].is_token() == true);
}
TEST_CASE("Lambda action test", "[general]")
{
TEST_CASE("Lambda action test", "[general]") {
parser parser(R"(
START <- (CHAR)*
CHAR <- .
)");
std::string ss;
parser["CHAR"] = [&](const SemanticValues& sv) {
ss += *sv.c_str();
};
parser["CHAR"] = [&](const SemanticValues &vs) { ss += *vs.sv().data(); };
bool ret = parser.parse("hello");
REQUIRE(ret == true);
REQUIRE(ss == "hello");
}
TEST_CASE("enter/leave handlers test", "[general]")
{
TEST_CASE("enter/leave handlers test", "[general]") {
parser parser(R"(
START <- LTOKEN '=' RTOKEN
LTOKEN <- TOKEN
@ -238,21 +214,22 @@ TEST_CASE("enter/leave handlers test", "[general]")
TOKEN <- [A-Za-z]+
)");
parser["LTOKEN"].enter = [&](const char*, size_t, any& dt) {
auto& require_upper_case = *any_cast<bool*>(dt);
parser["LTOKEN"].enter = [&](const char *, size_t, std::any &dt) {
auto &require_upper_case = *std::any_cast<bool *>(dt);
require_upper_case = false;
};
parser["LTOKEN"].leave = [&](const char*, size_t, size_t, any&, any& dt) {
auto& require_upper_case = *any_cast<bool*>(dt);
parser["LTOKEN"].leave = [&](const char *, size_t, size_t, std::any &,
std::any &dt) {
auto &require_upper_case = *std::any_cast<bool *>(dt);
require_upper_case = true;
};
auto message = "should be upper case string...";
parser["TOKEN"] = [&](const SemanticValues& sv, any& dt) {
auto& require_upper_case = *any_cast<bool*>(dt);
parser["TOKEN"] = [&](const SemanticValues &vs, std::any &dt) {
auto &require_upper_case = *std::any_cast<bool *>(dt);
if (require_upper_case) {
const auto& s = sv.str();
const auto &s = vs.sv();
if (!std::all_of(s.begin(), s.end(), ::isupper)) {
throw parse_error(message);
}
@ -260,13 +237,13 @@ TEST_CASE("enter/leave handlers test", "[general]")
};
bool require_upper_case = false;
any dt = &require_upper_case;
std::any dt = &require_upper_case;
REQUIRE(parser.parse("hello=world", dt) == false);
REQUIRE(parser.parse("HELLO=world", dt) == false);
REQUIRE(parser.parse("hello=WORLD", dt) == true);
REQUIRE(parser.parse("HELLO=WORLD", dt) == true);
parser.log = [&](size_t ln, size_t col, const std::string& msg) {
parser.log = [&](size_t ln, size_t col, const std::string &msg) {
REQUIRE(ln == 1);
REQUIRE(col == 7);
REQUIRE(msg == message);
@ -274,8 +251,7 @@ TEST_CASE("enter/leave handlers test", "[general]")
parser.parse("hello=world", dt);
}
TEST_CASE("WHITESPACE test", "[general]")
{
TEST_CASE("WHITESPACE test", "[general]") {
parser parser(R"(
# Rules
ROOT <- ITEM (',' ITEM)*
@ -293,8 +269,7 @@ TEST_CASE("WHITESPACE test", "[general]")
REQUIRE(ret == true);
}
TEST_CASE("WHITESPACE test2", "[general]")
{
TEST_CASE("WHITESPACE test2", "[general]") {
parser parser(R"(
# Rules
ROOT <- ITEM (',' ITEM)*
@ -305,9 +280,9 @@ TEST_CASE("WHITESPACE test2", "[general]")
TAB <- '\t'
)");
std::vector<std::string> items;
parser["ITEM"] = [&](const SemanticValues& sv) {
items.push_back(sv.token());
std::vector<std::string_view> items;
parser["ITEM"] = [&](const SemanticValues &vs) {
items.push_back(vs.token());
};
auto ret = parser.parse(R"([one], [two] ,[three] )");
@ -328,8 +303,8 @@ TEST_CASE("WHITESPACE test3", "[general]") {
%whitespace <- [ \t]*
)");
parser["StrQuot"] = [](const SemanticValues& sv) {
REQUIRE(sv.token() == R"( aaa \" bbb )");
parser["StrQuot"] = [](const SemanticValues &vs) {
REQUIRE(vs.token() == R"( aaa \" bbb )");
};
auto ret = parser.parse(R"( " aaa \" bbb " )");
@ -345,16 +320,14 @@ TEST_CASE("WHITESPACE test4", "[general]") {
%whitespace <- [ \t\r\n]*
)");
parser["HELLO"] = [](const SemanticValues& sv) {
REQUIRE(sv.token() == "hello");
parser["HELLO"] = [](const SemanticValues &vs) {
REQUIRE(vs.token() == "hello");
};
parser["OPE"] = [](const SemanticValues& sv) {
REQUIRE(sv.token() == "+");
};
parser["OPE"] = [](const SemanticValues &vs) { REQUIRE(vs.token() == "+"); };
parser["WORLD"] = [](const SemanticValues& sv) {
REQUIRE(sv.token() == "world");
parser["WORLD"] = [](const SemanticValues &vs) {
REQUIRE(vs.token() == "world");
};
auto ret = parser.parse(" hello + world ");
@ -375,42 +348,33 @@ TEST_CASE("Word expression test", "[general]") {
REQUIRE(parser.parse("hello , world") == true);
}
TEST_CASE("Skip token test", "[general]")
{
parser parser(
" ROOT <- _ ITEM (',' _ ITEM _)* "
TEST_CASE("Skip token test", "[general]") {
parser parser(" ROOT <- _ ITEM (',' _ ITEM _)* "
" ITEM <- ([a-z0-9])+ "
" ~_ <- [ \t]* "
);
" ~_ <- [ \t]* ");
parser["ROOT"] = [&](const SemanticValues& sv) {
REQUIRE(sv.size() == 2);
};
parser["ROOT"] = [&](const SemanticValues &vs) { REQUIRE(vs.size() == 2); };
auto ret = parser.parse(" item1, item2 ");
REQUIRE(ret == true);
}
TEST_CASE("Skip token test2", "[general]")
{
TEST_CASE("Skip token test2", "[general]") {
parser parser(R"(
ROOT <- ITEM (',' ITEM)*
ITEM <- < ([a-z0-9])+ >
%whitespace <- [ \t]*
)");
parser["ROOT"] = [&](const SemanticValues& sv) {
REQUIRE(sv.size() == 2);
};
parser["ROOT"] = [&](const SemanticValues &vs) { REQUIRE(vs.size() == 2); };
auto ret = parser.parse(" item1, item2 ");
REQUIRE(ret == true);
}
TEST_CASE("Custom AST test", "[general]")
{
TEST_CASE("Custom AST test", "[general]") {
struct CustomType {};
using CustomAst = AstBase<CustomType>;
@ -427,8 +391,7 @@ TEST_CASE("Custom AST test", "[general]")
REQUIRE(ast->nodes.size() == 4);
}
TEST_CASE("Backtracking test", "[general]")
{
TEST_CASE("Backtracking test", "[general]") {
parser parser(R"(
START <- PAT1 / PAT2
PAT1 <- HELLO ' One'
@ -437,9 +400,7 @@ TEST_CASE("Backtracking test", "[general]")
)");
size_t count = 0;
parser["HELLO"] = [&](const SemanticValues& /*sv*/) {
count++;
};
parser["HELLO"] = [&](const SemanticValues & /*vs*/) { count++; };
parser.enable_packrat_parsing();
@ -448,8 +409,7 @@ TEST_CASE("Backtracking test", "[general]")
REQUIRE(count == 1); // Skip second time
}
TEST_CASE("Backtracking with AST", "[general]")
{
TEST_CASE("Backtracking with AST", "[general]") {
parser parser(R"(
S <- A? B (A B)* A
A <- 'a'
@ -463,11 +423,8 @@ TEST_CASE("Backtracking with AST", "[general]")
REQUIRE(ast->nodes.size() == 2);
}
TEST_CASE("Octal/Hex/Unicode value test", "[general]")
{
parser parser(
R"( ROOT <- '\132\x7a\u30f3' )"
);
TEST_CASE("Octal/Hex/Unicode value test", "[general]") {
parser parser(R"( ROOT <- '\132\x7a\u30f3' )");
auto ret = parser.parse("Zzン");
@ -482,32 +439,30 @@ TEST_CASE("Ignore case test", "[general]") {
%whitespace <- [ \t\r\n]*
)");
parser["HELLO"] = [](const SemanticValues& sv) {
REQUIRE(sv.token() == "Hello");
parser["HELLO"] = [](const SemanticValues &vs) {
REQUIRE(vs.token() == "Hello");
};
parser["WORLD"] = [](const SemanticValues& sv) {
REQUIRE(sv.token() == "World");
parser["WORLD"] = [](const SemanticValues &vs) {
REQUIRE(vs.token() == "World");
};
auto ret = parser.parse(" Hello World ");
REQUIRE(ret == true);
}
TEST_CASE("mutable lambda test", "[general]")
{
std::vector<std::string> vec;
TEST_CASE("mutable lambda test", "[general]") {
std::vector<std::string_view> vec;
parser pg("ROOT <- 'mutable lambda test'");
// This test makes sure if the following code can be compiled.
pg["TOKEN"] = [=](const SemanticValues& sv) mutable {
vec.push_back(sv.str());
pg["TOKEN"] = [=](const SemanticValues &vs) mutable {
vec.push_back(vs.sv());
};
}
TEST_CASE("Simple calculator test", "[general]")
{
TEST_CASE("Simple calculator test", "[general]") {
parser parser(R"(
Additive <- Multitive '+' Additive / Multitive
Multitive <- Primary '*' Multitive / Primary
@ -515,27 +470,21 @@ TEST_CASE("Simple calculator test", "[general]")
Number <- [0-9]+
)");
parser["Additive"] = [](const SemanticValues& sv) {
switch (sv.choice()) {
case 0:
return any_cast<int>(sv[0]) + any_cast<int>(sv[1]);
default:
return any_cast<int>(sv[0]);
parser["Additive"] = [](const SemanticValues &vs) {
switch (vs.choice()) {
case 0: return std::any_cast<int>(vs[0]) + std::any_cast<int>(vs[1]);
default: return std::any_cast<int>(vs[0]);
}
};
parser["Multitive"] = [](const SemanticValues& sv) {
switch (sv.choice()) {
case 0:
return any_cast<int>(sv[0]) * any_cast<int>(sv[1]);
default:
return any_cast<int>(sv[0]);
parser["Multitive"] = [](const SemanticValues &vs) {
switch (vs.choice()) {
case 0: return std::any_cast<int>(vs[0]) * std::any_cast<int>(vs[1]);
default: return std::any_cast<int>(vs[0]);
}
};
parser["Number"] = [](const SemanticValues& sv) {
return atoi(sv.c_str());
};
parser["Number"] = [](const SemanticValues &vs) { return vs.token_to_number<int>(); };
int val;
parser.parse("(1+2)*3", val);
@ -543,8 +492,7 @@ TEST_CASE("Simple calculator test", "[general]")
REQUIRE(val == 9);
}
TEST_CASE("Calculator test", "[general]")
{
TEST_CASE("Calculator test", "[general]") {
// Construct grammer
Definition EXPRESSION, TERM, FACTOR, TERM_OPERATOR, FACTOR_OPERATOR, NUMBER;
@ -556,11 +504,11 @@ TEST_CASE("Calculator test", "[general]")
NUMBER <= oom(cls("0-9"));
// Setup actions
auto reduce = [](const SemanticValues& sv) -> long {
long ret = any_cast<long>(sv[0]);
for (auto i = 1u; i < sv.size(); i += 2) {
auto num = any_cast<long>(sv[i + 1]);
switch (any_cast<char>(sv[i])) {
auto reduce = [](const SemanticValues &vs) -> long {
long ret = std::any_cast<long>(vs[0]);
for (auto i = 1u; i < vs.size(); i += 2) {
auto num = std::any_cast<long>(vs[i + 1]);
switch (std::any_cast<char>(vs[i])) {
case '+': ret += num; break;
case '-': ret -= num; break;
case '*': ret *= num; break;
@ -572,9 +520,9 @@ TEST_CASE("Calculator test", "[general]")
EXPRESSION = reduce;
TERM = reduce;
TERM_OPERATOR = [](const SemanticValues& sv) { return *sv.c_str(); };
FACTOR_OPERATOR = [](const SemanticValues& sv) { return *sv.c_str(); };
NUMBER = [](const SemanticValues& sv) { return stol(sv.str(), nullptr, 10); };
TERM_OPERATOR = [](const SemanticValues &vs) { return *vs.sv().data(); };
FACTOR_OPERATOR = [](const SemanticValues &vs) { return *vs.sv().data(); };
NUMBER = [](const SemanticValues &vs) { return vs.token_to_number<long>(); };
// Parse
long val;
@ -584,8 +532,7 @@ TEST_CASE("Calculator test", "[general]")
REQUIRE(val == -3);
}
TEST_CASE("Calculator test2", "[general]")
{
TEST_CASE("Calculator test2", "[general]") {
// Parse syntax
auto syntax = R"(
# Grammar for Calculator...
@ -599,14 +546,14 @@ TEST_CASE("Calculator test2", "[general]")
std::string start;
auto grammar = ParserGenerator::parse(syntax, strlen(syntax), start, nullptr);
auto& g = *grammar;
auto &g = *grammar;
// Setup actions
auto reduce = [](const SemanticValues& sv) -> long {
long ret = any_cast<long>(sv[0]);
for (auto i = 1u; i < sv.size(); i += 2) {
auto num = any_cast<long>(sv[i + 1]);
switch (any_cast<char>(sv[i])) {
auto reduce = [](const SemanticValues &vs) -> long {
long ret = std::any_cast<long>(vs[0]);
for (auto i = 1u; i < vs.size(); i += 2) {
auto num = std::any_cast<long>(vs[i + 1]);
switch (std::any_cast<char>(vs[i])) {
case '+': ret += num; break;
case '-': ret -= num; break;
case '*': ret *= num; break;
@ -618,9 +565,9 @@ TEST_CASE("Calculator test2", "[general]")
g["EXPRESSION"] = reduce;
g["TERM"] = reduce;
g["TERM_OPERATOR"] = [](const SemanticValues& sv) { return *sv.c_str(); };
g["FACTOR_OPERATOR"] = [](const SemanticValues& sv) { return *sv.c_str(); };
g["NUMBER"] = [](const SemanticValues& sv) { return stol(sv.str(), nullptr, 10); };
g["TERM_OPERATOR"] = [](const SemanticValues &vs) { return *vs.sv().data(); };
g["FACTOR_OPERATOR"] = [](const SemanticValues &vs) { return *vs.sv().data(); };
g["NUMBER"] = [](const SemanticValues &vs) { return vs.token_to_number<long>(); };
// Parse
long val;
@ -630,8 +577,7 @@ TEST_CASE("Calculator test2", "[general]")
REQUIRE(val == -3);
}
TEST_CASE("Calculator test3", "[general]")
{
TEST_CASE("Calculator test3", "[general]") {
// Parse syntax
parser parser(R"(
# Grammar for Calculator...
@ -643,11 +589,11 @@ TEST_CASE("Calculator test3", "[general]")
NUMBER <- [0-9]+
)");
auto reduce = [](const SemanticValues& sv) -> long {
long ret = any_cast<long>(sv[0]);
for (auto i = 1u; i < sv.size(); i += 2) {
auto num = any_cast<long>(sv[i + 1]);
switch (any_cast<char>(sv[i])) {
auto reduce = [](const SemanticValues &vs) -> long {
long ret = std::any_cast<long>(vs[0]);
for (auto i = 1u; i < vs.size(); i += 2) {
auto num = std::any_cast<long>(vs[i + 1]);
switch (std::any_cast<char>(vs[i])) {
case '+': ret += num; break;
case '-': ret -= num; break;
case '*': ret *= num; break;
@ -660,9 +606,13 @@ TEST_CASE("Calculator test3", "[general]")
// Setup actions
parser["EXPRESSION"] = reduce;
parser["TERM"] = reduce;
parser["TERM_OPERATOR"] = [](const SemanticValues& sv) { return static_cast<char>(*sv.c_str()); };
parser["FACTOR_OPERATOR"] = [](const SemanticValues& sv) { return static_cast<char>(*sv.c_str()); };
parser["NUMBER"] = [](const SemanticValues& sv) { return stol(sv.str(), nullptr, 10); };
parser["TERM_OPERATOR"] = [](const SemanticValues &vs) {
return static_cast<char>(*vs.sv().data());
};
parser["FACTOR_OPERATOR"] = [](const SemanticValues &vs) {
return static_cast<char>(*vs.sv().data());
};
parser["NUMBER"] = [](const SemanticValues &vs) { return vs.token_to_number<long>(); };
// Parse
long val;
@ -672,8 +622,7 @@ TEST_CASE("Calculator test3", "[general]")
REQUIRE(val == -3);
}
TEST_CASE("Calculator test with AST", "[general]")
{
TEST_CASE("Calculator test with AST", "[general]") {
parser parser(R"(
EXPRESSION <- _ TERM (TERM_OPERATOR TERM)*
TERM <- FACTOR (FACTOR_OPERATOR FACTOR)*
@ -686,11 +635,11 @@ TEST_CASE("Calculator test with AST", "[general]")
parser.enable_ast();
std::function<long (const Ast&)> eval = [&](const Ast& ast) {
std::function<long(const Ast &)> eval = [&](const Ast &ast) {
if (ast.name == "NUMBER") {
return stol(ast.token);
return ast.token_to_number<long>();
} else {
const auto& nodes = ast.nodes;
const auto &nodes = ast.nodes;
auto result = eval(*nodes[0]);
for (auto i = 1u; i < nodes.size(); i += 2) {
auto num = eval(*nodes[i + 1]);
@ -717,7 +666,8 @@ TEST_CASE("Calculator test with AST", "[general]")
TEST_CASE("Calculator test with combinators and AST", "[general]") {
// Construct grammer
AST_DEFINITIONS(EXPRESSION, TERM, FACTOR, TERM_OPERATOR, FACTOR_OPERATOR, NUMBER);
AST_DEFINITIONS(EXPRESSION, TERM, FACTOR, TERM_OPERATOR, FACTOR_OPERATOR,
NUMBER);
EXPRESSION <= seq(TERM, zom(seq(TERM_OPERATOR, TERM)));
TERM <= seq(FACTOR, zom(seq(FACTOR_OPERATOR, FACTOR)));
@ -728,7 +678,7 @@ TEST_CASE("Calculator test with combinators and AST", "[general]") {
std::function<long(const Ast &)> eval = [&](const Ast &ast) {
if (ast.name == "NUMBER") {
return stol(ast.token);
return ast.token_to_number<long>();
} else {
const auto &nodes = ast.nodes;
auto result = eval(*nodes[0]);
@ -755,8 +705,7 @@ TEST_CASE("Calculator test with combinators and AST", "[general]") {
REQUIRE(val == -3);
}
TEST_CASE("Ignore semantic value test", "[general]")
{
TEST_CASE("Ignore semantic value test", "[general]") {
parser parser(R"(
START <- ~HELLO WORLD
HELLO <- 'Hello' _
@ -774,8 +723,7 @@ TEST_CASE("Ignore semantic value test", "[general]")
REQUIRE(ast->nodes[0]->name == "WORLD");
}
TEST_CASE("Ignore semantic value of 'or' predicate test", "[general]")
{
TEST_CASE("Ignore semantic value of 'or' predicate test", "[general]") {
parser parser(R"(
START <- _ !DUMMY HELLO_WORLD '.'
HELLO_WORLD <- HELLO 'World' _
@ -794,8 +742,7 @@ TEST_CASE("Ignore semantic value of 'or' predicate test", "[general]")
REQUIRE(ast->nodes[0]->name == "HELLO_WORLD");
}
TEST_CASE("Ignore semantic value of 'and' predicate test", "[general]")
{
TEST_CASE("Ignore semantic value of 'and' predicate test", "[general]") {
parser parser(R"(
START <- _ &HELLO HELLO_WORLD '.'
HELLO_WORLD <- HELLO 'World' _
@ -813,8 +760,7 @@ TEST_CASE("Ignore semantic value of 'and' predicate test", "[general]")
REQUIRE(ast->nodes[0]->name == "HELLO_WORLD");
}
TEST_CASE("Literal token on AST test1", "[general]")
{
TEST_CASE("Literal token on AST test1", "[general]") {
parser parser(R"(
STRING_LITERAL <- '"' (('\\"' / '\\t' / '\\n') / (!["] .))* '"'
)");
@ -829,8 +775,7 @@ TEST_CASE("Literal token on AST test1", "[general]")
REQUIRE(ast->nodes.empty());
}
TEST_CASE("Literal token on AST test2", "[general]")
{
TEST_CASE("Literal token on AST test2", "[general]") {
parser parser(R"(
STRING_LITERAL <- '"' (ESC / CHAR)* '"'
ESC <- ('\\"' / '\\t' / '\\n')
@ -847,8 +792,7 @@ TEST_CASE("Literal token on AST test2", "[general]")
REQUIRE(ast->nodes.size() == 3);
}
TEST_CASE("Literal token on AST test3", "[general]")
{
TEST_CASE("Literal token on AST test3", "[general]") {
parser parser(R"(
STRING_LITERAL <- < '"' (ESC / CHAR)* '"' >
ESC <- ('\\"' / '\\t' / '\\n')
@ -865,8 +809,7 @@ TEST_CASE("Literal token on AST test3", "[general]")
REQUIRE(ast->nodes.empty());
}
TEST_CASE("Missing missing definitions test", "[general]")
{
TEST_CASE("Missing missing definitions test", "[general]") {
parser parser(R"(
A <- B C
)");
@ -874,8 +817,7 @@ TEST_CASE("Missing missing definitions test", "[general]")
REQUIRE(!parser);
}
TEST_CASE("Definition duplicates test", "[general]")
{
TEST_CASE("Definition duplicates test", "[general]") {
parser parser(R"(
A <- ''
A <- ''
@ -884,8 +826,7 @@ TEST_CASE("Definition duplicates test", "[general]")
REQUIRE(!parser);
}
TEST_CASE("Semantic values test", "[general]")
{
TEST_CASE("Semantic values test", "[general]") {
parser parser(R"(
term <- ( a b c x )? a b c
a <- 'a'
@ -894,15 +835,15 @@ TEST_CASE("Semantic values test", "[general]")
x <- 'x'
)");
for (const auto& rule: parser.get_rule_names()){
parser[rule.c_str()] = [rule](const SemanticValues& sv, any&) {
for (const auto &rule : parser.get_rule_names()) {
parser[rule.data()] = [rule](const SemanticValues &vs, std::any &) {
if (rule == "term") {
REQUIRE(any_cast<std::string>(sv[0]) == "a at 0");
REQUIRE(any_cast<std::string>(sv[1]) == "b at 1");
REQUIRE(any_cast<std::string>(sv[2]) == "c at 2");
REQUIRE(std::any_cast<std::string>(vs[0]) == "a at 0");
REQUIRE(std::any_cast<std::string>(vs[1]) == "b at 1");
REQUIRE(std::any_cast<std::string>(vs[2]) == "c at 2");
return std::string();
} else {
return rule + " at " + std::to_string(sv.c_str() - sv.ss);
return rule + " at " + std::to_string(vs.sv().data() - vs.ss);
}
};
}
@ -910,36 +851,33 @@ TEST_CASE("Semantic values test", "[general]")
REQUIRE(parser.parse("abc"));
}
TEST_CASE("Ordered choice count", "[general]")
{
TEST_CASE("Ordered choice count", "[general]") {
parser parser(R"(
S <- 'a' / 'b'
)");
parser["S"] = [](const SemanticValues& sv) {
REQUIRE(sv.choice() == 1);
REQUIRE(sv.choice_count() == 2);
parser["S"] = [](const SemanticValues &vs) {
REQUIRE(vs.choice() == 1);
REQUIRE(vs.choice_count() == 2);
};
parser.parse("b");
}
TEST_CASE("Ordered choice count 2", "[general]")
{
TEST_CASE("Ordered choice count 2", "[general]") {
parser parser(R"(
S <- ('a' / 'b')*
)");
parser["S"] = [](const SemanticValues& sv) {
REQUIRE(sv.choice() == 0);
REQUIRE(sv.choice_count() == 0);
parser["S"] = [](const SemanticValues &vs) {
REQUIRE(vs.choice() == 0);
REQUIRE(vs.choice_count() == 0);
};
parser.parse("b");
}
TEST_CASE("Semantic value tag", "[general]")
{
TEST_CASE("Semantic value tag", "[general]") {
parser parser(R"(
S <- A? B* C?
A <- 'a'
@ -949,10 +887,10 @@ TEST_CASE("Semantic value tag", "[general]")
{
using namespace udl;
parser["S"] = [](const SemanticValues& sv) {
REQUIRE(sv.size() == 1);
REQUIRE(sv.tags.size() == 1);
REQUIRE(sv.tags[0] == "C"_);
parser["S"] = [](const SemanticValues &vs) {
REQUIRE(vs.size() == 1);
REQUIRE(vs.tags.size() == 1);
REQUIRE(vs.tags[0] == "C"_);
};
auto ret = parser.parse("c");
REQUIRE(ret == true);
@ -960,11 +898,11 @@ TEST_CASE("Semantic value tag", "[general]")
{
using namespace udl;
parser["S"] = [](const SemanticValues& sv) {
REQUIRE(sv.size() == 2);
REQUIRE(sv.tags.size() == 2);
REQUIRE(sv.tags[0] == "B"_);
REQUIRE(sv.tags[1] == "B"_);
parser["S"] = [](const SemanticValues &vs) {
REQUIRE(vs.size() == 2);
REQUIRE(vs.tags.size() == 2);
REQUIRE(vs.tags[0] == "B"_);
REQUIRE(vs.tags[1] == "B"_);
};
auto ret = parser.parse("bb");
REQUIRE(ret == true);
@ -972,19 +910,18 @@ TEST_CASE("Semantic value tag", "[general]")
{
using namespace udl;
parser["S"] = [](const SemanticValues& sv) {
REQUIRE(sv.size() == 2);
REQUIRE(sv.tags.size() == 2);
REQUIRE(sv.tags[0] == "A"_);
REQUIRE(sv.tags[1] == "C"_);
parser["S"] = [](const SemanticValues &vs) {
REQUIRE(vs.size() == 2);
REQUIRE(vs.tags.size() == 2);
REQUIRE(vs.tags[0] == "A"_);
REQUIRE(vs.tags[1] == "C"_);
};
auto ret = parser.parse("ac");
REQUIRE(ret == true);
}
}
TEST_CASE("Negated Class test", "[general]")
{
TEST_CASE("Negated Class test", "[general]") {
parser parser(R"(
ROOT <- [^a-z_]+
)");

281
test/test2.cc
View File

@ -101,8 +101,7 @@ TEST_CASE("Infinite loop 1", "[infinite loop]")
REQUIRE(!pg);
}
TEST_CASE("Infinite loop 2", "[infinite loop]")
{
TEST_CASE("Infinite loop 2", "[infinite loop]") {
parser pg(R"(
ROOT <- WH TOKEN+ WH
TOKEN <- [a-z0-9]*
@ -112,8 +111,7 @@ TEST_CASE("Infinite loop 2", "[infinite loop]")
REQUIRE(!pg);
}
TEST_CASE("Infinite loop 3", "[infinite loop]")
{
TEST_CASE("Infinite loop 3", "[infinite loop]") {
parser pg(R"(
ROOT <- WH TOKEN* WH
TOKEN <- !'word1'
@ -123,8 +121,7 @@ TEST_CASE("Infinite loop 3", "[infinite loop]")
REQUIRE(!pg);
}
TEST_CASE("Infinite loop 4", "[infinite loop]")
{
TEST_CASE("Infinite loop 4", "[infinite loop]") {
parser pg(R"(
ROOT <- WH TOKEN* WH
TOKEN <- &'word1'
@ -134,8 +131,7 @@ TEST_CASE("Infinite loop 4", "[infinite loop]")
REQUIRE(!pg);
}
TEST_CASE("Infinite loop 5", "[infinite loop]")
{
TEST_CASE("Infinite loop 5", "[infinite loop]") {
parser pg(R"(
Numbers <- Number*
Number <- [0-9]+ / Spacing
@ -146,8 +142,7 @@ TEST_CASE("Infinite loop 5", "[infinite loop]")
REQUIRE(!pg);
}
TEST_CASE("Infinite loop 6", "[infinite loop]")
{
TEST_CASE("Infinite loop 6", "[infinite loop]") {
parser pg(R"(
S <- ''*
)");
@ -155,8 +150,7 @@ TEST_CASE("Infinite loop 6", "[infinite loop]")
REQUIRE(!pg);
}
TEST_CASE("Infinite loop 7", "[infinite loop]")
{
TEST_CASE("Infinite loop 7", "[infinite loop]") {
parser pg(R"(
S <- A*
A <- ''
@ -165,8 +159,7 @@ TEST_CASE("Infinite loop 7", "[infinite loop]")
REQUIRE(!pg);
}
TEST_CASE("Not infinite 1", "[infinite loop]")
{
TEST_CASE("Not infinite 1", "[infinite loop]") {
parser pg(R"(
Numbers <- Number* EOF
Number <- [0-9]+ / Spacing
@ -177,8 +170,7 @@ TEST_CASE("Not infinite 1", "[infinite loop]")
REQUIRE(!!pg); // OK
}
TEST_CASE("Not infinite 2", "[infinite loop]")
{
TEST_CASE("Not infinite 2", "[infinite loop]") {
parser pg(R"(
ROOT <- _ ('[' TAG_NAME ']' _)*
# In a sequence operator, if there is at least one non-empty element, we can treat it as non-empty
@ -189,8 +181,7 @@ TEST_CASE("Not infinite 2", "[infinite loop]")
REQUIRE(!!pg); // OK
}
TEST_CASE("Not infinite 3", "[infinite loop]")
{
TEST_CASE("Not infinite 3", "[infinite loop]") {
parser pg(R"(
EXPRESSION <- _ TERM (TERM_OPERATOR TERM)*
TERM <- FACTOR (FACTOR_OPERATOR FACTOR)*
@ -204,8 +195,7 @@ TEST_CASE("Not infinite 3", "[infinite loop]")
REQUIRE(!!pg); // OK
}
TEST_CASE("Precedence climbing", "[precedence]")
{
TEST_CASE("Precedence climbing", "[precedence]") {
parser parser(R"(
START <- _ EXPRESSION
EXPRESSION <- ATOM (OPERATOR ATOM)* {
@ -223,11 +213,11 @@ TEST_CASE("Precedence climbing", "[precedence]")
parser.enable_packrat_parsing();
// Setup actions
parser["EXPRESSION"] = [](const SemanticValues& sv) -> long {
auto result = any_cast<long>(sv[0]);
if (sv.size() > 1) {
auto ope = any_cast<char>(sv[1]);
auto num = any_cast<long>(sv[2]);
parser["EXPRESSION"] = [](const SemanticValues &vs) -> long {
auto result = std::any_cast<long>(vs[0]);
if (vs.size() > 1) {
auto ope = std::any_cast<char>(vs[1]);
auto num = std::any_cast<long>(vs[2]);
switch (ope) {
case '+': result += num; break;
case '-': result -= num; break;
@ -237,8 +227,8 @@ TEST_CASE("Precedence climbing", "[precedence]")
}
return result;
};
parser["OPERATOR"] = [](const SemanticValues& sv) { return *sv.c_str(); };
parser["NUMBER"] = [](const SemanticValues& sv) { return atol(sv.c_str()); };
parser["OPERATOR"] = [](const SemanticValues &vs) { return *vs.sv().data(); };
parser["NUMBER"] = [](const SemanticValues &vs) { return vs.token_to_number<long>(); };
bool ret = parser;
REQUIRE(ret == true);
@ -262,8 +252,7 @@ TEST_CASE("Precedence climbing", "[precedence]")
}
}
TEST_CASE("Precedence climbing with macro", "[precedence]")
{
TEST_CASE("Precedence climbing with macro", "[precedence]") {
// Create a PEG parser
parser parser(R"(
EXPRESSION <- INFIX_EXPRESSION(ATOM, OPERATOR)
@ -284,11 +273,11 @@ TEST_CASE("Precedence climbing with macro", "[precedence]")
REQUIRE(ret == true);
// Setup actions
parser["INFIX_EXPRESSION"] = [](const SemanticValues& sv) -> long {
auto result = any_cast<long>(sv[0]);
if (sv.size() > 1) {
auto ope = any_cast<char>(sv[1]);
auto num = any_cast<long>(sv[2]);
parser["INFIX_EXPRESSION"] = [](const SemanticValues &vs) -> long {
auto result = std::any_cast<long>(vs[0]);
if (vs.size() > 1) {
auto ope = std::any_cast<char>(vs[1]);
auto num = std::any_cast<long>(vs[2]);
switch (ope) {
case '+': result += num; break;
case '-': result -= num; break;
@ -298,8 +287,8 @@ TEST_CASE("Precedence climbing with macro", "[precedence]")
}
return result;
};
parser["OPERATOR"] = [](const SemanticValues& sv) { return *sv.c_str(); };
parser["NUMBER"] = [](const SemanticValues& sv) { return atol(sv.c_str()); };
parser["OPERATOR"] = [](const SemanticValues &vs) { return *vs.sv().data(); };
parser["NUMBER"] = [](const SemanticValues &vs) { return vs.token_to_number<long>(); };
{
auto expr = " 1 + 2 * 3 * (4 - 5 + 6) / 7 - 8 ";
@ -320,8 +309,7 @@ TEST_CASE("Precedence climbing with macro", "[precedence]")
}
}
TEST_CASE("Precedence climbing error1", "[precedence]")
{
TEST_CASE("Precedence climbing error1", "[precedence]") {
parser parser(R"(
START <- _ EXPRESSION
EXPRESSION <- ATOM (OPERATOR ATOM1)* {
@ -341,8 +329,7 @@ TEST_CASE("Precedence climbing error1", "[precedence]")
REQUIRE(ret == false);
}
TEST_CASE("Precedence climbing error2", "[precedence]")
{
TEST_CASE("Precedence climbing error2", "[precedence]") {
parser parser(R"(
START <- _ EXPRESSION
EXPRESSION <- ATOM OPERATOR ATOM {
@ -392,8 +379,7 @@ TEST_CASE("Packrat parser test with %whitespace%", "[packrat]") {
REQUIRE(ret == true);
}
TEST_CASE("Packrat parser test with macro", "[packrat]")
{
TEST_CASE("Packrat parser test with macro", "[packrat]") {
parser parser(R"(
EXPRESSION <- _ LIST(TERM, TERM_OPERATOR)
TERM <- LIST(FACTOR, FACTOR_OPERATOR)
@ -412,7 +398,8 @@ TEST_CASE("Packrat parser test with macro", "[packrat]")
REQUIRE(ret == true);
}
TEST_CASE("Packrat parser test with precedence expression parser", "[packrat]") {
TEST_CASE("Packrat parser test with precedence expression parser",
"[packrat]") {
peg::parser parser(R"(
Expression <- Atom (Operator Atom)* { precedence L + - L * / }
Atom <- _? Number _?
@ -430,8 +417,7 @@ TEST_CASE("Packrat parser test with precedence expression parser", "[packrat]")
REQUIRE(ret == true);
}
TEST_CASE("Backreference test", "[backreference]")
{
TEST_CASE("Backreference test", "[backreference]") {
parser parser(R"(
START <- _ LQUOTE < (!RQUOTE .)* > RQUOTE _
LQUOTE <- 'R"' $delm< [a-zA-Z]* > '('
@ -440,9 +426,7 @@ TEST_CASE("Backreference test", "[backreference]")
)");
std::string token;
parser["START"] = [&](const SemanticValues& sv) {
token = sv.token();
};
parser["START"] = [&](const SemanticValues &vs) { token = vs.token(); };
{
token.clear();
@ -485,8 +469,7 @@ TEST_CASE("Backreference test", "[backreference]")
}
}
TEST_CASE("Invalid backreference test", "[backreference]")
{
TEST_CASE("Invalid backreference test", "[backreference]") {
parser parser(R"(
START <- _ LQUOTE (!RQUOTE .)* RQUOTE _
LQUOTE <- 'R"' $delm< [a-zA-Z]* > '('
@ -494,16 +477,13 @@ TEST_CASE("Invalid backreference test", "[backreference]")
~_ <- [ \t\r\n]*
)");
REQUIRE_THROWS_AS(
parser.parse(R"delm(
REQUIRE_THROWS_AS(parser.parse(R"delm(
R"foo("(hello world)")foo"
)delm"),
std::runtime_error);
}
TEST_CASE("Nested capture test", "[backreference]")
{
TEST_CASE("Nested capture test", "[backreference]") {
parser parser(R"(
ROOT <- CONTENT
CONTENT <- (ELEMENT / TEXT)*
@ -521,8 +501,7 @@ TEST_CASE("Nested capture test", "[backreference]")
REQUIRE(!parser.parse("This is a <u>test</u> text</b>."));
}
TEST_CASE("Backreference with Prioritized Choice test", "[backreference]")
{
TEST_CASE("Backreference with Prioritized Choice test", "[backreference]") {
parser parser(R"(
TREE <- WRONG_BRANCH / CORRECT_BRANCH
WRONG_BRANCH <- BRANCH THAT IS_capture WRONG
@ -538,8 +517,7 @@ TEST_CASE("Backreference with Prioritized Choice test", "[backreference]")
REQUIRE_THROWS_AS(parser.parse("branchthatiscorrect"), std::runtime_error);
}
TEST_CASE("Backreference with Zero or More test", "[backreference]")
{
TEST_CASE("Backreference with Zero or More test", "[backreference]") {
parser parser(R"(
TREE <- WRONG_BRANCH* CORRECT_BRANCH
WRONG_BRANCH <- BRANCH THAT IS_capture WRONG
@ -554,14 +532,16 @@ TEST_CASE("Backreference with Zero or More test", "[backreference]")
REQUIRE(parser.parse("branchthatiswrongbranchthatiscorrect"));
REQUIRE(!parser.parse("branchthatiswrongbranchthatIscorrect"));
REQUIRE(!parser.parse("branchthatiswrongbranchthatIswrongbranchthatiscorrect"));
REQUIRE(parser.parse("branchthatiswrongbranchthatIswrongbranchthatIscorrect"));
REQUIRE(
!parser.parse("branchthatiswrongbranchthatIswrongbranchthatiscorrect"));
REQUIRE(
parser.parse("branchthatiswrongbranchthatIswrongbranchthatIscorrect"));
REQUIRE_THROWS_AS(parser.parse("branchthatiscorrect"), std::runtime_error);
REQUIRE_THROWS_AS(parser.parse("branchthatiswron_branchthatiscorrect"), std::runtime_error);
REQUIRE_THROWS_AS(parser.parse("branchthatiswron_branchthatiscorrect"),
std::runtime_error);
}
TEST_CASE("Backreference with One or More test", "[backreference]")
{
TEST_CASE("Backreference with One or More test", "[backreference]") {
parser parser(R"(
TREE <- WRONG_BRANCH+ CORRECT_BRANCH
WRONG_BRANCH <- BRANCH THAT IS_capture WRONG
@ -576,14 +556,15 @@ TEST_CASE("Backreference with One or More test", "[backreference]")
REQUIRE(parser.parse("branchthatiswrongbranchthatiscorrect"));
REQUIRE(!parser.parse("branchthatiswrongbranchthatIscorrect"));
REQUIRE(!parser.parse("branchthatiswrongbranchthatIswrongbranchthatiscorrect"));
REQUIRE(parser.parse("branchthatiswrongbranchthatIswrongbranchthatIscorrect"));
REQUIRE(
!parser.parse("branchthatiswrongbranchthatIswrongbranchthatiscorrect"));
REQUIRE(
parser.parse("branchthatiswrongbranchthatIswrongbranchthatIscorrect"));
REQUIRE(!parser.parse("branchthatiscorrect"));
REQUIRE(!parser.parse("branchthatiswron_branchthatiscorrect"));
}
TEST_CASE("Backreference with Option test", "[backreference]")
{
TEST_CASE("Backreference with Option test", "[backreference]") {
parser parser(R"(
TREE <- WRONG_BRANCH? CORRECT_BRANCH
WRONG_BRANCH <- BRANCH THAT IS_capture WRONG
@ -598,14 +579,16 @@ TEST_CASE("Backreference with Option test", "[backreference]")
REQUIRE(parser.parse("branchthatiswrongbranchthatiscorrect"));
REQUIRE(!parser.parse("branchthatiswrongbranchthatIscorrect"));
REQUIRE(!parser.parse("branchthatiswrongbranchthatIswrongbranchthatiscorrect"));
REQUIRE(!parser.parse("branchthatiswrongbranchthatIswrongbranchthatIscorrect"));
REQUIRE(
!parser.parse("branchthatiswrongbranchthatIswrongbranchthatiscorrect"));
REQUIRE(
!parser.parse("branchthatiswrongbranchthatIswrongbranchthatIscorrect"));
REQUIRE_THROWS_AS(parser.parse("branchthatiscorrect"), std::runtime_error);
REQUIRE_THROWS_AS(parser.parse("branchthatiswron_branchthatiscorrect"), std::runtime_error);
REQUIRE_THROWS_AS(parser.parse("branchthatiswron_branchthatiscorrect"),
std::runtime_error);
}
TEST_CASE("Repetition {0}", "[repetition]")
{
TEST_CASE("Repetition {0}", "[repetition]") {
parser parser(R"(
START <- '(' DIGIT{3} ') ' DIGIT{3} '-' DIGIT{4}
DIGIT <- [0-9]
@ -616,8 +599,7 @@ TEST_CASE("Repetition {0}", "[repetition]")
REQUIRE(!parser.parse("(123) 45-7a90"));
}
TEST_CASE("Repetition {2,4}", "[repetition]")
{
TEST_CASE("Repetition {2,4}", "[repetition]") {
parser parser(R"(
START <- DIGIT{2,4}
DIGIT <- [0-9]
@ -629,8 +611,7 @@ TEST_CASE("Repetition {2,4}", "[repetition]")
REQUIRE(!parser.parse("12345"));
}
TEST_CASE("Repetition {2,1}", "[repetition]")
{
TEST_CASE("Repetition {2,1}", "[repetition]") {
parser parser(R"(
START <- DIGIT{2,1} # invalid range
DIGIT <- [0-9]
@ -640,8 +621,7 @@ TEST_CASE("Repetition {2,1}", "[repetition]")
REQUIRE(!parser.parse("123"));
}
TEST_CASE("Repetition {2,}", "[repetition]")
{
TEST_CASE("Repetition {2,}", "[repetition]") {
parser parser(R"(
START <- DIGIT{2,}
DIGIT <- [0-9]
@ -652,8 +632,7 @@ TEST_CASE("Repetition {2,}", "[repetition]")
REQUIRE(parser.parse("1234"));
}
TEST_CASE("Repetition {,2}", "[repetition]")
{
TEST_CASE("Repetition {,2}", "[repetition]") {
parser parser(R"(
START <- DIGIT{,2}
DIGIT <- [0-9]
@ -664,8 +643,7 @@ TEST_CASE("Repetition {,2}", "[repetition]")
REQUIRE(!parser.parse("1234"));
}
TEST_CASE("Left recursive test", "[left recursive]")
{
TEST_CASE("Left recursive test", "[left recursive]") {
parser parser(R"(
A <- A 'a'
B <- A 'a'
@ -674,8 +652,7 @@ TEST_CASE("Left recursive test", "[left recursive]")
REQUIRE(!parser);
}
TEST_CASE("Left recursive with option test", "[left recursive]")
{
TEST_CASE("Left recursive with option test", "[left recursive]") {
parser parser(R"(
A <- 'a' / 'b'? B 'c'
B <- A
@ -684,8 +661,7 @@ TEST_CASE("Left recursive with option test", "[left recursive]")
REQUIRE(!parser);
}
TEST_CASE("Left recursive with zom test", "[left recursive]")
{
TEST_CASE("Left recursive with zom test", "[left recursive]") {
parser parser(R"(
A <- 'a'* A*
)");
@ -693,8 +669,7 @@ TEST_CASE("Left recursive with zom test", "[left recursive]")
REQUIRE(!parser);
}
TEST_CASE("Left recursive with a ZOM content rule", "[left recursive]")
{
TEST_CASE("Left recursive with a ZOM content rule", "[left recursive]") {
parser parser(R"(
A <- B
B <- _ A
@ -704,49 +679,38 @@ TEST_CASE("Left recursive with a ZOM content rule", "[left recursive]")
REQUIRE(!parser);
}
TEST_CASE("Left recursive with empty string test", "[left recursive]")
{
parser parser(
" A <- '' A"
);
TEST_CASE("Left recursive with empty string test", "[left recursive]") {
parser parser(" A <- '' A");
REQUIRE(!parser);
}
TEST_CASE("User defined rule test", "[user rule]")
{
TEST_CASE("User defined rule test", "[user rule]") {
auto g = parser(R"(
ROOT <- _ 'Hello' _ NAME '!' _
)",
{
{
"NAME", usr([](const char* s, size_t n, SemanticValues& /*sv*/, any& /*dt*/) -> size_t {
static std::vector<std::string> names = { "PEG", "BNF" };
for (const auto& name: names) {
if (name.size() <= n && !name.compare(0, name.size(), s, name.size())) {
{{"NAME", usr([](const char *s, size_t n, SemanticValues &,
std::any &) -> size_t {
static std::vector<std::string> names = {"PEG", "BNF"};
for (const auto &name : names) {
if (name.size() <= n &&
!name.compare(0, name.size(), s, name.size())) {
return name.size();
}
}
return static_cast<size_t>(-1);
})
},
{
"~_", zom(cls(" \t\r\n"))
}
});
})},
{"~_", zom(cls(" \t\r\n"))}});
REQUIRE(g.parse(" Hello BNF! ") == true);
}
TEST_CASE("Semantic predicate test", "[predicate]")
{
TEST_CASE("Semantic predicate test", "[predicate]") {
parser parser("NUMBER <- [0-9]+");
parser["NUMBER"] = [](const SemanticValues& sv) {
auto val = stol(sv.token(), nullptr, 10);
if (val != 100) {
throw parse_error("value error!!");
}
parser["NUMBER"] = [](const SemanticValues &vs) {
auto val = vs.token_to_number<long>();
if (val != 100) { throw parse_error("value error!!"); }
return val;
};
@ -754,7 +718,7 @@ TEST_CASE("Semantic predicate test", "[predicate]")
REQUIRE(parser.parse("100", val));
REQUIRE(val == 100);
parser.log = [](size_t line, size_t col, const std::string& msg) {
parser.log = [](size_t line, size_t col, const std::string &msg) {
REQUIRE(line == 1);
REQUIRE(col == 1);
REQUIRE(msg == "value error!!");
@ -762,8 +726,7 @@ TEST_CASE("Semantic predicate test", "[predicate]")
REQUIRE(!parser.parse("200", val));
}
TEST_CASE("Japanese character", "[unicode]")
{
TEST_CASE("Japanese character", "[unicode]") {
peg::parser parser(u8R"(
文 <- 修飾語? 主語 述語 '。'
主語 <- 名詞 助詞
@ -781,20 +744,17 @@ TEST_CASE("Japanese character", "[unicode]")
REQUIRE(parser.parse(u8R"(サーバーを復旧します。)"));
}
TEST_CASE("dot with a code", "[unicode]")
{
TEST_CASE("dot with a code", "[unicode]") {
peg::parser parser(" S <- 'a' . 'b' ");
REQUIRE(parser.parse(u8R"(aあb)"));
}
TEST_CASE("dot with a char", "[unicode]")
{
TEST_CASE("dot with a char", "[unicode]") {
peg::parser parser(" S <- 'a' . 'b' ");
REQUIRE(parser.parse(u8R"(aåb)"));
}
TEST_CASE("character class", "[unicode]")
{
TEST_CASE("character class", "[unicode]") {
peg::parser parser(R"(
S <- 'a' [い-おAさC-Eた-とは] 'b'
)");
@ -828,8 +788,7 @@ TEST_CASE("dot with a grapheme", "[unicode]")
}
#endif
TEST_CASE("Macro simple test", "[macro]")
{
TEST_CASE("Macro simple test", "[macro]") {
parser parser(R"(
S <- HELLO WORLD
HELLO <- T('hello')
@ -840,8 +799,7 @@ TEST_CASE("Macro simple test", "[macro]")
REQUIRE(parser.parse("hello \tworld "));
}
TEST_CASE("Macro two parameters", "[macro]")
{
TEST_CASE("Macro two parameters", "[macro]") {
parser parser(R"(
S <- HELLO_WORLD
HELLO_WORLD <- T('hello', 'world')
@ -851,8 +809,7 @@ TEST_CASE("Macro two parameters", "[macro]")
REQUIRE(parser.parse("hello \tworld "));
}
TEST_CASE("Macro syntax error", "[macro]")
{
TEST_CASE("Macro syntax error", "[macro]") {
parser parser(R"(
S <- T('hello')
T (a) <- a [ \t]*
@ -862,8 +819,7 @@ TEST_CASE("Macro syntax error", "[macro]")
REQUIRE(ret == false);
}
TEST_CASE("Macro missing argument", "[macro]")
{
TEST_CASE("Macro missing argument", "[macro]") {
parser parser(R"(
S <- T ('hello')
T(a, b) <- a [ \t]* b
@ -873,8 +829,7 @@ TEST_CASE("Macro missing argument", "[macro]")
REQUIRE(ret == false);
}
TEST_CASE("Macro reference syntax error", "[macro]")
{
TEST_CASE("Macro reference syntax error", "[macro]") {
parser parser(R"(
S <- T ('hello')
T(a) <- a [ \t]*
@ -884,8 +839,7 @@ TEST_CASE("Macro reference syntax error", "[macro]")
REQUIRE(ret == false);
}
TEST_CASE("Macro invalid macro reference error", "[macro]")
{
TEST_CASE("Macro invalid macro reference error", "[macro]") {
parser parser(R"(
S <- T('hello')
T <- 'world'
@ -895,8 +849,7 @@ TEST_CASE("Macro invalid macro reference error", "[macro]")
REQUIRE(ret == false);
}
TEST_CASE("Macro calculator", "[macro]")
{
TEST_CASE("Macro calculator", "[macro]") {
// Create a PEG parser
parser parser(R"(
# Grammar for simple calculator...
@ -912,11 +865,11 @@ TEST_CASE("Macro calculator", "[macro]")
)");
// Setup actions
auto reduce = [](const SemanticValues& sv) -> long {
auto result = any_cast<long>(sv[0]);
for (auto i = 1u; i < sv.size(); i += 2) {
auto num = any_cast<long>(sv[i + 1]);
auto ope = any_cast<char>(sv[i]);
auto reduce = [](const SemanticValues &vs) {
auto result = std::any_cast<long>(vs[0]);
for (auto i = 1u; i < vs.size(); i += 2) {
auto num = std::any_cast<long>(vs[i + 1]);
auto ope = std::any_cast<char>(vs[i]);
switch (ope) {
case '+': result += num; break;
case '-': result -= num; break;
@ -929,9 +882,13 @@ TEST_CASE("Macro calculator", "[macro]")
parser["EXPRESSION"] = reduce;
parser["TERM"] = reduce;
parser["TERM_OPERATOR"] = [](const SemanticValues& sv) { return static_cast<char>(*sv.c_str()); };
parser["FACTOR_OPERATOR"] = [](const SemanticValues& sv) { return static_cast<char>(*sv.c_str()); };
parser["NUMBER"] = [](const SemanticValues& sv) { return atol(sv.c_str()); };
parser["TERM_OPERATOR"] = [](const SemanticValues &vs) {
return static_cast<char>(*vs.sv().data());
};
parser["FACTOR_OPERATOR"] = [](const SemanticValues &vs) {
return static_cast<char>(*vs.sv().data());
};
parser["NUMBER"] = [](const SemanticValues &vs) { return vs.token_to_number<long>(); };
bool ret = parser;
REQUIRE(ret == true);
@ -944,8 +901,7 @@ TEST_CASE("Macro calculator", "[macro]")
REQUIRE(val == -3);
}
TEST_CASE("Macro expression arguments", "[macro]")
{
TEST_CASE("Macro expression arguments", "[macro]") {
parser parser(R"(
S <- M('hello' / 'Hello', 'world' / 'World')
M(arg0, arg1) <- arg0 [ \t]+ arg1
@ -954,8 +910,7 @@ TEST_CASE("Macro expression arguments", "[macro]")
REQUIRE(parser.parse("Hello world"));
}
TEST_CASE("Macro recursive", "[macro]")
{
TEST_CASE("Macro recursive", "[macro]") {
parser parser(R"(
S <- M('abc')
M(s) <- !s / s ' ' M(s / '123') / s
@ -967,9 +922,8 @@ TEST_CASE("Macro recursive", "[macro]")
REQUIRE(parser.parse("abc 123 abc"));
}
TEST_CASE("Macro recursive2", "[macro]")
{
auto syntaxes = std::vector<const char*>{
TEST_CASE("Macro recursive2", "[macro]") {
auto syntaxes = std::vector<const char *>{
"S <- M('abc') M(s) <- !s / s ' ' M(s* '-' '123') / s",
"S <- M('abc') M(s) <- !s / s ' ' M(s+ '-' '123') / s",
"S <- M('abc') M(s) <- !s / s ' ' M(s? '-' '123') / s",
@ -979,14 +933,13 @@ TEST_CASE("Macro recursive2", "[macro]")
"S <- M('abc') M(s) <- !s / s ' ' M(~s '-' '123') / s",
};
for (const auto& syntax: syntaxes) {
for (const auto &syntax : syntaxes) {
parser parser(syntax);
REQUIRE(parser.parse("abc abc-123"));
}
}
TEST_CASE("Macro exclusive modifiers", "[macro]")
{
TEST_CASE("Macro exclusive modifiers", "[macro]") {
parser parser(R"(
S <- Modifiers(!"") _
Modifiers(Appeared) <- (!Appeared) (
@ -1006,8 +959,7 @@ TEST_CASE("Macro exclusive modifiers", "[macro]")
REQUIRE(!parser.parse("public static public"));
}
TEST_CASE("Macro token check test", "[macro]")
{
TEST_CASE("Macro token check test", "[macro]") {
parser parser(R"(
# Grammar for simple calculator...
EXPRESSION <- _ LIST(TERM, TERM_OPERATOR)
@ -1053,8 +1005,7 @@ TEST_CASE("Nested macro call", "[macro]") {
REQUIRE(parser.parse("val"));
}
TEST_CASE("Nested macro call2", "[macro]")
{
TEST_CASE("Nested macro call2", "[macro]") {
parser parser(R"(
START <- A('TestVal1', 'TestVal2')+
A(Aarg1, Aarg2) <- B(Aarg1) '#End'
@ -1072,8 +1023,8 @@ TEST_CASE("Line information test", "[line information]") {
)");
std::vector<std::pair<size_t, size_t>> locations;
parser["WORD"] = [&](const peg::SemanticValues& sv) {
locations.push_back(sv.line_info());
parser["WORD"] = [&](const peg::SemanticValues &vs) {
locations.push_back(vs.line_info());
};
bool ret = parser;
@ -1091,8 +1042,7 @@ TEST_CASE("Line information test", "[line information]") {
REQUIRE(locations[6] == std::make_pair<size_t, size_t>(3, 1));
}
TEST_CASE("Dictionary", "[dic]")
{
TEST_CASE("Dictionary", "[dic]") {
parser parser(R"(
START <- 'This month is ' MONTH '.'
MONTH <- 'Jan' | 'January' | 'Feb' | 'February'
@ -1104,8 +1054,7 @@ TEST_CASE("Dictionary", "[dic]")
REQUIRE_FALSE(parser.parse("This month is ."));
}
TEST_CASE("Dictionary invalid", "[dic]")
{
TEST_CASE("Dictionary invalid", "[dic]") {
parser parser(R"(
START <- 'This month is ' MONTH '.'
MONTH <- 'Jan' | 'January' | [a-z]+ | 'Feb' | 'February'