From eea8ccc79dd8070a8570a1c6c976d77feab97f3a Mon Sep 17 00:00:00 2001 From: Mark Skelton Date: Fri, 6 Jan 2023 22:28:56 -0600 Subject: [PATCH] External scanner --- bindings/rust/build.rs | 2 - src/scanner.cc | 153 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 153 insertions(+), 2 deletions(-) create mode 100644 src/scanner.cc diff --git a/bindings/rust/build.rs b/bindings/rust/build.rs index c6061f0..618e90a 100644 --- a/bindings/rust/build.rs +++ b/bindings/rust/build.rs @@ -25,7 +25,6 @@ fn main() { // If your language uses an external scanner written in C++, // then include this block of code: - /* let mut cpp_config = cc::Build::new(); cpp_config.cpp(true); cpp_config.include(&src_dir); @@ -36,5 +35,4 @@ fn main() { cpp_config.file(&scanner_path); cpp_config.compile("scanner"); println!("cargo:rerun-if-changed={}", scanner_path.to_str().unwrap()); - */ } diff --git a/src/scanner.cc b/src/scanner.cc new file mode 100644 index 0000000..1c10a3f --- /dev/null +++ b/src/scanner.cc @@ -0,0 +1,153 @@ +#include +#include +#include +#include +#include + +namespace { + +using std::iswspace; +using std::wstring; + +enum TokenType { + RAW_STRING_DELIMITER, + RAW_STRING_CONTENT, +}; + +// The spec limits delimiters to 16 chars, enforce this to bound serialization. +const unsigned RAW_STRING_DELIMITER_MAX = 16; + +struct Scanner { + // Last raw_string_delimiter, used to detect when raw_string_content ends. + wstring delimiter; + + // Scan a raw string delimiter in R"delimiter(content)delimiter". + bool scan_raw_string_delimiter(TSLexer *lexer) { + if (!delimiter.empty()) { + // Closing delimiter: must exactly match the opening delimiter. + // We already checked this when scanning content, but this is how we know + // when to stop. We can't stop at ", because R"""hello""" is valid. + for (std::size_t i = 0; i < delimiter.size(); ++i) { + if (lexer->lookahead != delimiter[i]) + return false; + lexer->advance(lexer, false); + } + delimiter.clear(); + return true; + } + // Opening delimiter: record the d-char-sequence up to (. + // d-char is any basic character except parens, backslashes, and spaces. + for (;;) { + if (delimiter.size() > RAW_STRING_DELIMITER_MAX || + lexer->lookahead == 0 || lexer->lookahead == '\\' || + iswspace(lexer->lookahead)) { + return false; + } + if (lexer->lookahead == '(') { + // Rather than create a token for an empty delimiter, we fail and let + // the grammar fall back to a delimiter-less rule. + return !delimiter.empty(); + } + delimiter += lexer->lookahead; + lexer->advance(lexer, false); + } + } + + // Scan the raw string content in R"delimiter(content)delimiter". + bool scan_raw_string_content(TSLexer *lexer) { + // The progress made through the delimiter since the last ')'. + // The delimiter may not contain ')' so a single counter suffices. + int delimiter_index = -1; + for (;;) { + // If we hit EOF, consider the content to terminate there. + // This forms an incomplete raw_string_literal, and models the code well. + if (lexer->lookahead == 0) { + lexer->mark_end(lexer); + return true; + } + + if (delimiter_index >= 0) { + if (static_cast(delimiter_index) == delimiter.size()) { + if (lexer->lookahead == '"') { + return true; + } else { + delimiter_index = -1; + } + } else { + if (lexer->lookahead == delimiter[delimiter_index]) { + delimiter_index++; + } else { + delimiter_index = -1; + } + } + } + + if (delimiter_index == -1 && lexer->lookahead == ')') { + // The content doesn't include the )delimiter" part. + // We must still scan through it, but exclude it from the token. + lexer->mark_end(lexer); + delimiter_index = 0; + } + + lexer->advance(lexer, false); + } + } + + bool scan(TSLexer *lexer, const bool *valid_symbols) { + // No skipping leading whitespace: raw-string grammar is space-sensitive. + + if (valid_symbols[RAW_STRING_CONTENT]) { + lexer->result_symbol = RAW_STRING_CONTENT; + return scan_raw_string_content(lexer); + } + + if (valid_symbols[RAW_STRING_DELIMITER]) { + lexer->result_symbol = RAW_STRING_DELIMITER; + return scan_raw_string_delimiter(lexer); + } + + return false; + } +}; + +} // namespace + +extern "C" { + +void *tree_sitter_mql5_external_scanner_create() { return new Scanner(); } + +bool tree_sitter_mql5_external_scanner_scan(void *payload, TSLexer *lexer, + const bool *valid_symbols) { + Scanner *scanner = static_cast(payload); + return scanner->scan(lexer, valid_symbols); +} + +unsigned tree_sitter_mql5_external_scanner_serialize(void *payload, + char *buffer) { +#if __cpp_static_assert >= 200410L + static_assert(RAW_STRING_DELIMITER_MAX * sizeof(wchar_t) < + TREE_SITTER_SERIALIZATION_BUFFER_SIZE, + "Raw string delimiters may not be serializable!"); +#endif + + Scanner *scanner = static_cast(payload); + size_t size = scanner->delimiter.size() * sizeof(wchar_t); + memcpy(buffer, scanner->delimiter.data(), size); + return size; +} + +void tree_sitter_mql5_external_scanner_deserialize(void *payload, + const char *buffer, + unsigned length) { + assert(length % sizeof(wchar_t) == 0 && "Can't decode serialized delimiter!"); + + Scanner *scanner = static_cast(payload); + scanner->delimiter.resize(length / sizeof(wchar_t)); + memcpy(&scanner->delimiter[0], buffer, length); +} + +void tree_sitter_mql5_external_scanner_destroy(void *payload) { + Scanner *scanner = static_cast(payload); + delete scanner; +} +}