make the lexer's requirement for null-terminated buffers more explicit (

sorbet#5092)

parser/Parser.cc

@@ -53,7 +53,13 @@ unique_ptr<Node> Parser::run(sorbet::core::GlobalState &gs, core::FileRef file,
                              std::vector<std::string> initialLocals) {
     Builder builder(gs, file);
     auto source = file.data(gs).source();
-    ruby_parser::typedruby27 driver(string(source.begin(), source.end()), Builder::interface);
+    // The lexer requires that its buffers end with a null terminator, which core::File
+    // does not guarantee.  Parsing heredocs for some mysterious reason requires two.
+    string buffer;
+    buffer.reserve(source.size() + 2);
+    buffer += source;
+    buffer += "\0\0"sv;
+    ruby_parser::typedruby27 driver(buffer, Builder::interface);
 
     for (string local : initialLocals) {
         driver.lex.declare(local);

third_party/parser/cc/driver.cc

@@ -6,10 +6,10 @@
 
 namespace ruby_parser {
 
-base_driver::base_driver(ruby_version version, const std::string &source, const struct builder &builder)
+base_driver::base_driver(ruby_version version, std::string_view source, const struct builder &builder)
     : build(builder), lex(diagnostics, version, source), pending_error(false), def_level(0), ast(nullptr) {}
 
-typedruby27::typedruby27(const std::string &source, const struct builder &builder)
+typedruby27::typedruby27(std::string_view source, const struct builder &builder)
     : base_driver(ruby_version::RUBY_27, source, builder) {}
 
 ForeignPtr typedruby27::parse(SelfPtr self, bool trace) {

third_party/parser/cc/lexer.rl

@@ -120,10 +120,10 @@ using namespace std::string_literals;
 
 %% prepush { check_stack_capacity(); }
 
-lexer::lexer(diagnostics_t &diag, ruby_version version, const std::string& source_buffer_)
+lexer::lexer(diagnostics_t &diag, ruby_version version, std::string_view source_buffer)
   : diagnostics(diag)
   , version(version)
-  , source_buffer(source_buffer_ + std::string("\0\0", 2))
+  , source_buffer(source_buffer)
   , cs(lex_en_line_begin)
   , _p(source_buffer.data())
   , _pe(source_buffer.data() + source_buffer.size())
@@ -144,6 +144,9 @@ lexer::lexer(diagnostics_t &diag, ruby_version version, const std::string& sourc
   , herebody_s(nullptr)
   , in_kwarg(false)
 {
+  assert(!source_buffer.empty());
+  assert(source_buffer.back() == '\0');
+
   // ensure the stack is non-empty so we can just double in
   // check_stack_capacity:
   stack.resize(16);

third_party/parser/include/ruby_parser/driver.hh

@@ -276,7 +276,7 @@ public:
     ForeignPtr ast;
     token_t last_token;
 
-    base_driver(ruby_version version, const std::string &source, const struct builder &builder);
+    base_driver(ruby_version version, std::string_view source, const struct builder &builder);
     virtual ~base_driver() {}
     virtual ForeignPtr parse(SelfPtr self, bool trace) = 0;
 
@@ -303,7 +303,7 @@ public:
 
 class typedruby27 : public base_driver {
 public:
-    typedruby27(const std::string &source, const struct builder &builder);
+    typedruby27(std::string_view source, const struct builder &builder);
     virtual ForeignPtr parse(SelfPtr self, bool trace);
     ~typedruby27() {}
 };

third_party/parser/include/ruby_parser/lexer.hh

@@ -52,7 +52,7 @@ private:
     pool<token, 64> mempool;
 
     ruby_version version;
-    const std::string source_buffer;
+    std::string_view source_buffer;
 
     std::stack<environment> static_env;
     std::stack<literal> literal_stack;
@@ -169,7 +169,7 @@ public:
     bool in_kwarg; // true at the end of "def foo a:"
     Context context;
 
-    lexer(diagnostics_t &diag, ruby_version version, const std::string &source_buffer_);
+    lexer(diagnostics_t &diag, ruby_version version, std::string_view source_buffer);
 
     token_t advance();