From 5a57f5ec16988a177655b8e9f5d533871c38d9ae Mon Sep 17 00:00:00 2001
From: Herb Sutter <herb.sutter@gmail.com>
Date: Sun, 18 Jun 2023 11:57:20 -0700
Subject: [PATCH] Replace `token` count,len with `string_view`, closes #517

---
 regression-tests/test-results/version |  2 +-
 source/build.info                     |  2 +-
 source/cppfront.cpp                   |  3 +--
 source/lex.h                          | 37 ++++++++++++---------------
 4 files changed, 19 insertions(+), 25 deletions(-)

diff --git a/regression-tests/test-results/version b/regression-tests/test-results/version
index fc27ff01fa..c2b005ca21 100644
--- a/regression-tests/test-results/version
+++ b/regression-tests/test-results/version
@@ -1,5 +1,5 @@
 
-cppfront compiler v0.2.1   Build 8618:1005
+cppfront compiler v0.2.1   Build 8618:1139
 Copyright(c) Herb Sutter   All rights reserved
 
 SPDX-License-Identifier: CC-BY-NC-ND-4.0
diff --git a/source/build.info b/source/build.info
index 25e1e99115..11c8f52d3b 100644
--- a/source/build.info
+++ b/source/build.info
@@ -1 +1 @@
-"8618:1005"
\ No newline at end of file
+"8618:1139"
\ No newline at end of file
diff --git a/source/cppfront.cpp b/source/cppfront.cpp
index ed30fcba3d..5b89964d3a 100644
--- a/source/cppfront.cpp
+++ b/source/cppfront.cpp
@@ -6028,8 +6028,7 @@ auto main(
             exit_status = EXIT_FAILURE;
         }
 
-        //  In any case, emit the debug information (during early development this is
-        //  on by default, later we'll flip the switch to turn it on instead of off)
+        //  And, if requested, the debug information
         if (enable_debug_output_files) {
             c.debug_print();
         }
diff --git a/source/lex.h b/source/lex.h
index feff94570b..805bf7e5bc 100644
--- a/source/lex.h
+++ b/source/lex.h
@@ -226,10 +226,9 @@ class token
         source_position pos,
         lexeme          type
     )
-      : start   {start}
-      , count   {int16_t(count)}
-      , pos     {pos  }
-      , lex_type{type }
+      : sv      {start, unsafe_narrow<ulong>(count)}
+      , pos     {pos}
+      , lex_type{type}
     {
     }
 
@@ -238,18 +237,17 @@ class token
         source_position pos,
         lexeme          type
     )
-      : start   {sz}
-      , count   {__as<int16_t>(std::strlen(sz))}
-      , pos     {pos  }
-      , lex_type{type }
+      : sv      {sz}
+      , pos     {pos}
+      , lex_type{type}
     {
     }
 
     auto as_string_view() const
         -> std::string_view
     {
-        assert (start);
-        return {start, static_cast<unsigned>(count)};
+        assert (sv.data());
+        return sv;
     }
 
     operator std::string_view() const
@@ -272,7 +270,7 @@ class token
     auto to_string( bool text_only = false ) const
         -> std::string
     {
-        auto text = std::string{start, static_cast<unsigned>(count)};
+        auto text = std::string{sv};
         if (text_only) {
             return text;
         }
@@ -294,13 +292,13 @@ class token
         pos.colno += offset;
     }
 
-    auto position() const -> source_position { return pos;      }
+    auto position() const -> source_position { return pos;       }
 
-    auto length  () const -> int             { return count;    }
+    auto length  () const -> int             { return sv.size(); }
 
-    auto type    () const -> lexeme          { return lex_type; }
+    auto type    () const -> lexeme          { return lex_type;  }
 
-    auto set_type(lexeme l) -> void          { lex_type = l;    }
+    auto set_type(lexeme l) -> void          { lex_type = l;     }
 
     auto visit(auto& v, int depth) const
         -> void
@@ -309,12 +307,9 @@ class token
     }
 
 private:
-    //  Store (char*,count) because it's smaller than a string_view
-    //
-    char const*     start;
-    int16_t         count;
-    source_position pos;
-    lexeme          lex_type;
+    std::string_view sv;
+    source_position  pos;
+    lexeme           lex_type;
 };
 
 static_assert (CHAR_BIT == 8);