From ce977020bb909056d79b368321ee737b01e22ed2 Mon Sep 17 00:00:00 2001
From: Keith Pinson <keith.pinson@banno.com>
Date: Tue, 16 Mar 2021 09:25:35 -0400
Subject: [PATCH 01/42] Read the code and fiddle toward Scala 3 a bit

---
 scala-mode-syntax.el | 11 ++++++++---
 scala-mode.el        |  8 +++++---
 2 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/scala-mode-syntax.el b/scala-mode-syntax.el
index edab1ab..185d31e 100644
--- a/scala-mode-syntax.el
+++ b/scala-mode-syntax.el
@@ -364,12 +364,14 @@
 (defconst scala-syntax:final-re
   (concat "\\(^\\|[^`'_]\\)\\(" scala-syntax:final-unsafe-re "\\)"))
 
+;; TODO open
 (defconst scala-syntax:sealed-unsafe-re
   (regexp-opt '("sealed") 'words))
 
 (defconst scala-syntax:sealed-re
   (concat "\\(^\\|[^`'_]\\)\\(" scala-syntax:sealed-unsafe-re "\\)"))
 
+;; TODO using/given
 (defconst scala-syntax:implicit-unsafe-re
   (regexp-opt '("implicit") 'words))
 
@@ -395,7 +397,7 @@
   (concat "\\(^\\|[^`'_]\\)\\(" scala-syntax:protected-unsafe-re "\\)"))
 
 (defconst scala-syntax:modifiers-unsafe-re
-  (regexp-opt '("override" "abstract" "final" "sealed" "implicit" "lazy"
+  (regexp-opt '("override" "abstract" "final" "sealed" "implicit" "lazy" "open"
                 "private" "protected") 'words))
 
 (defconst scala-syntax:modifiers-re
@@ -705,6 +707,7 @@ stableId"
     (and (looking-at scala-syntax:case-re)
          (goto-char (match-end 0))
          (scala-syntax:skip-forward-ignorable)
+         ;; TODO "not in an enum environment"
          (not (looking-at-p scala-syntax:class-or-object-re)))))
 
 (defun scala-syntax:looking-back-empty-line-p ()
@@ -784,6 +787,7 @@ one."
       (when (scala-syntax:looking-at "[[]")
         (forward-list)))))
 
+;; TODO if-then-else syntax
 (defun scala-syntax:looking-back-else-if-p ()
   ;; TODO: rewrite using (scala-syntax:if-skipped (scala:syntax:skip-backward-else-if))
   (save-excursion
@@ -961,14 +965,14 @@ not. A list must be either enclosed in parentheses or start with
 ;; Functions to help with finding the beginning and end of scala definitions.
 
 (defconst scala-syntax:modifiers-re
-  (regexp-opt '("override" "abstract" "final" "sealed" "implicit" "lazy"
+  (regexp-opt '("override" "abstract" "final" "sealed" "implicit" "lazy" "using" "extension"
                 "private" "protected" "case") 'words))
 
 (defconst scala-syntax:whitespace-delimeted-modifiers-re
   (concat "\\(?:" scala-syntax:modifiers-re "\\(?: *\\)" "\\)*"))
 
 (defconst scala-syntax:definition-words-re
-  (mapconcat 'regexp-quote '("class" "object" "trait" "val" "var" "def" "type") "\\|"))
+  (mapconcat 'regexp-quote '("class" "object" "trait" "val" "var" "def" "type" "enum" "given") "\\|"))
 
 (defun scala-syntax:build-definition-re (words-re)
   (concat " *"
@@ -1018,6 +1022,7 @@ val a, b = (1, 2)
   (scala-syntax:find-brace-equals-or-next)
   (scala-syntax:handle-brace-equals-or-next))
 
+;; TODO or : or... (indentation syntax)
 (defun scala-syntax:find-brace-equals-or-next ()
   (scala-syntax:go-to-pos
    (save-excursion
diff --git a/scala-mode.el b/scala-mode.el
index fe1bf58..0d961e9 100644
--- a/scala-mode.el
+++ b/scala-mode.el
@@ -88,15 +88,17 @@ If there is no plausible default, return nil."
 
 ;;;###autoload
 (defun scala-mode:goto-start-of-code ()
-  "Go to the start of the real code in the file: object, class or trait."
+  "Go to the start of the real code in the file.
+
+Object, class, trait, enum, def, val, or given."
   (interactive)
   (let* ((case-fold-search nil))
-    (search-forward-regexp "\\([[:space:]]+\\|^\\)\\(class\\|object\\|trait\\)" nil t)
+    (search-forward-regexp "\\([[:space:]]+\\|^\\)\\(class\\|enum\\|object\\|trait\\|def\\|val\\|given\\)" nil t)
     (move-beginning-of-line nil)))
 
 ;;;###autoload
 (define-derived-mode scala-mode prog-mode "Scala"
-  "Major mode for editing scala code.
+  "Major mode for editing Scala code.
 
 When started, runs `scala-mode-hook'.
 

From eef2bffd8f8d6c47c7df77c91f43f96b3e8bd243 Mon Sep 17 00:00:00 2001
From: Keith Pinson <keith.pinson@banno.com>
Date: Fri, 2 Apr 2021 10:07:10 -0400
Subject: [PATCH 02/42] Begin to match `scala-mode-syntax` against the 3.0 BNF

---
 scala-mode-syntax.el | 102 +++++++++++++++++++++++++++++--------------
 1 file changed, 70 insertions(+), 32 deletions(-)

diff --git a/scala-mode-syntax.el b/scala-mode-syntax.el
index 185d31e..71f414d 100644
--- a/scala-mode-syntax.el
+++ b/scala-mode-syntax.el
@@ -1,44 +1,91 @@
-;;;; scala-mode-syntax.el - Major mode for editing scala, syntax
-;;; Copyright (c) 2012 Heikki Vesalainen
+;;;; scala-mode-syntax.el - Major mode for editing Scala, syntax
+;;; Copyright (c) 2021 Heikki Vesalainen
 ;;; For information on the License, see the LICENSE file
 
-;;; Based on Scala Language Specification (SLS) Version 2.9
+;;; Based on Scala Language Specification (SLS) Version 3.0
+;;; https://dotty.epfl.ch/docs/internals/syntax.html
 
 ;;;;
 ;;;; Scala syntax regular expressions
 ;;;;
 
-;;; Based on the Scala language specification 2.9.  Note: order is not
+;;; Based on the Scala language specification 3.0.  Note: order is not
 ;;; the same as in the document, as here things are declared before
 ;;; used.
 
-;;; A note on naming. Things that end with '-re' are regular
-;;; expressions.  Things that end with '-group' are regular expression
+;;; A note on naming. Things that end with `-re' are regular
+;;; expressions.  Things that end with `-group' are regular expression
 ;;; character groups without the enclosing [], i.e. they are not
 ;;; regular expressions, but can be used in declaring one.
 
 ;; single letter matching groups (Chapter 1)
 (defconst scala-syntax:hexDigit-group "0-9A-Fa-f")
-(defconst scala-syntax:UnicodeEscape-re (concat "\\\\u[" scala-syntax:hexDigit-group "]\\{4\\}"))
-
+(defconst scala-syntax:UnicodeEscape-re
+  ;; using `format' allows editing these regexes with one step closer to a sane
+  ;; number of backslash escapes, via `string-edit', at the expense of making %
+  ;; a special character
+  (format "\\\\u[%s]\\{4\\}" scala-syntax:hexDigit-group))
+
+;; TODO BNF for `upper' adds the coments "and Unicode category Lu"; do Emacs
+;; regexes handle this naturally?
 (defconst scala-syntax:upper-group "[:upper:]\\$") ;; missing _ to make ids work
-(defconst scala-syntax:upperAndUnderscore-group (concat "_" scala-syntax:upper-group ))
+;; NOTE `upperAndUnderscore' corresponds to the `upper' group in the BNF
+(defconst scala-syntax:upperAndUnderscore-group
+  (concat "_" scala-syntax:upper-group ))
+;; TODO BNF for `lower' adds the coments "and Unicode category Ll"; do Emacs
+;; regexes handle this naturally?
 (defconst scala-syntax:lower-group "[:lower:]")
-(defconst scala-syntax:letter-group (concat scala-syntax:lower-group scala-syntax:upper-group)) ;; TODO: add Lt, Lo, Nl
+;; TODO BNF for `lower' adds the coments "and Unicode categories Lo, Lt, Nl"
+(defconst scala-syntax:letter-group (concat scala-syntax:lower-group
+                                            scala-syntax:upper-group))
 (defconst scala-syntax:digit-group "0-9")
-(defconst scala-syntax:letterOrDigit-group (concat
-                                            scala-syntax:upperAndUnderscore-group
-                                            scala-syntax:lower-group
-                                            scala-syntax:digit-group))
-(defconst scala-syntax:opchar-safe-group "!%&*+/?\\\\^|~-") ;; TODO: Sm, So
+;; NOTE `letterOrDigit' does not have a separate entry in the 3.0 BNF.
+(defconst scala-syntax:letterOrDigit-group
+  (concat
+   scala-syntax:upperAndUnderscore-group
+   scala-syntax:lower-group
+   scala-syntax:digit-group))
+;; TODO ensure Unicode Sm, So disallowed in `opchar'
+;; TODO do the math: check these positively stated symbols against the
+;; negatively stated BNF.
+(defconst scala-syntax:opchar-safe-group "!%&*+/?\\\\^|~-")
 (defconst scala-syntax:opchar-unsafe-group "#:<=>@")
 (defconst scala-syntax:opchar-group (concat scala-syntax:opchar-unsafe-group
                                             scala-syntax:opchar-safe-group))
 
-;; Scala delimiters (Chapter 1), but no quotes
+;; NOTE `delim' in the BNF
+;; TODO should backtick be here? I'm not sure it is handled correctly ATM.
+;; Scala delimiters, but no quotes
 (defconst scala-syntax:delimiter-group ".,;")
 
-;; Integer Literal (Chapter 1.3.1)
+;; NOTE BNF also has a definition here for `printabeChar'
+;; NOTE BNF also has a definition here for `charEscapeSeq'
+
+(defconst scala-syntax:op-re
+  (concat "[" scala-syntax:opchar-group "]+" ))
+
+(defconst scala-syntax:idrest-re
+  ;; Eagerness of regexp causes problems with _. The following is a workaround,
+  ;; but the resulting regexp matches only what SLS demands.
+  (format "\\([_]??[%s%s]+\\)*\\(_+%s\\|_\\)?"
+          scala-syntax:letter-group
+          scala-syntax:digit-group
+          scala-syntax:op-re))
+
+(defconst scala-syntax:varid-re
+  (concat "[" scala-syntax:lower-group "]" scala-syntax:idrest-re))
+
+;; `alphaid' introduced by SIP-11 - String Interpolation
+;; https://docs.scala-lang.org/sips/string-interpolation.html
+(defconst scala-syntax:alphaid-re
+  (format "\\([%s%s]%s\\)"
+          scala-syntax:lower-group
+          scala-syntax:upperAndUnderscore-group
+          scala-syntax:idrest-re))
+
+;; TODO resume here (`plainid' in the BNF)
+
+;; Integer Literal
 (defconst scala-syntax:nonZeroDigit-group "1-9")
 (defconst scala-syntax:octalDigit-group "0-7")
 (defconst scala-syntax:decimalNumeral-re
@@ -116,19 +163,10 @@
           "\\(?:" scala-syntax:multiLineStringLiteral-end-re "\\)?"
           "\\|\\(\"\\)" "\\(\\\\.\\|[^\"\n\\]\\)*" "\\(\"\\)"))
 
-;; Identifiers (Chapter 1.1)
-(defconst scala-syntax:op-re
-  (concat "[" scala-syntax:opchar-group "]+" ))
-(defconst scala-syntax:idrest-re
-  ;; Eagerness of regexp causes problems with _. The following is a workaround,
-  ;; but the resulting regexp matches only what SLS demands.
-  (concat "\\(" "[_]??" "[" scala-syntax:letter-group scala-syntax:digit-group "]+" "\\)*"
-          "\\(" "_+" scala-syntax:op-re "\\|" "_" "\\)?"))
-(defconst scala-syntax:varid-re (concat "[" scala-syntax:lower-group "]" scala-syntax:idrest-re))
-(defconst scala-syntax:capitalid-re (concat "[" scala-syntax:upperAndUnderscore-group "]" scala-syntax:idrest-re))
-;; alphaid introduce by SIP11
-(defconst scala-syntax:alphaid-re (concat "\\(" "[" scala-syntax:lower-group scala-syntax:upperAndUnderscore-group "]" scala-syntax:idrest-re "\\)"))
-(defconst scala-syntax:plainid-re (concat "\\(" scala-syntax:alphaid-re "\\|" scala-syntax:op-re "\\)"))
+(defconst scala-syntax:capitalid-re
+  (concat "[" scala-syntax:upperAndUnderscore-group "]" scala-syntax:idrest-re))
+(defconst scala-syntax:plainid-re
+  (concat "\\(" scala-syntax:alphaid-re "\\|" scala-syntax:op-re "\\)"))
 ;; stringlit is referred to, but not defined Scala Language Specification 2.9
 ;; we define it as consisting of anything but '`' and newline
 (defconst scala-syntax:stringlit-re "[^`\n\r]")
@@ -968,7 +1006,7 @@ not. A list must be either enclosed in parentheses or start with
   (regexp-opt '("override" "abstract" "final" "sealed" "implicit" "lazy" "using" "extension"
                 "private" "protected" "case") 'words))
 
-(defconst scala-syntax:whitespace-delimeted-modifiers-re
+(defconst scala-syntax:whitespace-delimited-modifiers-re
   (concat "\\(?:" scala-syntax:modifiers-re "\\(?: *\\)" "\\)*"))
 
 (defconst scala-syntax:definition-words-re
@@ -976,7 +1014,7 @@ not. A list must be either enclosed in parentheses or start with
 
 (defun scala-syntax:build-definition-re (words-re)
   (concat " *"
-	  scala-syntax:whitespace-delimeted-modifiers-re
+	  scala-syntax:whitespace-delimited-modifiers-re
 	  words-re
 	  "\\(?: *\\)"
 	  "\\(?2:"

From 7d190efccbab7acc9febccaa5d229d24aa05ce03 Mon Sep 17 00:00:00 2001
From: Keith Pinson <keith.pinson@banno.com>
Date: Tue, 13 Apr 2021 13:40:37 -0400
Subject: [PATCH 03/42] Octal numerals are no longer supported in Scala

I haven't been able to find exactly when it was dropped but it seems to have
actually been during Scala 2.
---
 scala-mode-syntax.el | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/scala-mode-syntax.el b/scala-mode-syntax.el
index 71f414d..93ff13f 100644
--- a/scala-mode-syntax.el
+++ b/scala-mode-syntax.el
@@ -87,15 +87,12 @@
 
 ;; Integer Literal
 (defconst scala-syntax:nonZeroDigit-group "1-9")
-(defconst scala-syntax:octalDigit-group "0-7")
 (defconst scala-syntax:decimalNumeral-re
   (concat "0"
           "\\|[" scala-syntax:nonZeroDigit-group "][" scala-syntax:digit-group "]*"))
 (defconst scala-syntax:hexNumeral-re (concat "0x[" scala-syntax:hexDigit-group "]+"))
-(defconst scala-syntax:octalNumeral-re (concat "0[" scala-syntax:octalDigit-group "]+"))
 (defconst scala-syntax:integerLiteral-re (concat "-?" ;; added from definition of literal
                                                  "\\(" scala-syntax:hexNumeral-re
-                                                 "\\|" scala-syntax:octalNumeral-re
                                                  "\\|" scala-syntax:decimalNumeral-re
                                                  "\\)[Ll]?"))
 

From d22aed22bc140e8d1fb3518cf2e06b25ef964e20 Mon Sep 17 00:00:00 2001
From: Keith Pinson <keith.pinson@banno.com>
Date: Tue, 13 Apr 2021 13:53:58 -0400
Subject: [PATCH 04/42] Continue to match `scala-mode-syntax` against the 3.0
 BNF

---
 scala-mode-syntax.el | 52 +++++++++++++++++++++++++++++++-------------
 1 file changed, 37 insertions(+), 15 deletions(-)

diff --git a/scala-mode-syntax.el b/scala-mode-syntax.el
index 93ff13f..ae15f29 100644
--- a/scala-mode-syntax.el
+++ b/scala-mode-syntax.el
@@ -18,6 +18,9 @@
 ;;; character groups without the enclosing [], i.e. they are not
 ;;; regular expressions, but can be used in declaring one.
 
+(defun scala-syntax:alt (&rest res)
+  (concat "\\(" (string-join res "\\|") "\\)"))
+
 ;; single letter matching groups (Chapter 1)
 (defconst scala-syntax:hexDigit-group "0-9A-Fa-f")
 (defconst scala-syntax:UnicodeEscape-re
@@ -58,8 +61,15 @@
 ;; Scala delimiters, but no quotes
 (defconst scala-syntax:delimiter-group ".,;")
 
-;; NOTE BNF also has a definition here for `printabeChar'
-;; NOTE BNF also has a definition here for `charEscapeSeq'
+;; NOTE BNF also has a definition here for `printableChar'
+;; `printableChar' has a definition that seems to restrict it to simple ASCII
+;; characters, though, which is surprising.
+;; We may not need it though, because e.g. in the definition of
+;; `characterLiteral' we use some faster regex.
+
+;; Escape Sequences (Chapter 1.3.6)
+;; NOTE BNF calls this `charEscapeSeq'
+(defconst scala-syntax:escapeSequence-re "\\\\['btnfr\"\\\\]")
 
 (defconst scala-syntax:op-re
   (concat "[" scala-syntax:opchar-group "]+" ))
@@ -83,20 +93,43 @@
           scala-syntax:upperAndUnderscore-group
           scala-syntax:idrest-re))
 
-;; TODO resume here (`plainid' in the BNF)
+(defconst scala-syntax:plainid-re
+  (scala-syntax:alt scala-syntax:alphaid-re
+                    scala-syntax:op-re))
+
+;; NOTE `stringlit' is referred to, but not defined in the Scala Language
+;; Specification 2.9. We define it as consisting of anything but '`' and newline
+;; TODO `stringLiteral' is defined in Scala 3, but probably differently, and for
+;; use in a different context.
+(defconst scala-syntax:stringlit-re "[^`\n\r]")
+;; NOTE there is `quoteId' but not `quotedId' in the Scala 3 BNF, and the
+;; `quoteId' is for something different.
+;; TODO The Scala 3 BNF has
+;;   { charNoBackQuoteOrNewline | UnicodeEscape | charEscapeSeq }
+;; rather than `stringlit'.
+(defconst scala-syntax:quotedid-re (concat "`" scala-syntax:stringlit-re "+`"))
+(defconst scala-syntax:id-re
+  (scala-syntax:alt scala-syntax:plainid-re scala-syntax:quotedid-re))
+
+(defconst scala-syntax:quoteid-re
+  (concat "'" scala-syntax:alphaid-re))
 
 ;; Integer Literal
 (defconst scala-syntax:nonZeroDigit-group "1-9")
+;; TODO add support for underscores to `decimalNumeral'
 (defconst scala-syntax:decimalNumeral-re
   (concat "0"
           "\\|[" scala-syntax:nonZeroDigit-group "][" scala-syntax:digit-group "]*"))
-(defconst scala-syntax:hexNumeral-re (concat "0x[" scala-syntax:hexDigit-group "]+"))
+;; TODO add support for underscores to `hexNumeral'
+(defconst scala-syntax:hexNumeral-re (concat "0[xX][" scala-syntax:hexDigit-group "]+"))
 (defconst scala-syntax:integerLiteral-re (concat "-?" ;; added from definition of literal
                                                  "\\(" scala-syntax:hexNumeral-re
                                                  "\\|" scala-syntax:decimalNumeral-re
                                                  "\\)[Ll]?"))
 
 
+;; TODO resume here (`floatingPointLiteral' in the BNF)
+
 ;; Floating Point Literal (Chapter 1.3.2)
 (defconst scala-syntax:exponentPart-re (concat "\\([eE][+-]?[" scala-syntax:digit-group "]+\\)"))
 (defconst scala-syntax:floatType-re "[fFdD]")
@@ -115,9 +148,6 @@
 ;; Boolean Literals (Chapter 1.3.3)
 (defconst scala-syntax:booleanLiteral-re "true|false")
 
-;; Escape Sequences (Chapter 1.3.6)
-(defconst scala-syntax:escapeSequence-re "\\\\['btnfr\"\\\\]")
-
 ;; Octal Escape Sequences (Chapter 1.3.6)
 (defconst scala-syntax:octalEscape-re (concat "\\\\[" scala-syntax:octalDigit-group "\\]\\{1,3\\}"))
 
@@ -162,14 +192,6 @@
 
 (defconst scala-syntax:capitalid-re
   (concat "[" scala-syntax:upperAndUnderscore-group "]" scala-syntax:idrest-re))
-(defconst scala-syntax:plainid-re
-  (concat "\\(" scala-syntax:alphaid-re "\\|" scala-syntax:op-re "\\)"))
-;; stringlit is referred to, but not defined Scala Language Specification 2.9
-;; we define it as consisting of anything but '`' and newline
-(defconst scala-syntax:stringlit-re "[^`\n\r]")
-(defconst scala-syntax:quotedid-re (concat "`" scala-syntax:stringlit-re "+`"))
-(defconst scala-syntax:id-re (concat "\\(" scala-syntax:plainid-re
-                              "\\|" scala-syntax:quotedid-re "\\)"))
 (defconst scala-syntax:id-first-char-group
   (concat scala-syntax:lower-group
           scala-syntax:upperAndUnderscore-group

From 18a27ce9c19942de4b2438770349de29c5fea715 Mon Sep 17 00:00:00 2001
From: Keith Pinson <keith.pinson@banno.com>
Date: Wed, 5 May 2021 12:05:54 -0400
Subject: [PATCH 05/42] Continue to match `scala-mode-syntax` against the 3.0
 BNF (2)

---
 scala-mode-syntax.el | 49 ++++++++++++++++++++++++++++++--------------
 1 file changed, 34 insertions(+), 15 deletions(-)

diff --git a/scala-mode-syntax.el b/scala-mode-syntax.el
index ae15f29..ff29f0c 100644
--- a/scala-mode-syntax.el
+++ b/scala-mode-syntax.el
@@ -99,8 +99,7 @@
 
 ;; NOTE `stringlit' is referred to, but not defined in the Scala Language
 ;; Specification 2.9. We define it as consisting of anything but '`' and newline
-;; TODO `stringLiteral' is defined in Scala 3, but probably differently, and for
-;; use in a different context.
+;; NOTE this is not the same thing as `stringLiteral'
 (defconst scala-syntax:stringlit-re "[^`\n\r]")
 ;; NOTE there is `quoteId' but not `quotedId' in the Scala 3 BNF, and the
 ;; `quoteId' is for something different.
@@ -116,31 +115,48 @@
 
 ;; Integer Literal
 (defconst scala-syntax:nonZeroDigit-group "1-9")
-;; TODO add support for underscores to `decimalNumeral'
 (defconst scala-syntax:decimalNumeral-re
   (concat "0"
-          "\\|[" scala-syntax:nonZeroDigit-group "][" scala-syntax:digit-group "]*"))
-;; TODO add support for underscores to `hexNumeral'
-(defconst scala-syntax:hexNumeral-re (concat "0[xX][" scala-syntax:hexDigit-group "]+"))
+          "\\|["
+          scala-syntax:nonZeroDigit-group
+          "]\\(["
+          scala-syntax:digit-group
+          "_]*["
+          scala-syntax:digit-group
+          "]\\)?"))
+(defconst scala-syntax:hexNumeral-re
+  (concat "0[xX]["
+          scala-syntax:hexDigit-group
+          "]\\(["
+          scala-syntax:hexDigit-group
+          "_]*["
+          scala-syntax:hexDigit-group
+          "]\\)?"))
+
 (defconst scala-syntax:integerLiteral-re (concat "-?" ;; added from definition of literal
                                                  "\\(" scala-syntax:hexNumeral-re
                                                  "\\|" scala-syntax:decimalNumeral-re
                                                  "\\)[Ll]?"))
 
-
-;; TODO resume here (`floatingPointLiteral' in the BNF)
-
 ;; Floating Point Literal (Chapter 1.3.2)
-(defconst scala-syntax:exponentPart-re (concat "\\([eE][+-]?[" scala-syntax:digit-group "]+\\)"))
+(defconst scala-syntax:exponentPart-re
+  (concat "\\([eE][+-]?["
+          scala-syntax:digit-group
+          "_]*["
+          scala-syntax:digit-group
+          "]\\)"))
+
 (defconst scala-syntax:floatType-re "[fFdD]")
+
 (defconst scala-syntax:floatingPointLiteral-re
   (concat "-?" ;; added from definition of literal
-          "\\([" scala-syntax:digit-group "]+\\.[" scala-syntax:digit-group "]*"
+          "\\(" "\\(" scala-syntax:decimalNumeral-re "\\)"
+          "\\.[" scala-syntax:digit-group "]"
+          "\\([" scala-syntax:digit-group "_]*" scala-syntax:digit-group "\\)?"
           scala-syntax:exponentPart-re "?" scala-syntax:floatType-re "?"
-          "\\|" "\\.[" scala-syntax:digit-group "]+"
-          scala-syntax:exponentPart-re "?" scala-syntax:floatType-re "?"
-          "\\|" "[" scala-syntax:digit-group "]+" scala-syntax:exponentPart-re
-          "\\|" "[" scala-syntax:digit-group "]+" scala-syntax:floatType-re "\\)"))
+          "\\|" "\\(" scala-syntax:decimalNumeral-re "\\)"
+          scala-syntax:exponentPart-re scala-syntax:floatType-re "?"
+          "\\|" "\\(" scala-syntax:decimalNumeral-re "\\)" scala-syntax:floatType-re "\\)"))
 
 (defconst scala-syntax:number-safe-start-re
   (concat "[^_" scala-syntax:letter-group "]"))
@@ -148,7 +164,10 @@
 ;; Boolean Literals (Chapter 1.3.3)
 (defconst scala-syntax:booleanLiteral-re "true|false")
 
+;; TODO resume here (`characterLiteral' in the BNF)
+
 ;; Octal Escape Sequences (Chapter 1.3.6)
+(defconst scala-syntax:octalDigit-group "0-7")
 (defconst scala-syntax:octalEscape-re (concat "\\\\[" scala-syntax:octalDigit-group "\\]\\{1,3\\}"))
 
 ;; Character Literals (Chapter 1.3.4)

From 265e69a2e44cbcfec0eea4f40c3efacf37d36c69 Mon Sep 17 00:00:00 2001
From: Keith Pinson <keith.pinson@banno.com>
Date: Tue, 15 Jun 2021 14:01:17 -0400
Subject: [PATCH 06/42] Beginning to see just how vastly daunting this change
 is

---
 scala-mode-indent.el | 92 ++++++++++++++++++++++++--------------------
 scala-mode-syntax.el | 19 ++++-----
 2 files changed, 57 insertions(+), 54 deletions(-)

diff --git a/scala-mode-indent.el b/scala-mode-indent.el
index 486ae65..89909c4 100644
--- a/scala-mode-indent.el
+++ b/scala-mode-indent.el
@@ -179,7 +179,7 @@ it is nilled."
 (defun scala-indent:backward-sexp-to-beginning-of-line ()
   "Skip sexps backwards until reaches beginning of line (i.e. the
 point is at the first non whitespace or comment character). It
-does not move outside enclosin list. Returns the current point or
+does not move outside enclosing list. Returns the current point or
 nil if the beginning of line could not be reached because of
 enclosing list."
   (let ((code-beg (scala-lib:point-after
@@ -196,17 +196,16 @@ enclosing list."
       (point))))
 
 (defun scala-indent:align-anchor ()
-  "Go to beginning of line, if a) scala-indent:align-parameters
-is nil or backward-sexp-to-beginning-of-line is non-nil. This has
-the effect of staying within lists if
-scala-indent:align-parameters is non-nil."
+  "Go to beginning of line, if a) `scala-indent:align-parameters' is nil or
+`scala-indent:backward-sexp-to-beginning-of-line' is non-nil. This has the
+effect of staying within lists if `scala-indent:align-parameters' is non-nil."
   (when (or (scala-indent:backward-sexp-to-beginning-of-line)
             (not scala-indent:align-parameters))
     (back-to-indentation)))
 
 (defun scala-indent:value-expression-lead (start anchor &optional not-block-p)
-  ;; calculate an indent lead. The lead is one indent step if there is
-  ;; a '=' between anchor and start, otherwise 0.
+  ;; calculate an indent lead. The lead is one indent step if there is a '='
+  ;; between anchor and start, otherwise 0.
   (if (and scala-indent:indent-value-expression
            (ignore-errors
              (save-excursion
@@ -229,11 +228,11 @@ expression")
 (defconst scala-indent:mustNotTerminate-line-beginning-re
   (concat "\\(" scala-indent:mustNotTerminate-keywords-re
           "\\|:\\("  scala-syntax:after-reserved-symbol-re "\\)\\)")
-  "All keywords and symbols that cannot terminate a expression
+  "All keywords and symbols that cannot terminate an expression
 and must be handled by run-on. Reserved-symbols not included.")
 
 (defconst scala-indent:mustTerminate-re
-  (concat "\\([,;\u21D2]\\|=>?" scala-syntax:end-of-code-line-re
+  (concat "\\([,;]\\|=>?" scala-syntax:end-of-code-line-re
           "\\|\\s(\\|" scala-syntax:empty-line-re "\\)")
   "Symbols that must terminate an expression or start a
 sub-expression, i.e the following expression cannot be a
@@ -251,8 +250,8 @@ and the empty line")
 (defconst scala-indent:mustBeContinued-line-end-re
   (concat "\\(" scala-syntax:other-keywords-unsafe-re
           "\\|:" scala-syntax:end-of-code-line-re "\\)")
-  "All keywords and symbols that cannot terminate a expression
-and are infact a sign of run-on. Reserved-symbols not included.")
+  "All keywords and symbols that cannot terminate an expression
+and are in fact a sign of run-on. Reserved-symbols not included.")
 
 (defun scala-indent:run-on-p (&optional point strategy)
   "Returns t if the current point is in the middle of an expression"
@@ -261,9 +260,8 @@ and are infact a sign of run-on. Reserved-symbols not included.")
   (save-excursion
     (when point (goto-char point))
     (unless (eobp)
-      ;; Note: ofcourse this 'cond' could be written as one big boolean
-      ;; expression, but I doubt that would be so readable and
-      ;; maintainable
+      ;; NOTE: of course this 'cond' could be written as one big boolean
+      ;; expression, but I doubt that would be so readable and maintainable
       (cond
        ;; NO: this line starts with close parenthesis
        ((= (char-syntax (char-after)) ?\))
@@ -358,7 +356,8 @@ is not on a run-on line."
     (point)))
 
 (defconst scala-indent:double-indent-re
-  (concat (regexp-opt '("with" "extends" "forSome") 'words)
+  ;; used to include with but given...with is a counterexample
+  (concat (regexp-opt '("extends" "forSome") 'words)
           "\\|:\\("  scala-syntax:after-reserved-symbol-re "\\)"))
 
 (defun scala-indent:resolve-run-on-step (start &optional anchor)
@@ -392,12 +391,12 @@ is not on a run-on line."
              scala-indent:step))))))
 
 (defconst scala-indent:forms-align-re
-  (regexp-opt '("yield" "else" "catch" "finally") 'words))
+  (regexp-opt '("yield" "then" "else" "catch" "finally") 'words))
 
 (defun scala-indent:forms-align-p (&optional point)
-  "Returns scala-syntax:beginning-of-code-line for the line on
+  "Returns `scala-syntax:beginning-of-code-line' for the line on
 which current point (or point 'point') is, if the line starts
-with one of 'yield', 'else', 'catch' and 'finally', otherwise
+with one of 'yield', 'then', 'else', 'catch' and 'finally', otherwise
 nil. Also, the previous line must not be with '}'"
   (save-excursion
     (when point (goto-char point))
@@ -406,7 +405,6 @@ nil. Also, the previous line must not be with '}'"
       (goto-char (match-beginning 0))
       (point))))
 
-
 (defun scala-indent:goto-forms-align-anchor (&optional point)
   "Moves back to the point whose column will be used as the
 anchor relative to which indenting of special words on beginning
@@ -428,6 +426,13 @@ special word found. Special words include 'yield', 'else',
                           (point)
                         (message "matching 'for' not found")
                         nil))
+                     ((looking-at "\\<then\\>")
+                      ;; align with 'if' or 'else if'
+                      (if (scala-syntax:search-backward-sexp "\\<if\\>")
+                          (if (scala-syntax:looking-back-token "\\<then\\>")
+                              (goto-char (match-beginning 0))
+                            (point))
+                        nil))
                      ((looking-at "\\<else\\>")
                       ;; align with 'if' or 'else if'
                       (if (scala-syntax:search-backward-sexp "\\<if\\>")
@@ -508,6 +513,7 @@ if not in a list of enumerators or at the first enumerator."
             (when (< (point) point)
               (1+ (nth 1 state)))))))))
 
+;; TODO this does not work properly with indent-based syntax
 (defun scala-indent:goto-for-enumerators-anchor (&optional point)
   "Moves back to the point whose column will be used to indent
 for enumerator at current point (or point 'point'). Returns the new
@@ -521,12 +527,14 @@ point or nil if the point is not in a enumerator element > 1."
 ;;;
 
 (defconst scala-indent:control-keywords-cond-re
-  (regexp-opt '("if" "while" "for") 'words)
+  ;; TODO `if' and `for' and possibly `while' no longer have to be followed by a
+  ;; parenthetical.
+  (regexp-opt '("for" "if" "while") 'words)
   "All the flow control keywords that are followed by a
 condition (or generators in the case of 'for') in parentheses.")
 
 (defconst scala-indent:control-keywords-other-re
-  (regexp-opt '("else" "do" "yield" "try" "finally" "catch") 'words)
+  (regexp-opt '("for" "if" "then" "else" "do" "yield" "try" "finally" "catch") 'words)
   "Other flow control keywords (not followed by parentheses)")
 
 (defconst scala-indent:control-keywords-re
@@ -535,22 +543,26 @@ condition (or generators in the case of 'for') in parentheses.")
 
 (defun scala-indent:body-p (&optional point)
   "Returns the position of '=' symbol, or one of the
-scala-indent:control-keywords-re or
-scala-indent:control-keywords-cond-re keywords if current
+`scala-indent:control-keywords-re' or
+`scala-indent:control-keywords-cond-re' keywords if current
 point (or point 'point) is on a line that follows said symbol or
 keyword, or nil if not."
+  ;; TODO this needs to look back more than one line
   (save-excursion
     (when point (goto-char point))
     (scala-syntax:beginning-of-code-line)
     (or (scala-syntax:looking-back-token scala-syntax:body-start-re 3)
         (let ((case-fold-search nil))
+          ;; This handles cases like
+          ;;   for
+          ;;     x <- List(1, 2, 3)
           (scala-syntax:looking-back-token scala-indent:control-keywords-other-re))
         (progn
           ;; if, else if
           (when (scala-syntax:looking-back-token ")" 1)
             (goto-char (match-end 0))
             (backward-list))
-          (when (scala-syntax:looking-back-token scala-indent:control-keywords-cond-re)
+          (when (scala-syntax:looking-back-token scala-indent:control-keywords-other-re)
             (goto-char (match-beginning 0))
             (when (and (looking-at "\\<if\\>")
                        (scala-syntax:looking-back-token "\\<else\\>"))
@@ -560,18 +572,18 @@ keyword, or nil if not."
             (point))))))
 
 (defun scala-indent:goto-body-anchor (&optional point)
-  (let ((declaration-end (scala-indent:body-p point)))
-    (when declaration-end
-      (goto-char declaration-end)
-      (if (let ((case-fold-search nil))
-            (looking-at scala-indent:control-keywords-re))
-          (point)
-        (when (scala-indent:backward-sexp-to-beginning-of-line)
-          (scala-indent:goto-run-on-anchor
-           nil
-           scala-indent:keywords-only-strategy))
-        (scala-indent:align-anchor)
-        (point)))))
+  ;; TODO this does not work right in indentation syntax
+  (when-let ((declaration-end (scala-indent:body-p point)))
+    (goto-char declaration-end)
+    (if (let ((case-fold-search nil))
+          (looking-at scala-indent:control-keywords-re))
+        (point)
+      (when (scala-indent:backward-sexp-to-beginning-of-line)
+        (scala-indent:goto-run-on-anchor
+         nil
+         scala-indent:keywords-only-strategy))
+      (scala-indent:align-anchor)
+      (point))))
 
 (defun scala-indent:resolve-body-step (start &optional anchor)
   (if (and (not (= start (point-max))) (= (char-after start) ?\{))
@@ -856,7 +868,7 @@ comment is outside the comment region. "
 
 (defconst scala-indent:indent-on-words-re
   (concat "^\\s *"
-          (regexp-opt '("catch" "case" "else" "finally" "yield") 'words)))
+          (regexp-opt '("catch" "case" "then" "else" "finally" "yield") 'words)))
 
 (defun scala-indent:indent-on-special-words ()
   "This function is meant to be used with post-self-insert-hook.
@@ -922,12 +934,8 @@ of a line inside a multi-line comment "
       (insert "*")
       (scala-indent:indent-on-scaladoc-asterisk))))
 
-(defun scala-mode:indent-scaladoc-asterisk (&optional insert-space-p)
-  (message "scala-mode:indent-scaladoc-asterisk has been deprecated"))
-
-
 (defun scala-indent:fixup-whitespace ()
-  "scala-mode version of `fixup-whitespace'"
+  "`scala-mode' version of `fixup-whitespace'"
   (interactive "*")
   (save-excursion
     (delete-horizontal-space)
diff --git a/scala-mode-syntax.el b/scala-mode-syntax.el
index ff29f0c..34b422c 100644
--- a/scala-mode-syntax.el
+++ b/scala-mode-syntax.el
@@ -343,8 +343,8 @@
 (defconst scala-syntax:end-of-code-line-re
   (concat "\\([ ]\\|$\\|" scala-syntax:comment-start-re "\\)")
   "A special regexp that can be concatenated to an other regular
-  expression when used with scala-syntax:looking-back-token. Not
-  meaningfull in other contexts.")
+  expression when used with `scala-syntax:looking-back-token'. Not
+  meaningful in other contexts.")
 
 (defconst scala-syntax:path-keywords-unsafe-re
   (regexp-opt '("super" "this") 'words))
@@ -381,13 +381,6 @@
           "\\|" scala-syntax:other-keywords-unsafe-re
           "\\)"))
 
-;; TODO: remove
-;; (defconst scala-syntax:keywords-re
-;;   (concat "\\(^\\|[^`'_]\\)\\(" scala-syntax:value-keywords-unsafe-re
-;;           "\\|" scala-syntax:path-keywords-unsafe-re
-;;           "\\|" scala-syntax:other-keywords-unsafe-re "\\)"))
-
-
 (defconst scala-syntax:after-reserved-symbol-underscore-re
   (concat "$\\|" scala-syntax:comment-start-re
           "\\|[^" scala-syntax:letterOrDigit-group "]"))
@@ -412,6 +405,7 @@
 
 (defconst scala-syntax:reserved-symbols-re
   ;; reserved symbols and XML starts ('<!' and '<?')
+  ;; TODO XML starts have been dropped from the language
   (concat "\\(^\\|[^" scala-syntax:opchar-group "]\\)"
           scala-syntax:reserved-symbols-unsafe-re
           "\\(" scala-syntax:after-reserved-symbol-re "\\)"))
@@ -821,8 +815,8 @@ and whitespace are skipped before matching."
     (looking-at re)))
 
 (defun scala-syntax:looking-back-token (re &optional max-chars)
-  "Return the start position of the token matched by re, if the
-current position is preceeded by it, or nil if not. All ignorable
+  "Return the start position of the token matched by RE, if the
+current position is preceded by it, or nil if not. All ignorable
 comments and whitespace are ignored, i.e. does not search past an
 empty line. Expects to be outside of comment. A limit for the
 search is calculated based on max-chars. The function won't look
@@ -871,6 +865,8 @@ one."
              (backward-list)
              (prog1 (scala-syntax:looking-back-token "if")
                (goto-char (match-beginning 0)))
+             (prog1 (scala-syntax:looking-back-token "then")
+               (goto-char (match-beginning 0)))
              (prog1 (scala-syntax:looking-back-token "else")
                (goto-char (match-beginning 0))))
         (point) nil)))
@@ -881,7 +877,6 @@ point 'point') as specified by SLS chapter 1.2"
   ;; newlines are disabled if
   ;; - in '()' or '[]'
   ;; - between 'case' and '=>'
-  ;; - XML mode (not implemented here)
   (unless point (setq point (point)))
   (save-excursion
     (let* ((state (syntax-ppss point))

From 5e73968e86ca29e230bd43c1c0d3f5a7f2a42cb4 Mon Sep 17 00:00:00 2001
From: Keith Pinson <keith.pinson@banno.com>
Date: Wed, 23 Jun 2021 17:03:31 -0600
Subject: [PATCH 07/42] Realized that whitespace syntax centers around block,
 not body

---
 scala-mode-indent.el | 76 +++++++++++++++++++++++---------------------
 1 file changed, 39 insertions(+), 37 deletions(-)

diff --git a/scala-mode-indent.el b/scala-mode-indent.el
index 89909c4..0f3817c 100644
--- a/scala-mode-indent.el
+++ b/scala-mode-indent.el
@@ -547,7 +547,7 @@ condition (or generators in the case of 'for') in parentheses.")
 `scala-indent:control-keywords-cond-re' keywords if current
 point (or point 'point) is on a line that follows said symbol or
 keyword, or nil if not."
-  ;; TODO this needs to look back more than one line
+  ;; TODO does this need to be adjusted for whitespace syntax?
   (save-excursion
     (when point (goto-char point))
     (scala-syntax:beginning-of-code-line)
@@ -567,7 +567,7 @@ keyword, or nil if not."
             (when (and (looking-at "\\<if\\>")
                        (scala-syntax:looking-back-token "\\<else\\>"))
               (goto-char (match-beginning 0)))
-            (when (not scala-indent:align-forms)
+            (unless scala-indent:align-forms
               (scala-indent:align-anchor))
             (point))))))
 
@@ -586,7 +586,7 @@ keyword, or nil if not."
       (point))))
 
 (defun scala-indent:resolve-body-step (start &optional anchor)
-  (if (and (not (= start (point-max))) (= (char-after start) ?\{))
+  (if (and (not (= start (point-max))) (or (= (char-after start) ?\{) (= (char-after start) ?:)))
       0
     (+ (scala-indent:value-expression-lead start anchor t)
        scala-indent:step)))
@@ -599,25 +599,26 @@ keyword, or nil if not."
   "Moves back to the point whose column will be used as the
 anchor for calculating block indent for current point (or point
 'point'). Returns point or (point-min) if not inside a block."
-  (let ((block-beg (nth 1 (syntax-ppss
-                           (scala-lib:point-after (beginning-of-line))))))
-    (when block-beg
-      ;; check if the opening paren is the first on the line,
-      ;; if so, it is the anchor. If not, then go back to the
-      ;; start of the line
-      (goto-char block-beg)
-      (if (= (point) (scala-lib:point-after
-                      (scala-syntax:beginning-of-code-line)))
-          (point)
-        (goto-char (or (scala-syntax:looking-back-token
-                        scala-syntax:body-start-re 3)
-                       (point)))
-        (scala-syntax:backward-parameter-groups)
-        (when (scala-indent:backward-sexp-to-beginning-of-line)
-          (scala-indent:goto-run-on-anchor nil
-                                           scala-indent:keywords-only-strategy))
-        (scala-indent:align-anchor)
-        (point)))))
+  ;; TODO this may be the heart of why indentation is not working for whitespace
+  ;; syntax. Need to finish updating the syntax definition on the BNF, and some
+  ;; of this may resolve naturally.
+  (when-let ((block-beg (nth 1 (syntax-ppss
+                                (scala-lib:point-after (beginning-of-line))))))
+    ;; Check if the opening paren is the first on the line, if so, it is the
+    ;; anchor. If not, then go back to the start of the line
+    (goto-char block-beg)
+    (if (= (point) (scala-lib:point-after
+                    (scala-syntax:beginning-of-code-line)))
+        (point)
+      (goto-char (or (scala-syntax:looking-back-token
+                      scala-syntax:body-start-re 3)
+                     (point)))
+      (scala-syntax:backward-parameter-groups)
+      (when (scala-indent:backward-sexp-to-beginning-of-line)
+        (scala-indent:goto-run-on-anchor nil
+                                         scala-indent:keywords-only-strategy))
+      (scala-indent:align-anchor)
+      (point))))
 
 (defun scala-indent:resolve-block-step (start anchor)
   "Resolves the appropriate indent step for block line at position
@@ -823,21 +824,22 @@ strings"
   "Indents the current line."
   (interactive "*")
   (let ((state (save-excursion (syntax-ppss (line-beginning-position)))))
-    (if (not (nth 8 state)) ;; 8 = start pos of comment or string, nil if none
-        (scala-indent:indent-code-line strategy)
-      (scala-indent:indent-line-to
-       (cond ((integerp (nth 4 state))    ;; 4 = nesting level of multi-line comment
-              (scala-indent:scaladoc-indent (nth 8 state)))
-             ((eq t (nth 3 state))   ;; 3 = t for multi-line string
-              (or (save-excursion
-                    (beginning-of-line)
-                    (when (and (looking-at "\\s *|")
-                               (progn (goto-char (nth 8 state))
-                                      (looking-at "\\(\"\"\"\\)|")))
-                      (goto-char (match-end 1))
-                      (current-column)))
-                  (current-indentation)))
-             (t (current-indentation)))))))
+    (if (nth 8 state) ;; 8 = start pos of comment or string
+        (scala-indent:indent-line-to
+         (cond ((integerp (nth 4 state))    ;; 4 = nesting level of multi-line comment
+                (scala-indent:scaladoc-indent (nth 8 state)))
+               ((eq t (nth 3 state))   ;; 3 = t for multi-line string
+                (or (save-excursion
+                      (beginning-of-line)
+                      (when (and (looking-at "\\s *|")
+                                 (progn (goto-char (nth 8 state))
+                                        (looking-at "\\(\"\"\"\\)|")))
+                        (goto-char (match-end 1))
+                        (current-column)))
+                    (current-indentation)))
+               (t (current-indentation))))
+      (scala-indent:indent-code-line strategy)))
+  )
 
 (defun scala-indent:indent-with-reluctant-strategy ()
   (interactive "*")

From cfeef682d945fea0c0675415f1fcf46fe44d61ed Mon Sep 17 00:00:00 2001
From: Keith Pinson <keith.pinson@banno.com>
Date: Wed, 23 Jun 2021 17:51:38 -0600
Subject: [PATCH 08/42] Still trying to figure out my way around

---
 scala-mode-syntax.el | 68 +++++++++++++++++++++-----------------------
 1 file changed, 32 insertions(+), 36 deletions(-)

diff --git a/scala-mode-syntax.el b/scala-mode-syntax.el
index 34b422c..6a6d800 100644
--- a/scala-mode-syntax.el
+++ b/scala-mode-syntax.el
@@ -460,6 +460,7 @@
 (defconst scala-syntax:private-re
   (concat "\\(^\\|[^`'_]\\)\\(" scala-syntax:private-unsafe-re "\\)"))
 
+;; TODO isn't this being dropped?
 (defconst scala-syntax:protected-unsafe-re
   (regexp-opt '("protected") 'words))
 
@@ -490,7 +491,6 @@
 (defconst scala-syntax:class-or-object-re
   (regexp-opt '("class" "object") 'words))
 
-
 ;;;;
 ;;;; Character syntax table and related syntax-propertize functions
 ;;;;
@@ -503,10 +503,9 @@
   "Syntax table used in `scala-mode' buffers.")
 (when (not scala-syntax:syntax-table)
   (let ((syntab (make-syntax-table)))
-    ;; 1. start by reseting the syntax table: only (){}[] are
-    ;; parentheses, so all others marked as parentheses in the parent
-    ;; table must be marked as symbols, nothing is a punctuation
-    ;; unless otherwise stated
+    ;; 1. start by reseting the syntax table: only (){}[] are parentheses, so
+    ;; all others marked as parentheses in the parent table must be marked as
+    ;; symbols, nothing is a punctuation unless otherwise stated
     (map-char-table
      #'(lambda (key value)
          (when (or (= (syntax-class value) 4) ; open
@@ -515,8 +514,8 @@
            (modify-syntax-entry key "_" syntab)))
      (char-table-parent syntab))
 
-    ;; Below 'space', everything is either illegal or whitespace.
-    ;; Consider as whitespace, unless otherwise stated below.
+    ;; Below 'space', everything is either illegal or whitespace. Consider as
+    ;; whitespace, unless otherwise stated below.
     (modify-syntax-entry '(0 . 32) " " syntab)
 
     ;; The scala parentheses
@@ -527,44 +526,40 @@
     (modify-syntax-entry ?\] ")[" syntab)
     (modify-syntax-entry ?\} "){" syntab)
 
-    ;; _ is upper-case letter, but will be modified to be symbol
-    ;; constituent when in reserved symbol position by
-    ;; syntax-propertize-function
+    ;; _ is upper-case letter, but will be modified to be symbol constituent
+    ;; when in reserved symbol position by `syntax-propertize-function'
     (modify-syntax-entry ?\_ "w" syntab)
 
-    ;; by default all opchars are punctuation, but they will be
-    ;; modified by syntax-propertize-function to be symbol
-    ;; constituents when a part of varid or capitalid
-    (dolist (char (mapcar 'identity "!#%&*+/:<=>?@^|~-\u21D2\u2190")) ;; TODO: Sm, So
+    ;; By default all opchars are punctuation, but they will be modified by
+    ;; `syntax-propertize-function' to be symbol constituents when a part of
+    ;; varid or capitalid
+    (dolist (char (mapcar #'identity "!#%&*+/:<=>?@^|~-")) ;; TODO: Sm, So
       (modify-syntax-entry char "." syntab))
 
-    ;; for clarity, the \ is alone here and not in the string above
+    ;; For clarity, the \ is alone here and not in the string above
     (modify-syntax-entry ?\\ "." syntab)
 
-    ;; scala strings cannot span lines, so we mark
-    ;; " as punctuation, but do the real stuff
-    ;; in syntax-propertize-function for properly
-    ;; formatted strings.
+    ;; Scala strings cannot span lines, so we mark " as punctuation, but do the
+    ;; real stuff in `syntax-propertize-function' for properly formatted
+    ;; strings.
     (modify-syntax-entry ?\" "." syntab)
 
-    ;; backquote is given paired delimiter syntax so that
-    ;; quoted ids are parsed as one sexp. Fontification
-    ;; is done separately.
+    ;; Backquote is given paired delimiter syntax so that quoted ids are parsed
+    ;; as one sexp. Fontification is done separately.
     (modify-syntax-entry ?\` "$" syntab)
 
-    ;; ' is considered an expression prefix, since it can
-    ;; both start a Symbol and is a char quote. It
-    ;; will be given string syntax by syntax-propertize-function
-    ;; for properly formatted char literals.
+    ;; ' is considered an expression prefix, since it can both start a Symbol
+    ;; and is a char quote. It will be given string syntax by
+    ;; `syntax-propertize-function' for properly formatted char literals.
     (modify-syntax-entry ?\' "'" syntab)
 
-    ;; punctuation as specified by SLS
+    ;; Punctuation as specified by SLS
     (modify-syntax-entry ?\. "." syntab)
     (modify-syntax-entry ?\; "." syntab)
     (modify-syntax-entry ?\, "." syntab)
 
-    ;; comments
-    ;; the `n' means that comments can be nested
+    ;; Comments
+    ;; The `n' means that comments can be nested
     (modify-syntax-entry ?\/  ". 124b" syntab)
     (modify-syntax-entry ?\*  ". 23n"   syntab)
     (modify-syntax-entry ?\n  "> b" syntab)
@@ -573,7 +568,7 @@
     (setq scala-syntax:syntax-table syntab)))
 
 (defun scala-syntax:propertize-extend-region (start end)
-  "See syntax-propertize-extend-region-functions"
+  "See `syntax-propertize-extend-region-functions'"
   ;; nothing yet
   nil)
 
@@ -706,7 +701,7 @@ symbol constituents (syntax 3)."
       (scala-syntax:put-syntax-table-property 0 '(1 . nil)))))
 
 (defun scala-syntax:propertize (start end)
-  "See syntax-propertize-function"
+  "See `syntax-propertize-function'"
   (scala-syntax:propertize-char-and-string-literals start end)
   (scala-syntax:propertize-shell-preamble start end)
   (scala-syntax:propertize-underscore-and-idrest start end)
@@ -719,12 +714,12 @@ symbol constituents (syntax 3)."
 ;;;;
 
 (defun scala-syntax:beginning-of-code-line ()
-  (interactive)
   "Move to the beginning of code on the line, or to the end of
 the line, if the line is empty. Return the new point.  Not to be
 called on a line whose start is inside a comment, i.e. a comment
 begins on the previous line and continues past the start of this
 line."
+  (interactive)
   ;; TODO: make it work even if the start IS inside a comment
   (beginning-of-line)
   (let ((eol (line-end-position))
@@ -975,7 +970,7 @@ is placed at the end of the skipped token."
   expression."
   (interactive)
   (syntax-propertize (point))
-  ;; for implementation comments, see scala-syntax:forward-sexp
+  ;; For implementation comments, see `scala-syntax:forward-sexp'
   (forward-comment (- (buffer-size)))
   (while (> 0 (+ (skip-syntax-backward " ")
                  (skip-chars-backward scala-syntax:delimiter-group))))
@@ -1015,7 +1010,7 @@ is returned, otherwise nil is returned"
 not. A list must be either enclosed in parentheses or start with
 'val', 'var' or 'import'."
   (save-excursion
-    ;; first check that the previous line ended with ','
+    ;; First check that the previous line ended with ','
     (when point (goto-char point))
     (scala-syntax:beginning-of-code-line)
     (when (and (scala-syntax:looking-back-token "," 1) (not (looking-at-p ")")))
@@ -1064,12 +1059,13 @@ not. A list must be either enclosed in parentheses or start with
   (condition-case ex (backward-sexp) ('error (backward-char))))
 
 (defun scala-syntax:forward-sexp-or-next-line ()
+  ;; TODO this is a candidate for updating for whitespace
   (interactive)
   (cond ((looking-at "\n") (forward-line 1) (beginning-of-line))
 	(t (forward-sexp))))
 
 (defun scala-syntax:beginning-of-definition ()
-  "This function may not work properly with certain types of scala definitions.
+  "This function may not work properly with certain types of Scala definitions.
 For example, no care has been taken to support multiple assignments to vals such as
 
 val a, b = (1, 2)
@@ -1083,7 +1079,7 @@ val a, b = (1, 2)
     (when found-position (progn (goto-char found-position) (back-to-indentation)))))
 
 (defun scala-syntax:end-of-definition ()
-  "This function may not work properly with certain types of scala definitions.
+  "This function may not work properly with certain types of Scala definitions.
 For example, no care has been taken to support multiple assignments to vals such as
 
 val a, b = (1, 2)

From 225081e02b7379e2b791e8e55305b00a89902aec Mon Sep 17 00:00:00 2001
From: Keith Pinson <keith.pinson@banno.com>
Date: Thu, 24 Jun 2021 12:52:45 -0600
Subject: [PATCH 09/42] Some crude but meaningful progress on indentation

---
 scala-mode-indent.el |  9 +++++----
 scala-mode-syntax.el | 45 +++++++++++++++++++++++++++++++-------------
 2 files changed, 37 insertions(+), 17 deletions(-)

diff --git a/scala-mode-indent.el b/scala-mode-indent.el
index 0f3817c..1841cf8 100644
--- a/scala-mode-indent.el
+++ b/scala-mode-indent.el
@@ -569,7 +569,11 @@ keyword, or nil if not."
               (goto-char (match-beginning 0)))
             (unless scala-indent:align-forms
               (scala-indent:align-anchor))
-            (point))))))
+            (point)))
+        (progn
+          ;; TODO this is crude and not quite right yet
+          (scala-syntax:beginning-of-definition)
+          (point)))))
 
 (defun scala-indent:goto-body-anchor (&optional point)
   ;; TODO this does not work right in indentation syntax
@@ -599,9 +603,6 @@ keyword, or nil if not."
   "Moves back to the point whose column will be used as the
 anchor for calculating block indent for current point (or point
 'point'). Returns point or (point-min) if not inside a block."
-  ;; TODO this may be the heart of why indentation is not working for whitespace
-  ;; syntax. Need to finish updating the syntax definition on the BNF, and some
-  ;; of this may resolve naturally.
   (when-let ((block-beg (nth 1 (syntax-ppss
                                 (scala-lib:point-after (beginning-of-line))))))
     ;; Check if the opening paren is the first on the line, if so, it is the
diff --git a/scala-mode-syntax.el b/scala-mode-syntax.el
index 6a6d800..ebfa4a3 100644
--- a/scala-mode-syntax.el
+++ b/scala-mode-syntax.el
@@ -468,6 +468,8 @@
   (concat "\\(^\\|[^`'_]\\)\\(" scala-syntax:protected-unsafe-re "\\)"))
 
 (defconst scala-syntax:modifiers-unsafe-re
+  ;; TODO is `protected` deprecated? Can `given` ever be a modifier, or does it
+  ;; always eat its `def`?
   (regexp-opt '("override" "abstract" "final" "sealed" "implicit" "lazy" "open"
                 "private" "protected") 'words))
 
@@ -475,6 +477,8 @@
   (concat "\\(^\\|[^`'_]\\)\\(" scala-syntax:modifiers-unsafe-re "\\)"))
 
 (defconst scala-syntax:body-start-re
+  ;; TODO this is getting complicated in Scala 3. A block can be introduced by
+  ;; `:` or `with`, but it depends on what kind of definition it is.
   (concat "=" scala-syntax:end-of-code-line-re)
   "A regexp for detecting if a line ends with '='")
 
@@ -1035,19 +1039,25 @@ not. A list must be either enclosed in parentheses or start with
                 "private" "protected" "case") 'words))
 
 (defconst scala-syntax:whitespace-delimited-modifiers-re
+  ;; NOTE this is sloppy because not all of these modifiers play together.
+  ;; Perhaps it's better that Emacs not get overly intelligent about this, lest
+  ;; its behavior be fragile in the face of partially incorrect code.
   (concat "\\(?:" scala-syntax:modifiers-re "\\(?: *\\)" "\\)*"))
 
 (defconst scala-syntax:definition-words-re
+  ;; TODO is `type` the odd man out here? That is, can it introduce a block, or
+  ;; not?
+  ;; TODO Also, is it necessary to worry about which is introduced by which of
+  ;; these is introduced by `:`, `=`, `with`, &c?
   (mapconcat 'regexp-quote '("class" "object" "trait" "val" "var" "def" "type" "enum" "given") "\\|"))
 
 (defun scala-syntax:build-definition-re (words-re)
-  (concat " *"
-	  scala-syntax:whitespace-delimited-modifiers-re
-	  words-re
-	  "\\(?: *\\)"
-	  "\\(?2:"
-	  scala-syntax:id-re
-	  "\\)"))
+  (concat scala-syntax:whitespace-delimited-modifiers-re
+          words-re
+          "\\(?: *\\)"
+          "\\(?2:"
+          scala-syntax:id-re ;; TODO should this be plain only?
+          "\\)"))
 
 (defconst scala-syntax:all-definition-re
   (scala-syntax:build-definition-re
@@ -1064,6 +1074,19 @@ not. A list must be either enclosed in parentheses or start with
   (cond ((looking-at "\n") (forward-line 1) (beginning-of-line))
 	(t (forward-sexp))))
 
+(defun scala-syntax:try-beginning-of-definition ()
+  (let ((max-indent (current-indentation))
+        (new-indent nil)
+        (pos nil))
+    (while (or (not new-indent) (and pos (>= new-indent max-indent)))
+      (scala-syntax:backward-sexp-forcing)
+      (setq pos (scala-syntax:movement-function-until-re
+                 scala-syntax:all-definition-re
+                 #'scala-syntax:backward-sexp-forcing))
+      (when pos (goto-char pos) (back-to-indentation))
+      (setq new-indent (current-indentation)))
+    pos))
+
 (defun scala-syntax:beginning-of-definition ()
   "This function may not work properly with certain types of Scala definitions.
 For example, no care has been taken to support multiple assignments to vals such as
@@ -1071,12 +1094,8 @@ For example, no care has been taken to support multiple assignments to vals such
 val a, b = (1, 2)
 "
   (interactive)
-  (let ((found-position
-	 (save-excursion
-	   (scala-syntax:backward-sexp-forcing)
-	   (scala-syntax:movement-function-until-re scala-syntax:all-definition-re
-						    'scala-syntax:backward-sexp-forcing))))
-    (when found-position (progn (goto-char found-position) (back-to-indentation)))))
+  (let ((found-position (save-excursion (scala-syntax:try-beginning-of-definition))))
+    (when found-position (goto-char found-position) (back-to-indentation))))
 
 (defun scala-syntax:end-of-definition ()
   "This function may not work properly with certain types of Scala definitions.

From 7dad271c919431213b10f98620ed966262b88085 Mon Sep 17 00:00:00 2001
From: Keith Pinson <keith.pinson@banno.com>
Date: Fri, 25 Jun 2021 10:22:48 -0400
Subject: [PATCH 10/42] Small and imprecise progress on indentation

---
 scala-mode-indent.el | 5 +++--
 scala-mode-syntax.el | 2 ++
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/scala-mode-indent.el b/scala-mode-indent.el
index 1841cf8..9eea7cd 100644
--- a/scala-mode-indent.el
+++ b/scala-mode-indent.el
@@ -573,7 +573,7 @@ keyword, or nil if not."
         (progn
           ;; TODO this is crude and not quite right yet
           (scala-syntax:beginning-of-definition)
-          (point)))))
+          (+ scala-indent:step (point))))))
 
 (defun scala-indent:goto-body-anchor (&optional point)
   ;; TODO this does not work right in indentation syntax
@@ -771,9 +771,10 @@ cannot be determined."
          (scala-indent:goto-for-enumerators-anchor scala-indent:resolve-list-step)
          (scala-indent:goto-forms-align-anchor scala-indent:resolve-forms-align-step)
          (scala-indent:goto-list-anchor scala-indent:resolve-list-step)
-         (scala-indent:goto-body-anchor scala-indent:resolve-body-step)
          (scala-indent:goto-run-on-anchor scala-indent:resolve-run-on-step)
          (scala-indent:goto-block-anchor scala-indent:resolve-block-step)
+         ;; TODO how order-sensitive is this logic?
+         (scala-indent:goto-body-anchor scala-indent:resolve-body-step)
      )
        point)
       0))
diff --git a/scala-mode-syntax.el b/scala-mode-syntax.el
index ebfa4a3..2eb281e 100644
--- a/scala-mode-syntax.el
+++ b/scala-mode-syntax.el
@@ -1049,6 +1049,8 @@ not. A list must be either enclosed in parentheses or start with
   ;; not?
   ;; TODO Also, is it necessary to worry about which is introduced by which of
   ;; these is introduced by `:`, `=`, `with`, &c?
+  ;; TODO what about `new`? Following by a trait or abstract class name and a
+  ;; colon, it can now introduce a block.
   (mapconcat 'regexp-quote '("class" "object" "trait" "val" "var" "def" "type" "enum" "given") "\\|"))
 
 (defun scala-syntax:build-definition-re (words-re)

From cd11904802416d7c09cdbfd79bccfde11e6160e3 Mon Sep 17 00:00:00 2001
From: Keith Pinson <keith.pinson@banno.com>
Date: Thu, 22 Jul 2021 15:14:03 -0400
Subject: [PATCH 11/42] finish going through the lexical syntax, and some

---
 scala-mode-fontlock.el |  26 ++---
 scala-mode-syntax.el   | 217 ++++++++++++++++++++---------------------
 2 files changed, 111 insertions(+), 132 deletions(-)

diff --git a/scala-mode-fontlock.el b/scala-mode-fontlock.el
index 7617259..9f3eaf9 100644
--- a/scala-mode-fontlock.el
+++ b/scala-mode-fontlock.el
@@ -5,9 +5,10 @@
 (require 'scala-mode-syntax)
 
 (defcustom scala-font-lock:constant-list '()
-  "A list of strigs that should be fontified in constant
-face. This customization property takes effect only after the
-scala-mode has been reloaded."
+  "A list of strings that should be fontified in constant face.
+
+This customization property takes effect only after the `scala-mode' has been
+reloaded."
   :type '(repeat string)
   :group 'scala)
 
@@ -22,8 +23,6 @@ scala-mode has been reloaded."
   (when (re-search-forward scala-syntax:reserved-symbol-underscore-re limit t)
       (goto-char (match-end 2)))) ;; step back to the match (re matches futher)
 
-;(defun scala-font-lock:extend-region-function ()
-
 (defun scala-font-lock:limit-pattern2 (&optional start)
   (save-excursion
     (when start (goto-char start))
@@ -363,20 +362,9 @@ Does not continue past limit.
               "\\)")
      2 font-lock-type-face)
 
-    ;; ;; extends, with, new
-    ;; (,(concat "\\<\\(extends\\|with\\|new\\)[ \t]+\\([("
-    ;;           scala-syntax:id-first-char-group "]\\)")
-    ;;  (scala-font-lock:mark-simpleType (scala-font-lock:limit-simpleType
-    ;;                                    (goto-char (match-beginning 2)))
-    ;;                                   nil
-    ;;                                   (0 font-lock-type-face nil t)))
-
-    ;; ;; ':'
-    ;; (,scala-syntax:colon-re
-    ;;  (scala-font-lock:mark-simpleType (scala-font-lock:limit-simpleType
-    ;;                                    (goto-char (match-end 2)))
-    ;;                                   nil
-    ;;                                   (0 font-lock-type-face nil t)))
+    ;; TODO extends, with, new
+
+    ;; TODO ':'
 
     ;; def
     (,(concat "\\<def[ \t]+\\(" scala-syntax:id-re "\\)") 1 font-lock-function-name-face)
diff --git a/scala-mode-syntax.el b/scala-mode-syntax.el
index 2eb281e..6e44365 100644
--- a/scala-mode-syntax.el
+++ b/scala-mode-syntax.el
@@ -21,7 +21,7 @@
 (defun scala-syntax:alt (&rest res)
   (concat "\\(" (string-join res "\\|") "\\)"))
 
-;; single letter matching groups (Chapter 1)
+;; single letter matching groups
 (defconst scala-syntax:hexDigit-group "0-9A-Fa-f")
 (defconst scala-syntax:UnicodeEscape-re
   ;; using `format' allows editing these regexes with one step closer to a sane
@@ -67,9 +67,8 @@
 ;; We may not need it though, because e.g. in the definition of
 ;; `characterLiteral' we use some faster regex.
 
-;; Escape Sequences (Chapter 1.3.6)
-;; NOTE BNF calls this `charEscapeSeq'
-(defconst scala-syntax:escapeSequence-re "\\\\['btnfr\"\\\\]")
+;; Escape Sequences
+(defconst scala-syntax:charEscapeSeq-re "\\\\['btnfr\"\\\\]")
 
 (defconst scala-syntax:op-re
   (concat "[" scala-syntax:opchar-group "]+" ))
@@ -138,7 +137,7 @@
                                                  "\\|" scala-syntax:decimalNumeral-re
                                                  "\\)[Ll]?"))
 
-;; Floating Point Literal (Chapter 1.3.2)
+;; Floating Point Literal
 (defconst scala-syntax:exponentPart-re
   (concat "\\([eE][+-]?["
           scala-syntax:digit-group
@@ -161,32 +160,26 @@
 (defconst scala-syntax:number-safe-start-re
   (concat "[^_" scala-syntax:letter-group "]"))
 
-;; Boolean Literals (Chapter 1.3.3)
+;; Boolean Literals
 (defconst scala-syntax:booleanLiteral-re "true|false")
 
-;; TODO resume here (`characterLiteral' in the BNF)
-
-;; Octal Escape Sequences (Chapter 1.3.6)
-(defconst scala-syntax:octalDigit-group "0-7")
-(defconst scala-syntax:octalEscape-re (concat "\\\\[" scala-syntax:octalDigit-group "\\]\\{1,3\\}"))
-
-;; Character Literals (Chapter 1.3.4)
+;; Character Literals
 (defconst scala-syntax:characterLiteral-re
+  ;; TODO Scala 3 character literal does not mention Unicode escape. Is that a
+  ;; mistake?
   (concat "\\('\\)\\(" "[^\\\\]" ;; should be just printable char, but this is faster
-          "\\|" scala-syntax:escapeSequence-re
-          "\\|" scala-syntax:octalEscape-re
+          "\\|" scala-syntax:charEscapeSeq-re
           "\\|" scala-syntax:UnicodeEscape-re "\\)\\('\\)"))
 
-(defconst scala-syntax:string-escape-re
-  (concat scala-syntax:escapeSequence-re
-          "\\|" scala-syntax:octalEscape-re
-          "\\|" scala-syntax:UnicodeEscape-re))
-
-;; String Literals (Chapter 1.3.5)
+;; String Literals
 (defconst scala-syntax:stringElement-re
   (concat "\\(" "[^\n\"\\\\]"
-          "\\|" scala-syntax:string-escape-re  "\\)"))
-(defconst scala-syntax:oneLineStringLiteral-re (concat "\\(\"\\)" scala-syntax:stringElement-re "*\\(\"\\)"))
+          "\\|" scala-syntax:charEscapeSeq-re
+          "\\|" scala-syntax:UnicodeEscape-re  "\\)"))
+
+(defconst scala-syntax:oneLineStringLiteral-re
+  (concat "\\(\"\\)" scala-syntax:stringElement-re "*\\(\"\\)"))
+
 (defconst scala-syntax:multiLineStringLiteral-start-re
   "\\(\"\\)\"\"\\(\"?\"?[^\"]\\)*")
 (defconst scala-syntax:multiLineStringLiteral-end-re
@@ -198,30 +191,30 @@
   (concat "\\(" scala-syntax:multiLineStringLiteral-re
           "\\|" scala-syntax:oneLineStringLiteral-re "\\)" ))
 
-;; If you change this or any of the used regex, be sure to
-;; maintain this or update propertize function accordingly:
-;; group 1 = char start, 3 = char end
-;; group 4 = multi-line string start, 6 = end
-;; group 7 = string start, 9 = end
+;; If you change this or any of the used regex, be sure to maintain this or
+;; update propertize function accordingly. Start and end group numbers:
+;; |-------------------+-------+-----|
+;; | Syntax item       | Start | End |
+;; |-------------------+-------+-----|
+;; | Character         |     1 |   3 |
+;; | Multi-line string |     4 |   6 |
+;; | One-line string   |     7 |   9 |
+;; |-------------------+-------+-----|
 (defconst scala-syntax:relaxed-char-and-string-literal-re
   (concat scala-syntax:characterLiteral-re
           "\\|" scala-syntax:multiLineStringLiteral-start-re
           "\\(?:" scala-syntax:multiLineStringLiteral-end-re "\\)?"
           "\\|\\(\"\\)" "\\(\\\\.\\|[^\"\n\\]\\)*" "\\(\"\\)"))
 
-(defconst scala-syntax:capitalid-re
-  (concat "[" scala-syntax:upperAndUnderscore-group "]" scala-syntax:idrest-re))
-(defconst scala-syntax:id-first-char-group
-  (concat scala-syntax:lower-group
-          scala-syntax:upperAndUnderscore-group
-          scala-syntax:opchar-group))
-
-;; Symbol literals (Chapter 1.3.7)
+;; Symbol literals
 (defconst scala-syntax:symbolLiteral-re
-  ;; must end with non-' to not conflict with scala-syntax:characterLiteral-re
+  ;; NOTE must end with non-' to not conflict with
+  ;; `scala-syntax:characterLiteral-re'
   (concat "\\('" scala-syntax:plainid-re "\\)\\([^']\\|$\\)"))
 
-;; Literals (Chapter 1.3)
+;; Literals
+;; TODO Scala 3 distinguishes between Literal and SimpleLiteral; if that becomes
+;; important below, then distinguish here.
 (defconst scala-syntax:literal-re
   (concat "\\(" scala-syntax:integerLiteral-re
           "\\|" scala-syntax:floatingPointLiteral-re
@@ -231,6 +224,11 @@
           "\\|" scala-syntax:symbolLiteral-re
           "\\|" "null" "\\)"))
 
+;; TODO Scala 3 defines QualId, ids and SimpleRef here. SimpleRef seems to bear
+;; a relation to what was before called "path".
+
+;; TODO does not occur under this name in Scala 3 BNF; is this was what is there
+;; called escape, processedStringLiteral, processedStringPart, &c?
 (defconst scala-syntax:interpolation-re
   (concat "\\(" "\\$"  scala-syntax:id-re "\\|" "\\${[^}\n\\\\]*}" "\\)"))
 
@@ -254,18 +252,9 @@
               t))) ;; keep going
     pos))
 
-;; Paths (Chapter 3.1)
-;; emacs has a problem with these regex, don't use them
-;; (defconst scala-syntax:classQualifier-re (concat "[[]" scala-syntax:id-re "[]]"))
-;; (defconst scala-syntax:stableId-re
-;;   (concat "\\(\\(" "this"
-;;           "\\|" "super" scala-syntax:classQualifier-re
-;;           "\\|" scala-syntax:id-re
-;;           "\\)\\.\\)*"
-;;           scala-syntax:id-re))
-;; (defconst scala-syntax:path-re
-;;   (concat "\\(" scala-syntax:stableId-re
-;;           "\\|" "\\(" scala-syntax:id-re "\\." "\\)?" "this" "\\)"))
+;; NOTE Emacs has a problem with these regexes; don't use them.
+;; (defconst scala-syntax:classQualifier-re
+;;   (concat "[[]" scala-syntax:id-re "[]]"))
 
 (defun scala-syntax:looking-at-super ()
   (save-excursion
@@ -292,6 +281,8 @@
           (set-match-data `(,beg ,(match-end 0)))
           t)))))
 
+;; NOTE written as a function because Emacs struggled with the regex
+;; TODO I don't see references to "stable" identifiers in Scala 3 BNF
 (defun scala-syntax:looking-at-stableIdOrPath (&optional path-p beg)
   (unless beg (setq beg (point)))
   (save-excursion
@@ -312,21 +303,12 @@
              (set-match-data `(,beg ,(match-end 0)))
              (point))))))
 
+;; TODO update per SimplePattern in Scala 3 BNF
 (defun scala-syntax:looking-at-simplePattern-beginning ()
   (or (looking-at "[_(]")
       (looking-at scala-syntax:literal-re)
       (scala-syntax:looking-at-stableIdOrPath)))
 
-
-(defun scala-syntax:regexp-for-id (id)
-  (let ((prefix-regex
-         (if (string-match scala-syntax:alphaid-re id)
-             "\\b" (concat "\\(^\\|[^" scala-syntax:opchar-group "]\\)")))
-        (suffix-regex
-         (if (string-match scala-syntax:op-re (substring id -1 nil))
-             (concat "\\([^" scala-syntax:opchar-group "]\\|$\\)") "\\b")))
-    (concat prefix-regex id suffix-regex)))
-
 ;;;
 ;;; Other regular expressions
 ;;;
@@ -346,6 +328,8 @@
   expression when used with `scala-syntax:looking-back-token'. Not
   meaningful in other contexts.")
 
+;; TODO there is only one reference to the word "path" in the Scala 3 BNF. Is
+;; that a vestige? Should this be renamed?
 (defconst scala-syntax:path-keywords-unsafe-re
   (regexp-opt '("super" "this") 'words))
 
@@ -385,19 +369,17 @@
   (concat "$\\|" scala-syntax:comment-start-re
           "\\|[^" scala-syntax:letterOrDigit-group "]"))
 
+;; Reserved symbol _
 (defconst scala-syntax:reserved-symbol-underscore-re
-  ;; reserved symbol _
   (concat "\\(^\\|[^" scala-syntax:letterOrDigit-group "]\\)"
           "\\(_\\)"
           "\\(" scala-syntax:after-reserved-symbol-underscore-re "\\)"))
 
+;; Reserved symbols. The regexp is unsafe as it does not check the context.
 (defconst scala-syntax:reserved-symbols-unsafe-re
-  ;; reserved symbols. The regexp is unsafe as it does not
-  ;; check the context.
-  "\\([:#@\u21D2\u2190]\\|=>?\\|<[:%!?\\-]\\|>:\\)" )
+  "\\([:#@]\\|=>?\\|<[:%!?\\-]\\|>:\\)" )
 
-(defconst scala-syntax:double-arrow-unsafe-re
-  "\\(=>\\|\u21D2\\)")
+(defconst scala-syntax:double-arrow-unsafe-re "=>")
 
 (defconst scala-syntax:after-reserved-symbol-re
   (concat "\\($\\|" scala-syntax:comment-start-re
@@ -410,12 +392,6 @@
           scala-syntax:reserved-symbols-unsafe-re
           "\\(" scala-syntax:after-reserved-symbol-re "\\)"))
 
-(defconst scala-syntax:colon-re
-  (concat "\\(^\\|[^" scala-syntax:opchar-group "]\\)"
-          "\\(:\\)"
-          "\\(" scala-syntax:after-reserved-symbol-re "\\)"))
-
-
 (defconst scala-syntax:override-unsafe-re
   (regexp-opt '("override") 'words))
 
@@ -482,6 +458,9 @@
   (concat "=" scala-syntax:end-of-code-line-re)
   "A regexp for detecting if a line ends with '='")
 
+;; NOTE "list" here seems to refer to the Emacs S-Expr concept of a List, not
+;; anything that is considered a list in Scala.
+;; TODO what might be new here for Scala 3?
 (defconst scala-syntax:list-keywords-re
   (regexp-opt '("var" "val" "import") 'words)
   ("Keywords that can start a list"))
@@ -499,17 +478,16 @@
 ;;;; Character syntax table and related syntax-propertize functions
 ;;;;
 
-;;; The syntax table relies havily on the syntax-propertize-functions being
-;;; run. Hence this syntax requires at least emacs 24, which introduced
-;;; this new facility.
+;;; The syntax table relies havily on the `syntax-propertize-function's being
+;;; run. Hence this syntax requires at least Emacs 24.
 
 (defvar scala-syntax:syntax-table nil
   "Syntax table used in `scala-mode' buffers.")
 (when (not scala-syntax:syntax-table)
   (let ((syntab (make-syntax-table)))
-    ;; 1. start by reseting the syntax table: only (){}[] are parentheses, so
-    ;; all others marked as parentheses in the parent table must be marked as
-    ;; symbols, nothing is a punctuation unless otherwise stated
+    ;; Start by reseting the syntax table: only (){}[] are parentheses, so all
+    ;; others marked as parentheses in the parent table must be marked as
+    ;; symbols. Nothing is a punctuation unless otherwise stated.
     (map-char-table
      #'(lambda (key value)
          (when (or (= (syntax-class value) 4) ; open
@@ -522,7 +500,7 @@
     ;; whitespace, unless otherwise stated below.
     (modify-syntax-entry '(0 . 32) " " syntab)
 
-    ;; The scala parentheses
+    ;; The Scala parentheses
     (modify-syntax-entry ?\( "()" syntab)
     (modify-syntax-entry ?\[ "(]" syntab)
     (modify-syntax-entry ?\{ "(}" syntab)
@@ -536,7 +514,7 @@
 
     ;; By default all opchars are punctuation, but they will be modified by
     ;; `syntax-propertize-function' to be symbol constituents when a part of
-    ;; varid or capitalid
+    ;; varid or capitalid ;; TODO capitalid not mentioned in Scala 3 BNF
     (dolist (char (mapcar #'identity "!#%&*+/:<=>?@^|~-")) ;; TODO: Sm, So
       (modify-syntax-entry char "." syntab))
 
@@ -549,7 +527,7 @@
     (modify-syntax-entry ?\" "." syntab)
 
     ;; Backquote is given paired delimiter syntax so that quoted ids are parsed
-    ;; as one sexp. Fontification is done separately.
+    ;; as one S-expression. Fontification is done separately.
     (modify-syntax-entry ?\` "$" syntab)
 
     ;; ' is considered an expression prefix, since it can both start a Symbol
@@ -577,21 +555,20 @@
   nil)
 
 (defmacro scala-syntax:put-syntax-table-property (match-group value)
-  "Add 'syntax-table entry 'value' to the region marked by the
-match-group 'match-group'"
+  "Add `syntax-table' entry VALUE to the region marked by MATCH-GROUP"
   `(put-text-property (match-beginning ,match-group)
                       (match-end ,match-group)
                       'syntax-table
                       ,value))
 
 (defun scala-syntax:propertize-char-and-string-literals (start end)
-  "Mark start and end of character literals as well as one-line
-and multi-line string literals. One-line strings and characters
-use syntax class 7 (string quotes), while multi-line strings are
-marked with 15 (generic string delimiter). Multi-line string
-literals are marked even if they are unbalanced. One-line string
-literals have to be balanced to get marked. This means invalid
-characters and one-line strings will not be fontified."
+  "Mark start and end of character and string literals.
+
+One-line strings and characters use syntax class 7 (string quotes), while
+multi-line strings are marked with 15 (generic string delimiter). Multi-line
+string literals are marked even if they are unbalanced. One-line string literals
+have to be balanced to get marked. This means invalid characters and one-line
+strings will not be fontified."
 
   (let* ((string-state (nth 3 (syntax-ppss start)))
          (unbalanced-p (eq string-state t)))
@@ -650,6 +627,7 @@ characters and one-line strings will not be fontified."
       (when (re-search-forward "\n" end t)
         (scala-syntax:put-syntax-table-property 0 '(12 . nil))))))
 
+;; TODO what makes underscores to be considered uppercase?
 (defun scala-syntax:propertize-underscore-and-idrest (start end)
   "Mark all underscores (_) as symbol constituents (syntax 3) or
 upper case letter (syntax 2). Also mark opchars in idrest as
@@ -685,7 +663,8 @@ symbol constituents (syntax 3)."
         (unless (or
                  (string-suffix-p "*/" match)
                  (member match '("</"))
-                 (member 0 (mapcar (lambda (regexp) (string-match regexp match)) '("^*+/$" "^//.*$" "^/\\*+$")))
+                 (member 0 (mapcar (lambda (regexp) (string-match regexp match))
+                                   '("^*+/$" "^//.*$" "^/\\*+$")))
                  (equal 2 (syntax-class (syntax-after match-end)))
                  (equal 2 (syntax-class (syntax-after (1- match-beg)))))
           (put-text-property match-beg match-end 'syntax-table '(3 . nil)))))))
@@ -739,9 +718,9 @@ line."
       (skip-syntax-forward " " eol)
       (point))))
 
+;; TODO Scala 3: nothing called stable
 (defun scala-syntax:looking-at-varid-p (&optional point)
-  "Return true if looking-at varid, and it is not the start of a
-stableId"
+  "Return true if `looking-at' varid, and it is not the start of a stableId"
   (save-excursion
     (when point (goto-char point))
     (scala-syntax:skip-forward-ignorable)
@@ -871,8 +850,7 @@ one."
         (point) nil)))
 
 (defun scala-syntax:newlines-disabled-p (&optional point)
-  "Return true if newlines are disabled at the current point (or
-point 'point') as specified by SLS chapter 1.2"
+  "Whether newlines are disabled at the current point (or point POINT)"
   ;; newlines are disabled if
   ;; - in '()' or '[]'
   ;; - between 'case' and '=>'
@@ -912,15 +890,15 @@ point 'point') as specified by SLS chapter 1.2"
                              (not (looking-at scala-syntax:class-or-object-re)))))))))))))
 
 (defun scala-syntax:forward-sexp ()
-  "Move forward one scala expression. It can be: parameter list (value or type),
-id, reserved symbol, keyword, block, or literal. Punctuation (.,;)
-and comments are skipped silently. Position is placed at the
-end of the skipped expression."
+  "Move forward one Scala expression.
+
+It can be: parameter list (value or type), id, reserved symbol, keyword, block,
+or literal. Punctuation (.,;) and comments are skipped silently. Position is
+placed at the end of the skipped expression."
   (interactive)
   (syntax-propertize (point-max))
-  ;; emacs knows how to properly skip: lists, varid, capitalid,
-  ;; strings, symbols, chars, quotedid. What we have to handle here is
-  ;; most of all ids made of op chars
+  ;; Emacs knows how to properly skip: lists, varid, strings, symbols, chars,
+  ;; quotedid. What we have to handle here is most of all ids made of op chars.
 
   ;; skip comments, whitespace and scala delimiter chars .,; so we
   ;; will be at the start of something interesting
@@ -933,11 +911,11 @@ end of the skipped expression."
     (goto-char (or (scan-sexps (point) 1) (buffer-end 1)))))
 
 (defun scala-syntax:forward-token ()
-  "Move forward one scala token, comment word or string word. It
-can be: start or end of list (value or type), id, reserved
-symbol, keyword, block, or literal. Punctuation (.,;), comment
-delimiters and string delimiters are skipped silently. Position
-is placed at the end of the skipped token."
+  "Move forward one Scala token, comment word or string word.
+
+It can be: start or end of list (value or type), id, reserved symbol, keyword,
+block, or literal. Punctuation (.,;), comment delimiters and string delimiters
+are skipped silently. Position is placed at the end of the skipped token."
   (interactive)
   (syntax-propertize (point-max))
   (skip-syntax-forward " >" (point-max))
@@ -1009,8 +987,8 @@ is returned, otherwise nil is returned"
     (when found (goto-char found))))
 
 (defun scala-syntax:list-p (&optional point)
-  "Returns the start of the list, if the current point (or point
-'point') is on the first line of a list element > 1, or nil if
+  "Returns the start of the list, if the current point (or POINT)
+is on the first line of a list element > 1, or nil if
 not. A list must be either enclosed in parentheses or start with
 'val', 'var' or 'import'."
   (save-excursion
@@ -1032,11 +1010,21 @@ not. A list must be either enclosed in parentheses or start with
           (when (looking-at scala-syntax:list-keywords-re)
             (goto-char (match-end 0))))))))
 
-;; Functions to help with finding the beginning and end of scala definitions.
+;; Functions to help with finding the beginning and end of Scala definitions.
+
+(defconst scala-syntax:local-modifiers
+  ;; TODO what about case, extension?
+  '("abstract" "final" "sealed" "open" "implicit" "lazy" "inline"))
+;; TODO Scala 3 BNF has:
+;;   (‘private’ | ‘protected’) [AccessQualifier]
+(defconst scala-syntax:access-modifiers
+  '("private" "protected"))
 
 (defconst scala-syntax:modifiers-re
-  (regexp-opt '("override" "abstract" "final" "sealed" "implicit" "lazy" "using" "extension"
-                "private" "protected" "case") 'words))
+  (regexp-opt (concat '("override" "opaque")
+                      scala-syntax:local-modifiers
+                      scala-syntax:access-modifiers)
+              'words))
 
 (defconst scala-syntax:whitespace-delimited-modifiers-re
   ;; NOTE this is sloppy because not all of these modifiers play together.
@@ -1044,6 +1032,7 @@ not. A list must be either enclosed in parentheses or start with
   ;; its behavior be fragile in the face of partially incorrect code.
   (concat "\\(?:" scala-syntax:modifiers-re "\\(?: *\\)" "\\)*"))
 
+;; TODO work on the "Declarations and Definitions" section of the Scala 3 BNF
 (defconst scala-syntax:definition-words-re
   ;; TODO is `type` the odd man out here? That is, can it introduce a block, or
   ;; not?
@@ -1051,7 +1040,9 @@ not. A list must be either enclosed in parentheses or start with
   ;; these is introduced by `:`, `=`, `with`, &c?
   ;; TODO what about `new`? Following by a trait or abstract class name and a
   ;; colon, it can now introduce a block.
-  (mapconcat 'regexp-quote '("class" "object" "trait" "val" "var" "def" "type" "enum" "given") "\\|"))
+  (mapconcat 'regexp-quote
+             '("class" "object" "trait" "val" "var" "def" "type" "enum" "given")
+             "\\|"))
 
 (defun scala-syntax:build-definition-re (words-re)
   (concat scala-syntax:whitespace-delimited-modifiers-re

From 1fa19e63a6cd6c37bb1886a3f0eeca048b44819b Mon Sep 17 00:00:00 2001
From: Keith Pinson <keith.pinson@banno.com>
Date: Thu, 22 Jul 2021 15:59:29 -0400
Subject: [PATCH 12/42] I am not going to make progress just by tweaking

I have to tear down first, before I can rebuild. 20% of the algorithm gets us
80% of the way there for braces. This totally breaks whitespace again, but it
wasn't _really_ working with what I had before, and it will be much easier to
get it working once we get a bunch of stuff out of the way. So many concepts of
have shifted, and it's just too hard to read Emacs-style code which is not
declarative. More tearing down may come after this.
---
 scala-mode-indent.el | 101 +++++++------------------------------------
 scala-mode-syntax.el |  15 +++----
 2 files changed, 22 insertions(+), 94 deletions(-)

diff --git a/scala-mode-indent.el b/scala-mode-indent.el
index 9eea7cd..1975890 100644
--- a/scala-mode-indent.el
+++ b/scala-mode-indent.el
@@ -87,11 +87,11 @@ val x = if (foo)
   :group 'scala)
 
 (defconst scala-indent:eager-strategy 0
-  "See 'scala-indent:run-on-strategy'")
+  "See `scala-indent:run-on-strategy'")
 (defconst scala-indent:operator-strategy 1
-  "See 'scala-indent:run-on-strategy'")
+  "See `scala-indent:run-on-strategy'")
 (defconst scala-indent:reluctant-strategy 2
-  "See 'scala-indent:run-on-strategy'")
+  "See `scala-indent:run-on-strategy'")
 (defconst scala-indent:keywords-only-strategy 3
   "A strategy used internally by indent engine")
 
@@ -484,7 +484,7 @@ special word found. Special words include 'yield', 'else',
 
 (defun scala-indent:goto-list-anchor (&optional point)
   "Moves back to the point whose column will be used to indent
-list rows at current point (or point `point'). Returns the new
+list rows at current point (or POINT). Returns the new
 point or nil if the point is not in a list element > 1."
   (let ((list-beg (scala-syntax:list-p point)))
     (when list-beg
@@ -601,8 +601,8 @@ keyword, or nil if not."
 
 (defun scala-indent:goto-block-anchor (&optional point)
   "Moves back to the point whose column will be used as the
-anchor for calculating block indent for current point (or point
-'point'). Returns point or (point-min) if not inside a block."
+anchor for calculating block indent for current point (or POINT).
+Returns point or (point-min) if not inside a block."
   (when-let ((block-beg (nth 1 (syntax-ppss
                                 (scala-lib:point-after (beginning-of-line))))))
     ;; Check if the opening paren is the first on the line, if so, it is the
@@ -650,75 +650,6 @@ anchor for calculating block indent for current point (or point
 ;;; Open parentheses
 ;;;
 
-(defun scala-indent:open-parentheses-line-p (&optional point)
-  "Returns the position of the first character of the line,
-if the current point (or point 'point') is on a line that starts
-with an opening parentheses, or nil if not."
-  (save-excursion
-    (when point (goto-char point))
-    (scala-syntax:beginning-of-code-line)
-    (if (looking-at "\\s(") (point) nil)))
-
-(defun scala-indent:goto-open-parentheses-anchor (&optional point)
-  "Moves back to the point whose column will be used as the
-anchor for calculating opening parenthesis indent for the current
-point (or point 'point'). Returns point or nil, if line does not
-start with opening parenthesis."
-  ;; There are five cases we need to consider:
-  ;; 1. curry parentheses, i.e. 2..n parentheses groups.
-  ;; 2. value body parentheses (follows '=').
-  ;; 3. parameters, etc on separate line (who would be so mad?)
-  ;; 4. non-value body parentheses (follows class, trait, new, def, etc).
-  (let ((parentheses-beg (scala-indent:open-parentheses-line-p point)))
-    (when parentheses-beg
-      (goto-char parentheses-beg)
-      (cond
-       ;; case 1
-       ((and scala-indent:align-parameters
-             (= (char-after) ?\()
-             (scala-indent:run-on-p)
-             (scala-syntax:looking-back-token ")" 1))
-        (scala-syntax:backward-parameter-groups)
-        (let ((curry-beg (point)))
-          (forward-char)
-          (forward-comment (buffer-size))
-          (if (= (line-number-at-pos curry-beg)
-                 (line-number-at-pos))
-              (goto-char curry-beg)
-            nil)))
-       ;; case 2
-       ((scala-syntax:looking-back-token "=" 1)
-        nil) ; let body rule handle it
-       ;; case 4
-       ((and (= (char-after) ?\{)
-             (scala-indent:goto-run-on-anchor
-              nil scala-indent:keywords-only-strategy)) ; use customized strategy
-        (point))
-       ;; case 3
-       ;;((scala-indent:run-on-p)
-       ;; (scala-syntax:skip-backward-ignorable)
-       ;; (back-to-indentation)
-       ;; (point))
-       (t
-        nil)
-       ))))
-
-(defun scala-indent:resolve-open-parentheses-step (start anchor)
-  "Resolves the appropriate indent step for an open paren
-anchored at 'anchor'."
-  (cond ((scala-syntax:looking-back-token ")")
-;         (message "curry")
-         0)
-        ((save-excursion
-           (goto-char anchor)
-           ;; find =
-           (scala-syntax:has-char-before ?= start))
-;         (message "=")
-         scala-indent:step)
-        (t
-;         (message "normal at %d" (current-column))
-         0)))
-
 (defun scala-indent:goto-line-comment-anchor (&optional point)
   "Goto and return the position relative to which a line comment
 will be indented. This will be the start of the line-comment on
@@ -762,19 +693,17 @@ nothing was applied."
           (scala-indent:apply-indent-rules (cdr rule-indents)))))))
 
 (defun scala-indent:calculate-indent-for-line (&optional point)
-  "Calculate the appropriate indent for the current point or the
-point 'point'. Returns the new column, or nil if the indent
-cannot be determined."
+  "Calculate the appropriate indent for the current point or POINT.
+
+Returns the new column, or nil if the indent cannot be determined."
   (or (scala-indent:apply-indent-rules
-       `((scala-indent:goto-line-comment-anchor 0)
-         (scala-indent:goto-open-parentheses-anchor scala-indent:resolve-open-parentheses-step)
-         (scala-indent:goto-for-enumerators-anchor scala-indent:resolve-list-step)
-         (scala-indent:goto-forms-align-anchor scala-indent:resolve-forms-align-step)
-         (scala-indent:goto-list-anchor scala-indent:resolve-list-step)
-         (scala-indent:goto-run-on-anchor scala-indent:resolve-run-on-step)
+       `(;;(scala-indent:goto-line-comment-anchor 0)
          (scala-indent:goto-block-anchor scala-indent:resolve-block-step)
-         ;; TODO how order-sensitive is this logic?
-         (scala-indent:goto-body-anchor scala-indent:resolve-body-step)
+         ;; (scala-indent:goto-for-enumerators-anchor scala-indent:resolve-list-step)
+         ;; (scala-indent:goto-forms-align-anchor scala-indent:resolve-forms-align-step)
+         ;; (scala-indent:goto-run-on-anchor scala-indent:resolve-run-on-step)
+         ;; (scala-indent:goto-body-anchor scala-indent:resolve-body-step)
+         ;; (scala-indent:goto-list-anchor scala-indent:resolve-list-step)
      )
        point)
       0))
diff --git a/scala-mode-syntax.el b/scala-mode-syntax.el
index 6e44365..b635b05 100644
--- a/scala-mode-syntax.el
+++ b/scala-mode-syntax.el
@@ -9,14 +9,13 @@
 ;;;; Scala syntax regular expressions
 ;;;;
 
-;;; Based on the Scala language specification 3.0.  Note: order is not
-;;; the same as in the document, as here things are declared before
-;;; used.
-
-;;; A note on naming. Things that end with `-re' are regular
-;;; expressions.  Things that end with `-group' are regular expression
-;;; character groups without the enclosing [], i.e. they are not
-;;; regular expressions, but can be used in declaring one.
+;;; Based on the Scala language specification 3.0. Note: order is not the same
+;;; as in the document, as here things are declared before used.
+
+;;; A note on naming. Things that end with `-re' are regular expressions. Things
+;;; that end with `-group' are regular expression character groups without the
+;;; enclosing [], i.e. they are not regular expressions, but can be used in
+;;; declaring one.
 
 (defun scala-syntax:alt (&rest res)
   (concat "\\(" (string-join res "\\|") "\\)"))

From 1859e83b750c898984f30795de6ac7bb43dcdeff Mon Sep 17 00:00:00 2001
From: Keith Pinson <keith.pinson@banno.com>
Date: Thu, 22 Jul 2021 16:24:25 -0400
Subject: [PATCH 13/42] Really go nuclear on the existing indenting algorithms

---
 scala-mode-indent.el | 234 +------------------------------------------
 scala-mode-syntax.el |   9 +-
 2 files changed, 10 insertions(+), 233 deletions(-)

diff --git a/scala-mode-indent.el b/scala-mode-indent.el
index 1975890..829e4af 100644
--- a/scala-mode-indent.el
+++ b/scala-mode-indent.el
@@ -221,9 +221,8 @@ effect of staying within lists if `scala-indent:align-parameters' is non-nil."
 ;;;
 
 (defconst scala-indent:mustNotTerminate-keywords-re
-  (regexp-opt '("extends" "forSome" "match" "with") 'words)
-  "Some keywords which occure only in the middle of an
-expression")
+  (regexp-opt '("extends" "match" "with") 'words)
+  "Some keywords which occur only in the middle of an expression")
 
 (defconst scala-indent:mustNotTerminate-line-beginning-re
   (concat "\\(" scala-indent:mustNotTerminate-keywords-re
@@ -360,36 +359,6 @@ is not on a run-on line."
   (concat (regexp-opt '("extends" "forSome") 'words)
           "\\|:\\("  scala-syntax:after-reserved-symbol-re "\\)"))
 
-(defun scala-indent:resolve-run-on-step (start &optional anchor)
-  "Resolves the appropriate indent step for run-on line at position
-'start'"
-  (save-excursion
-    (goto-char anchor)
-    (if (scala-syntax:looking-at-case-p)
-        ;; case run-on lines get double indent, except '|' which get
-        ;; special indents
-        (progn (goto-char start)
-               (- (* 2 scala-indent:step)
-                  (skip-chars-forward "|")))
-      (goto-char start)
-      (cond
-       ;; some keywords get double indent
-       ((or (looking-at scala-indent:double-indent-re)
-            (scala-syntax:looking-back-token scala-indent:double-indent-re))
-        (* 2 scala-indent:step))
-       ;; no indent if the previous line is just close parens
-       ;; ((save-excursion
-       ;;    (scala-syntax:skip-backward-ignorable)
-       ;;    (let ((end (point)))
-       ;;      (scala-syntax:beginning-of-code-line)
-       ;;      (skip-syntax-forward ")")
-       ;;      (= (point) end)))
-       ;;  0)
-       ;; else normal indent
-       (t (+ (if scala-indent:align-parameters 0
-               (scala-indent:value-expression-lead start anchor))
-             scala-indent:step))))))
-
 (defconst scala-indent:forms-align-re
   (regexp-opt '("yield" "then" "else" "catch" "finally") 'words))
 
@@ -405,96 +374,6 @@ nil. Also, the previous line must not be with '}'"
       (goto-char (match-beginning 0))
       (point))))
 
-(defun scala-indent:goto-forms-align-anchor (&optional point)
-  "Moves back to the point whose column will be used as the
-anchor relative to which indenting of special words on beginning
-of the line on which point (or point 'point') is, or nul if not
-special word found. Special words include 'yield', 'else',
-'catch' and 'finally'"
-  (let ((special-beg (scala-indent:forms-align-p point)))
-    (when special-beg
-      (goto-char special-beg)
-      (if (and (scala-syntax:looking-back-token "}")
-               (save-excursion
-                 (goto-char (match-beginning 0))
-                 (= (match-beginning 0) (scala-lib:point-after (scala-syntax:beginning-of-code-line)))))
-          (goto-char (match-beginning 0))
-        (let ((anchor
-               (cond ((looking-at "\\<yield\\>")
-                      ;; align with 'for'
-                      (if (scala-syntax:search-backward-sexp "\\<for\\>")
-                          (point)
-                        (message "matching 'for' not found")
-                        nil))
-                     ((looking-at "\\<then\\>")
-                      ;; align with 'if' or 'else if'
-                      (if (scala-syntax:search-backward-sexp "\\<if\\>")
-                          (if (scala-syntax:looking-back-token "\\<then\\>")
-                              (goto-char (match-beginning 0))
-                            (point))
-                        nil))
-                     ((looking-at "\\<else\\>")
-                      ;; align with 'if' or 'else if'
-                      (if (scala-syntax:search-backward-sexp "\\<if\\>")
-                          (if (scala-syntax:looking-back-token "\\<else\\>")
-                              (goto-char (match-beginning 0))
-                            (point))
-                        nil))
-                     ((looking-at "\\<catch\\>")
-                      ;; align with 'try'
-                      (if (scala-syntax:search-backward-sexp "\\<try\\>")
-                          (point)
-                        (message "matching 'try' not found")
-                        nil))
-                     ((looking-at "\\<finally\\>")
-                      ;; align with 'try'
-                      (if (scala-syntax:search-backward-sexp "\\<try\\>")
-                          (point)
-                        (message "matching 'try' not found")
-                        nil)))))
-          (if scala-indent:align-forms
-              anchor
-            (when anchor
-              (scala-indent:align-anchor)
-              (point))))))))
-
-(defun scala-indent:resolve-forms-align-step (start anchor)
-  (if scala-indent:align-forms
-      0
-    (scala-indent:value-expression-lead start anchor t)))
-
-;;;
-;;; Lists and enumerators
-;;;
-
-(defun scala-indent:goto-list-anchor-impl (point)
-  (goto-char point)
-  ;; find the first element of the list
-  (if (not scala-indent:align-parameters)
-      (progn (back-to-indentation) (point))
-    (forward-comment (buffer-size))
-    (if (= (line-number-at-pos point)
-           (line-number-at-pos))
-        (goto-char point)
-      (beginning-of-line))
-
-    ;; align list with first non-whitespace character
-    (skip-syntax-forward " ")
-    (point)))
-
-(defun scala-indent:goto-list-anchor (&optional point)
-  "Moves back to the point whose column will be used to indent
-list rows at current point (or POINT). Returns the new
-point or nil if the point is not in a list element > 1."
-  (let ((list-beg (scala-syntax:list-p point)))
-    (when list-beg
-      (scala-indent:goto-list-anchor-impl list-beg))))
-
-(defun scala-indent:resolve-list-step (start anchor)
-  (if scala-indent:align-parameters
-      0
-    (scala-indent:resolve-block-step start anchor)))
-
 (defun scala-indent:for-enumerators-p (&optional point)
   "Returns the point after opening parentheses if the current
 point (or point 'point') is in a block of enumerators. Return nil
@@ -513,88 +392,6 @@ if not in a list of enumerators or at the first enumerator."
             (when (< (point) point)
               (1+ (nth 1 state)))))))))
 
-;; TODO this does not work properly with indent-based syntax
-(defun scala-indent:goto-for-enumerators-anchor (&optional point)
-  "Moves back to the point whose column will be used to indent
-for enumerator at current point (or point 'point'). Returns the new
-point or nil if the point is not in a enumerator element > 1."
-  (let ((enumerators-beg (scala-indent:for-enumerators-p point)))
-    (when enumerators-beg
-      (scala-indent:goto-list-anchor-impl enumerators-beg))))
-
-;;;
-;;; Body
-;;;
-
-(defconst scala-indent:control-keywords-cond-re
-  ;; TODO `if' and `for' and possibly `while' no longer have to be followed by a
-  ;; parenthetical.
-  (regexp-opt '("for" "if" "while") 'words)
-  "All the flow control keywords that are followed by a
-condition (or generators in the case of 'for') in parentheses.")
-
-(defconst scala-indent:control-keywords-other-re
-  (regexp-opt '("for" "if" "then" "else" "do" "yield" "try" "finally" "catch") 'words)
-  "Other flow control keywords (not followed by parentheses)")
-
-(defconst scala-indent:control-keywords-re
-  (concat "\\(" scala-indent:control-keywords-cond-re
-          "\\|" scala-indent:control-keywords-other-re "\\)"))
-
-(defun scala-indent:body-p (&optional point)
-  "Returns the position of '=' symbol, or one of the
-`scala-indent:control-keywords-re' or
-`scala-indent:control-keywords-cond-re' keywords if current
-point (or point 'point) is on a line that follows said symbol or
-keyword, or nil if not."
-  ;; TODO does this need to be adjusted for whitespace syntax?
-  (save-excursion
-    (when point (goto-char point))
-    (scala-syntax:beginning-of-code-line)
-    (or (scala-syntax:looking-back-token scala-syntax:body-start-re 3)
-        (let ((case-fold-search nil))
-          ;; This handles cases like
-          ;;   for
-          ;;     x <- List(1, 2, 3)
-          (scala-syntax:looking-back-token scala-indent:control-keywords-other-re))
-        (progn
-          ;; if, else if
-          (when (scala-syntax:looking-back-token ")" 1)
-            (goto-char (match-end 0))
-            (backward-list))
-          (when (scala-syntax:looking-back-token scala-indent:control-keywords-other-re)
-            (goto-char (match-beginning 0))
-            (when (and (looking-at "\\<if\\>")
-                       (scala-syntax:looking-back-token "\\<else\\>"))
-              (goto-char (match-beginning 0)))
-            (unless scala-indent:align-forms
-              (scala-indent:align-anchor))
-            (point)))
-        (progn
-          ;; TODO this is crude and not quite right yet
-          (scala-syntax:beginning-of-definition)
-          (+ scala-indent:step (point))))))
-
-(defun scala-indent:goto-body-anchor (&optional point)
-  ;; TODO this does not work right in indentation syntax
-  (when-let ((declaration-end (scala-indent:body-p point)))
-    (goto-char declaration-end)
-    (if (let ((case-fold-search nil))
-          (looking-at scala-indent:control-keywords-re))
-        (point)
-      (when (scala-indent:backward-sexp-to-beginning-of-line)
-        (scala-indent:goto-run-on-anchor
-         nil
-         scala-indent:keywords-only-strategy))
-      (scala-indent:align-anchor)
-      (point))))
-
-(defun scala-indent:resolve-body-step (start &optional anchor)
-  (if (and (not (= start (point-max))) (or (= (char-after start) ?\{) (= (char-after start) ?:)))
-      0
-    (+ (scala-indent:value-expression-lead start anchor t)
-       scala-indent:step)))
-
 ;;;
 ;;; Block
 ;;;
@@ -646,23 +443,6 @@ Returns point or (point-min) if not inside a block."
      ;; normal block line
      (t  (+ scala-indent:step lead)))))
 
-;;;
-;;; Open parentheses
-;;;
-
-(defun scala-indent:goto-line-comment-anchor (&optional point)
-  "Goto and return the position relative to which a line comment
-will be indented. This will be the start of the line-comment on
-previous line, if any."
-  (let ((pos (point)))
-    (when (save-excursion
-            (when point (goto-char point))
-            (when (and (looking-at "\\s *//")
-                       (not (scala-syntax:looking-back-empty-line-p))
-                       (forward-comment -1))
-              (setq pos (point))))
-      (goto-char pos))))
-
 ;;;
 ;;; Indentation engine
 ;;;
@@ -697,14 +477,8 @@ nothing was applied."
 
 Returns the new column, or nil if the indent cannot be determined."
   (or (scala-indent:apply-indent-rules
-       `(;;(scala-indent:goto-line-comment-anchor 0)
-         (scala-indent:goto-block-anchor scala-indent:resolve-block-step)
-         ;; (scala-indent:goto-for-enumerators-anchor scala-indent:resolve-list-step)
-         ;; (scala-indent:goto-forms-align-anchor scala-indent:resolve-forms-align-step)
-         ;; (scala-indent:goto-run-on-anchor scala-indent:resolve-run-on-step)
-         ;; (scala-indent:goto-body-anchor scala-indent:resolve-body-step)
-         ;; (scala-indent:goto-list-anchor scala-indent:resolve-list-step)
-     )
+       `((scala-indent:goto-block-anchor scala-indent:resolve-block-step)
+         )
        point)
       0))
 
diff --git a/scala-mode-syntax.el b/scala-mode-syntax.el
index b635b05..e82d12e 100644
--- a/scala-mode-syntax.el
+++ b/scala-mode-syntax.el
@@ -170,11 +170,14 @@
           "\\|" scala-syntax:charEscapeSeq-re
           "\\|" scala-syntax:UnicodeEscape-re "\\)\\('\\)"))
 
+(defconst scala-syntax:string-escape-re
+  (concat scala-syntax:charEscapeSeq-re
+          "\\|" scala-syntax:UnicodeEscape-re))
+
 ;; String Literals
 (defconst scala-syntax:stringElement-re
   (concat "\\(" "[^\n\"\\\\]"
-          "\\|" scala-syntax:charEscapeSeq-re
-          "\\|" scala-syntax:UnicodeEscape-re  "\\)"))
+         "\\|" scala-syntax:string-escape-re  "\\)"))
 
 (defconst scala-syntax:oneLineStringLiteral-re
   (concat "\\(\"\\)" scala-syntax:stringElement-re "*\\(\"\\)"))
@@ -1020,7 +1023,7 @@ not. A list must be either enclosed in parentheses or start with
   '("private" "protected"))
 
 (defconst scala-syntax:modifiers-re
-  (regexp-opt (concat '("override" "opaque")
+  (regexp-opt (append '("override" "opaque")
                       scala-syntax:local-modifiers
                       scala-syntax:access-modifiers)
               'words))

From c11cbbd7c12605858ecebf75b6afa003842ee85e Mon Sep 17 00:00:00 2001
From: Keith Pinson <keith.pinson@banno.com>
Date: Mon, 4 Oct 2021 14:34:20 -0400
Subject: [PATCH 14/42] baby steps on a new approach

---
 scala-mode-indent.el | 69 ++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 64 insertions(+), 5 deletions(-)

diff --git a/scala-mode-indent.el b/scala-mode-indent.el
index 829e4af..34b5571 100644
--- a/scala-mode-indent.el
+++ b/scala-mode-indent.el
@@ -418,6 +418,63 @@ Returns point or (point-min) if not inside a block."
       (scala-indent:align-anchor)
       (point))))
 
+(defun scala-indent:analyze-syntax-stack (stack)
+  "TODO document"
+  (pcase stack
+    (`(val . ,_) 'decl)
+    (`(= ,_) 'decl-lhs)))
+
+(defun scala-indent:analyze-context (&optional point)
+  "TODO document"
+  (save-excursion
+    (when point (goto-char point))
+    (let (result stack)
+      ;; TODO probably want to bound this much more tightly than the beginning
+      ;; of the buffer
+      (while (and (not result) (> (point) 1))
+        (setq stack
+              (cons (sexp-at-point)
+                    stack))
+        (message (format "stack: %s" stack))
+        (setq result
+              (scala-indent:analyze-syntax-stack stack))
+        (message (format "result: %s" result))
+        (unless result
+          (scala-syntax:backward-sexp-forcing)))
+      (list result
+            (line-number-at-pos)
+            (current-indentation)
+            (point)))))
+
+(defun scala-indent:relative-indent-by-elem (syntax-elem)
+  "TODO document"
+  (pcase syntax-elem
+    ('decl-lhs 2)
+    ('(decl decl) 0)))
+
+(defun scala-indent:new-calculate-indent-for-line (&optional point)
+  "TODO document"
+  (let* ((line-no (line-number-at-pos))
+         (analysis (scala-indent:analyze-context point))
+         (syntax-elem (list (car analysis)))
+         (ctxt-line (cadr analysis))
+         (ctxt-indent (caddr analysis))
+         (stopped-point (cadddr analysis)))
+    (while (and (= ctxt-line line-no) (> line-no 1))
+      (setq analysis
+            (scala-indent:analyze-context
+             (save-excursion
+               (goto-char stopped-point)
+               (scala-syntax:backward-sexp-forcing)
+               (point))))
+      (setq syntax-elem (cons (car analysis) syntax-elem))
+      (setq ctxt-line (cadr analysis))
+      (setq ctxt-indent (caddr analysis))
+      (setq stopped-point (cadddr analysis)))
+    (when-let ((_ (< ctxt-line line-no))
+               (relative (scala-indent:relative-indent-by-elem syntax-elem)))
+      (+ ctxt-indent relative))))
+
 (defun scala-indent:resolve-block-step (start anchor)
   "Resolves the appropriate indent step for block line at position
 'start' relative to the block anchor 'anchor'."
@@ -476,11 +533,13 @@ nothing was applied."
   "Calculate the appropriate indent for the current point or POINT.
 
 Returns the new column, or nil if the indent cannot be determined."
-  (or (scala-indent:apply-indent-rules
-       `((scala-indent:goto-block-anchor scala-indent:resolve-block-step)
-         )
-       point)
-      0))
+  (or
+   (scala-indent:new-calculate-indent-for-line point)
+   (scala-indent:apply-indent-rules
+    `((scala-indent:goto-block-anchor scala-indent:resolve-block-step)
+      )
+    point)
+   0))
 
 (defun scala-indent:indent-line-to (column)
   "Indent the line to column and move cursor to the indent

From 738b88efea4275ba2c2ad291006919221587c05b Mon Sep 17 00:00:00 2001
From: Keith Pinson <keith.pinson@banno.com>
Date: Mon, 4 Oct 2021 15:00:53 -0400
Subject: [PATCH 15/42] primitive support for object blocks with whitespace

---
 scala-mode-indent.el | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/scala-mode-indent.el b/scala-mode-indent.el
index 34b5571..278bd4e 100644
--- a/scala-mode-indent.el
+++ b/scala-mode-indent.el
@@ -422,7 +422,8 @@ Returns point or (point-min) if not inside a block."
   "TODO document"
   (pcase stack
     (`(val . ,_) 'decl)
-    (`(= ,_) 'decl-lhs)))
+    (`(= ,_) 'decl-lhs)
+    (`(object ,_ : . ,_) 'block)))
 
 (defun scala-indent:analyze-context (&optional point)
   "TODO document"
@@ -435,10 +436,10 @@ Returns point or (point-min) if not inside a block."
         (setq stack
               (cons (sexp-at-point)
                     stack))
-        (message (format "stack: %s" stack))
+        ;(message (format "stack: %s" stack))
         (setq result
               (scala-indent:analyze-syntax-stack stack))
-        (message (format "result: %s" result))
+        ;(message (format "result: %s" result))
         (unless result
           (scala-syntax:backward-sexp-forcing)))
       (list result
@@ -449,7 +450,8 @@ Returns point or (point-min) if not inside a block."
 (defun scala-indent:relative-indent-by-elem (syntax-elem)
   "TODO document"
   (pcase syntax-elem
-    ('decl-lhs 2)
+    ('(decl-lhs) 2)
+    ('(block) 2)
     ('(decl decl) 0)))
 
 (defun scala-indent:new-calculate-indent-for-line (&optional point)

From b00f9331c3bbd9d9981e5910054f7ce3966b8198 Mon Sep 17 00:00:00 2001
From: Keith Pinson <keith.pinson@banno.com>
Date: Mon, 4 Oct 2021 15:06:44 -0400
Subject: [PATCH 16/42] support enum and more flexible object blocks

---
 scala-mode-indent.el | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scala-mode-indent.el b/scala-mode-indent.el
index 278bd4e..b7be8c3 100644
--- a/scala-mode-indent.el
+++ b/scala-mode-indent.el
@@ -423,7 +423,8 @@ Returns point or (point-min) if not inside a block."
   (pcase stack
     (`(val . ,_) 'decl)
     (`(= ,_) 'decl-lhs)
-    (`(object ,_ : . ,_) 'block)))
+    ((and `(enum . ,tail) (guard (memq ': tail))) 'block)
+    ((and `(object . ,tail) (guard (memq ': tail))) 'block)))
 
 (defun scala-indent:analyze-context (&optional point)
   "TODO document"

From 80d75fc5821165295ece2d3a4ef7df35d8acf777 Mon Sep 17 00:00:00 2001
From: Keith Pinson <keith.pinson@banno.com>
Date: Mon, 4 Oct 2021 15:19:42 -0400
Subject: [PATCH 17/42] initial support for indentation-based match-case
 expressions

---
 scala-mode-indent.el | 24 +++++++++++++++---------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/scala-mode-indent.el b/scala-mode-indent.el
index b7be8c3..4305470 100644
--- a/scala-mode-indent.el
+++ b/scala-mode-indent.el
@@ -418,11 +418,22 @@ Returns point or (point-min) if not inside a block."
       (scala-indent:align-anchor)
       (point))))
 
+(defun scala-indent:relative-indent-by-elem (syntax-elem)
+  "TODO document"
+  (pcase syntax-elem
+    (`(decl-lhs . ,_) 2)
+    ('(match case) 2)
+    ('(block) 2)
+    ('(case case) 0)
+    ('(decl decl) 0)))
+
 (defun scala-indent:analyze-syntax-stack (stack)
   "TODO document"
   (pcase stack
     (`(val . ,_) 'decl)
-    (`(= ,_) 'decl-lhs)
+    ('(match) 'match)
+    (`(case . ,_) 'case)
+    (`(= . ,_) 'decl-lhs)
     ((and `(enum . ,tail) (guard (memq ': tail))) 'block)
     ((and `(object . ,tail) (guard (memq ': tail))) 'block)))
 
@@ -430,9 +441,11 @@ Returns point or (point-min) if not inside a block."
   "TODO document"
   (save-excursion
     (when point (goto-char point))
+    (when (> (current-indentation) (current-column))
+      (scala-syntax:forward-token))
     (let (result stack)
       ;; TODO probably want to bound this much more tightly than the beginning
-      ;; of the buffer
+      ;; of the buffer. This means worse case performance could be bad.
       (while (and (not result) (> (point) 1))
         (setq stack
               (cons (sexp-at-point)
@@ -448,13 +461,6 @@ Returns point or (point-min) if not inside a block."
             (current-indentation)
             (point)))))
 
-(defun scala-indent:relative-indent-by-elem (syntax-elem)
-  "TODO document"
-  (pcase syntax-elem
-    ('(decl-lhs) 2)
-    ('(block) 2)
-    ('(decl decl) 0)))
-
 (defun scala-indent:new-calculate-indent-for-line (&optional point)
   "TODO document"
   (let* ((line-no (line-number-at-pos))

From 06494c3cb2ea8fe8bf9242526ac0d01f6701cb2e Mon Sep 17 00:00:00 2001
From: Keith Pinson <keith.pinson@banno.com>
Date: Mon, 4 Oct 2021 15:35:49 -0400
Subject: [PATCH 18/42] fix misbehavior on whitespace around `case`

---
 scala-mode-indent.el | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scala-mode-indent.el b/scala-mode-indent.el
index 4305470..cc7de31 100644
--- a/scala-mode-indent.el
+++ b/scala-mode-indent.el
@@ -443,6 +443,8 @@ Returns point or (point-min) if not inside a block."
     (when point (goto-char point))
     (when (> (current-indentation) (current-column))
       (scala-syntax:forward-token))
+    (when (looking-at-p " ")
+      (scala-syntax:backward-sexp-forcing))
     (let (result stack)
       ;; TODO probably want to bound this much more tightly than the beginning
       ;; of the buffer. This means worse case performance could be bad.

From ac22bd6eddd71e49a4b176934957a59a792bfef7 Mon Sep 17 00:00:00 2001
From: Keith Pinson <keith.pinson@banno.com>
Date: Mon, 4 Oct 2021 16:39:41 -0400
Subject: [PATCH 19/42] support many more syntactic scenarios

---
 scala-mode-indent.el | 88 +++++++++++++++++++++++++++++++-------------
 1 file changed, 62 insertions(+), 26 deletions(-)

diff --git a/scala-mode-indent.el b/scala-mode-indent.el
index cc7de31..656c706 100644
--- a/scala-mode-indent.el
+++ b/scala-mode-indent.el
@@ -418,46 +418,77 @@ Returns point or (point-min) if not inside a block."
       (scala-indent:align-anchor)
       (point))))
 
-(defun scala-indent:relative-indent-by-elem (syntax-elem)
-  "TODO document"
-  (pcase syntax-elem
-    (`(decl-lhs . ,_) 2)
-    ('(match case) 2)
-    ('(block) 2)
-    ('(case case) 0)
-    ('(decl decl) 0)))
-
 (defun scala-indent:analyze-syntax-stack (stack)
   "TODO document"
   (pcase stack
+    ('(final) 'decl)
+    ('(override) 'decl)
+    ('(class) 'decl)
+    ('(object) 'decl)
+    ('(implicit) 'decl)
+    ('(while) 'decl)
+    ('(enum) 'decl)
+    ('(trait) 'decl)
+    (`(def . ,_) 'decl)
     (`(val . ,_) 'decl)
+    (`(var . ,_) 'decl)
+    (`(given . ,_) 'decl)
     ('(match) 'match)
+    ('(with) 'block)
+    (`(do . ,_) 'block)
+    ((and `(class . ,tail) (guard (memq ': tail))) 'block)
     (`(case . ,_) 'case)
-    (`(= . ,_) 'decl-lhs)
+    (`(= ,_ . ,_) 'decl-lhs)
+    (`(=> . ,_) 'arrow-lhs)
     ((and `(enum . ,tail) (guard (memq ': tail))) 'block)
+    ((and `(trait . ,tail) (guard (memq ': tail))) 'block)
     ((and `(object . ,tail) (guard (memq ': tail))) 'block)))
 
+(defun scala-indent:relative-indent-by-elem (syntax-elem)
+  "TODO document"
+  (pcase syntax-elem
+    (`(dedented decl . ,_) 'maintain)
+    ('(dedented blank) 2)
+    ('(dedented) 2)
+    (`(decl-lhs decl . ,_) 0)
+    (`(decl-lhs . ,_) 2)
+    (`(match case . ,_) 2)
+    ('(arrow-lhs) 2)
+    ('(block) 2)
+    (`(block . ,_) 2)
+    ('(case case) 0) ;; e.g. in enums
+    (`(arrow-lhs case . ,_) 0) ;; within match
+    (`(decl decl . ,_) 2)))
+
 (defun scala-indent:analyze-context (&optional point)
   "TODO document"
   (save-excursion
-    (when point (goto-char point))
-    (when (> (current-indentation) (current-column))
-      (scala-syntax:forward-token))
-    (when (looking-at-p " ")
-      (scala-syntax:backward-sexp-forcing))
-    (let (result stack)
+    (let ((line-indent (current-indentation))
+          result
+          stack)
+      (when point (goto-char point))
+      ;; Always look at a token on the current for starters
+      (when (> line-indent (current-column))
+        (scala-syntax:forward-token))
+      (if (= (line-beginning-position) (line-end-position))
+          ;; Handle blank lines
+          (setq result 'blank)
+        ;; Avoid double-reading curent symbol
+        (beginning-of-thing 'sexp))
       ;; TODO probably want to bound this much more tightly than the beginning
       ;; of the buffer. This means worse case performance could be bad.
       (while (and (not result) (> (point) 1))
-        (setq stack
-              (cons (sexp-at-point)
-                    stack))
-        ;(message (format "stack: %s" stack))
-        (setq result
-              (scala-indent:analyze-syntax-stack stack))
-        ;(message (format "result: %s" result))
-        (unless result
-          (scala-syntax:backward-sexp-forcing)))
+        (if (< line-indent (current-indentation))
+            (setq result 'dedented)
+          (setq stack
+                (cons (sexp-at-point)
+                      stack))
+          ;(message (format "stack: %s" stack))
+          (setq result
+                (scala-indent:analyze-syntax-stack stack))
+          ;(message (format "result: %s" result))
+          (unless result
+            (scala-syntax:backward-sexp-forcing))))
       (list result
             (line-number-at-pos)
             (current-indentation)
@@ -471,6 +502,7 @@ Returns point or (point-min) if not inside a block."
          (ctxt-line (cadr analysis))
          (ctxt-indent (caddr analysis))
          (stopped-point (cadddr analysis)))
+    ;(message "analysis: %s" analysis)
     (while (and (= ctxt-line line-no) (> line-no 1))
       (setq analysis
             (scala-indent:analyze-context
@@ -478,13 +510,17 @@ Returns point or (point-min) if not inside a block."
                (goto-char stopped-point)
                (scala-syntax:backward-sexp-forcing)
                (point))))
+      ;(message "analysis: %s" analysis)
       (setq syntax-elem (cons (car analysis) syntax-elem))
       (setq ctxt-line (cadr analysis))
       (setq ctxt-indent (caddr analysis))
       (setq stopped-point (cadddr analysis)))
+    ;(message "syntax-elem: %s" syntax-elem)
     (when-let ((_ (< ctxt-line line-no))
                (relative (scala-indent:relative-indent-by-elem syntax-elem)))
-      (+ ctxt-indent relative))))
+      (if (eq 'maintain relative)
+          (current-indentation)
+        (+ ctxt-indent relative)))))
 
 (defun scala-indent:resolve-block-step (start anchor)
   "Resolves the appropriate indent step for block line at position

From 8218464219120b441dd3304255f06b93d759e6e2 Mon Sep 17 00:00:00 2001
From: Keith Pinson <keith.pinson@banno.com>
Date: Mon, 4 Oct 2021 17:02:56 -0400
Subject: [PATCH 20/42] initial support for for-comprehensions, which are
 problematic

---
 scala-mode-indent.el | 46 +++++++++++++++++++++++++++++---------------
 1 file changed, 31 insertions(+), 15 deletions(-)

diff --git a/scala-mode-indent.el b/scala-mode-indent.el
index 656c706..8619425 100644
--- a/scala-mode-indent.el
+++ b/scala-mode-indent.el
@@ -429,6 +429,10 @@ Returns point or (point-min) if not inside a block."
     ('(while) 'decl)
     ('(enum) 'decl)
     ('(trait) 'decl)
+    (`(for) 'for-comp)
+    (`(for . ,_) 'for-body)
+    (`(yield . ,_) 'yield-from-comp)
+    (`(<- . ,_) 'generator)
     (`(def . ,_) 'decl)
     (`(val . ,_) 'decl)
     (`(var . ,_) 'decl)
@@ -448,37 +452,48 @@ Returns point or (point-min) if not inside a block."
   "TODO document"
   (pcase syntax-elem
     (`(dedented decl . ,_) 'maintain)
-    ('(dedented blank) 2)
     ('(dedented) 2)
     (`(decl-lhs decl . ,_) 0)
+    (`(decl-lhs for-comp) 0)
+    (`(decl-lhs generator) 0) ;; TODO for comprehensions are problematic
+    ('(for-comp yield-from-comp) 0)
+    (`(for-body . ,_) 2)
     (`(decl-lhs . ,_) 2)
     (`(match case . ,_) 2)
     ('(arrow-lhs) 2)
     ('(block) 2)
+    ('(decl) 2)
+    (`(generator . ,_) 0)
     (`(block . ,_) 2)
     ('(case case) 0) ;; e.g. in enums
     (`(arrow-lhs case . ,_) 0) ;; within match
     (`(decl decl . ,_) 2)))
 
-(defun scala-indent:analyze-context (&optional point)
+(defun scala-indent:find-analysis-start (&optional point)
   "TODO document"
   (save-excursion
-    (let ((line-indent (current-indentation))
+    (when point (goto-char point))
+    ;; Always look at a token on the current for starters
+    (when (> (current-indentation) (current-column))
+      (scala-syntax:forward-token))
+    (if (= (line-beginning-position) (line-end-position))
+        ;; Handle blank lines
+        (scala-syntax:backward-sexp-forcing)
+      ;; Avoid double-reading curent symbol
+      (beginning-of-thing 'sexp))
+    (point)))
+
+(defun scala-indent:analyze-context (point)
+  "TODO document"
+  (save-excursion
+    (goto-char point)
+    (let ((orig-indent (current-indentation))
           result
           stack)
-      (when point (goto-char point))
-      ;; Always look at a token on the current for starters
-      (when (> line-indent (current-column))
-        (scala-syntax:forward-token))
-      (if (= (line-beginning-position) (line-end-position))
-          ;; Handle blank lines
-          (setq result 'blank)
-        ;; Avoid double-reading curent symbol
-        (beginning-of-thing 'sexp))
       ;; TODO probably want to bound this much more tightly than the beginning
       ;; of the buffer. This means worse case performance could be bad.
       (while (and (not result) (> (point) 1))
-        (if (< line-indent (current-indentation))
+        (if (< orig-indent (current-indentation))
             (setq result 'dedented)
           (setq stack
                 (cons (sexp-at-point)
@@ -496,7 +511,8 @@ Returns point or (point-min) if not inside a block."
 
 (defun scala-indent:new-calculate-indent-for-line (&optional point)
   "TODO document"
-  (let* ((line-no (line-number-at-pos))
+  (let* ((point (scala-indent:find-analysis-start point))
+         (line-no (line-number-at-pos point))
          (analysis (scala-indent:analyze-context point))
          (syntax-elem (list (car analysis)))
          (ctxt-line (cadr analysis))
@@ -581,11 +597,11 @@ nothing was applied."
 
 Returns the new column, or nil if the indent cannot be determined."
   (or
-   (scala-indent:new-calculate-indent-for-line point)
    (scala-indent:apply-indent-rules
     `((scala-indent:goto-block-anchor scala-indent:resolve-block-step)
       )
     point)
+   (scala-indent:new-calculate-indent-for-line point)
    0))
 
 (defun scala-indent:indent-line-to (column)

From 8eb91cd986dfda4f8e6c8c8f42f62f7aa5c48a6c Mon Sep 17 00:00:00 2001
From: Keith Pinson <keith.pinson@banno.com>
Date: Mon, 4 Oct 2021 17:33:23 -0400
Subject: [PATCH 21/42] initial support for dot-chaining, which is problematic

For both supporting this and for-comprehensions, and perhaps supporting other
things better, we need to generator newline tokens.
---
 scala-mode-indent.el | 31 ++++++++++++++++++++-----------
 1 file changed, 20 insertions(+), 11 deletions(-)

diff --git a/scala-mode-indent.el b/scala-mode-indent.el
index 8619425..995c4b5 100644
--- a/scala-mode-indent.el
+++ b/scala-mode-indent.el
@@ -421,6 +421,7 @@ Returns point or (point-min) if not inside a block."
 (defun scala-indent:analyze-syntax-stack (stack)
   "TODO document"
   (pcase stack
+    ('(?.) 'dot)
     ('(final) 'decl)
     ('(override) 'decl)
     ('(class) 'decl)
@@ -453,6 +454,8 @@ Returns point or (point-min) if not inside a block."
   (pcase syntax-elem
     (`(dedented decl . ,_) 'maintain)
     ('(dedented) 2)
+    ('(dedented dot) 2)
+    ('(decl-lhs dot) 2)
     (`(decl-lhs decl . ,_) 0)
     (`(decl-lhs for-comp) 0)
     (`(decl-lhs generator) 0) ;; TODO for comprehensions are problematic
@@ -479,30 +482,34 @@ Returns point or (point-min) if not inside a block."
     (if (= (line-beginning-position) (line-end-position))
         ;; Handle blank lines
         (scala-syntax:backward-sexp-forcing)
-      ;; Avoid double-reading curent symbol
-      (beginning-of-thing 'sexp))
-    (point)))
-
-(defun scala-indent:analyze-context (point)
+      ;; (beginning-of-thing 'sexp) gets confused by `.'
+      (unless (looking-at-p "\.")
+        ;; Avoid double-reading curent symbol
+        (beginning-of-thing 'sexp)))
+    (list (point)
+          (current-indentation))))
+
+(defun scala-indent:analyze-context (orig-indent point)
   "TODO document"
   (save-excursion
     (goto-char point)
-    (let ((orig-indent (current-indentation))
-          result
-          stack)
+    (let (result stack)
       ;; TODO probably want to bound this much more tightly than the beginning
       ;; of the buffer. This means worse case performance could be bad.
       (while (and (not result) (> (point) 1))
         (if (< orig-indent (current-indentation))
             (setq result 'dedented)
           (setq stack
-                (cons (sexp-at-point)
+                (cons (if (looking-at-p "\\.")
+                          ?.
+                        (sexp-at-point))
                       stack))
           ;(message (format "stack: %s" stack))
           (setq result
                 (scala-indent:analyze-syntax-stack stack))
           ;(message (format "result: %s" result))
           (unless result
+            (while (looking-at-p "\\.") (backward-char))
             (scala-syntax:backward-sexp-forcing))))
       (list result
             (line-number-at-pos)
@@ -511,9 +518,11 @@ Returns point or (point-min) if not inside a block."
 
 (defun scala-indent:new-calculate-indent-for-line (&optional point)
   "TODO document"
-  (let* ((point (scala-indent:find-analysis-start point))
+  (let* ((pointAndIndent (scala-indent:find-analysis-start point))
+         (point (car pointAndIndent))
+         (orig-indent (cadr pointAndIndent))
          (line-no (line-number-at-pos point))
-         (analysis (scala-indent:analyze-context point))
+         (analysis (scala-indent:analyze-context orig-indent point))
          (syntax-elem (list (car analysis)))
          (ctxt-line (cadr analysis))
          (ctxt-indent (caddr analysis))

From 0e77db6c75f33406682fd0e43ae5a8509cbda105 Mon Sep 17 00:00:00 2001
From: Keith Pinson <keith.pinson@banno.com>
Date: Mon, 4 Oct 2021 18:23:03 -0400
Subject: [PATCH 22/42] get for-comps working tolerably, and a bit of work on
 dot chaining

---
 scala-mode-indent.el | 69 +++++++++++++++++++++++++++++---------------
 1 file changed, 45 insertions(+), 24 deletions(-)

diff --git a/scala-mode-indent.el b/scala-mode-indent.el
index 995c4b5..4512e43 100644
--- a/scala-mode-indent.el
+++ b/scala-mode-indent.el
@@ -421,7 +421,7 @@ Returns point or (point-min) if not inside a block."
 (defun scala-indent:analyze-syntax-stack (stack)
   "TODO document"
   (pcase stack
-    ('(?.) 'dot)
+    ('(?\n ?.) 'dot-chain)
     ('(final) 'decl)
     ('(override) 'decl)
     ('(class) 'decl)
@@ -452,20 +452,23 @@ Returns point or (point-min) if not inside a block."
 (defun scala-indent:relative-indent-by-elem (syntax-elem)
   "TODO document"
   (pcase syntax-elem
-    (`(dedented decl . ,_) 'maintain)
+    (`(dedented decl . ,_) :maintain)
     ('(dedented) 2)
-    ('(dedented dot) 2)
-    ('(decl-lhs dot) 2)
+    ('(dedented dot-chain) 4)
+    ('(decl-lhs dot-chain) 4)
     (`(decl-lhs decl . ,_) 0)
     (`(decl-lhs for-comp) 0)
     (`(decl-lhs generator) 0) ;; TODO for comprehensions are problematic
     ('(for-comp yield-from-comp) 0)
+    ('(dedented yield-from-comp) :maintain)
+    ('(decl-lhs yield-from-comp) -2)
     (`(for-body . ,_) 2)
     (`(decl-lhs . ,_) 2)
     (`(match case . ,_) 2)
     ('(arrow-lhs) 2)
     ('(block) 2)
     ('(decl) 2)
+    ('(generator yield-from-comp) -2)
     (`(generator . ,_) 0)
     (`(block . ,_) 2)
     ('(case case) 0) ;; e.g. in enums
@@ -476,24 +479,30 @@ Returns point or (point-min) if not inside a block."
   "TODO document"
   (save-excursion
     (when point (goto-char point))
-    ;; Always look at a token on the current for starters
-    (when (> (current-indentation) (current-column))
-      (scala-syntax:forward-token))
-    (if (= (line-beginning-position) (line-end-position))
-        ;; Handle blank lines
-        (scala-syntax:backward-sexp-forcing)
-      ;; (beginning-of-thing 'sexp) gets confused by `.'
-      (unless (looking-at-p "\.")
-        ;; Avoid double-reading curent symbol
-        (beginning-of-thing 'sexp)))
-    (list (point)
-          (current-indentation))))
-
-(defun scala-indent:analyze-context (orig-indent point)
+    (let (stack)
+      ;; Always look at a token on the current for starters
+      (when (> (current-indentation) (current-column))
+        (scala-syntax:forward-token)
+        (setq stack (cons ?\n stack)))
+      (if (= (line-beginning-position) (line-end-position))
+          ;; Handle blank lines
+          (progn
+            (scala-syntax:backward-sexp-forcing)
+            (setq stack (cons ?\n stack)))
+        ;; (beginning-of-thing 'sexp) gets confused by `.'
+        (unless (looking-at-p "\\.")
+          ;; Avoid double-reading curent symbol
+          (beginning-of-thing 'sexp)))
+      (list (point)
+            (current-indentation)
+            stack))))
+
+(defun scala-indent:analyze-context (orig-indent point &optional init-stack)
   "TODO document"
   (save-excursion
     (goto-char point)
-    (let (result stack)
+    (let (result
+          (stack init-stack))
       ;; TODO probably want to bound this much more tightly than the beginning
       ;; of the buffer. This means worse case performance could be bad.
       (while (and (not result) (> (point) 1))
@@ -508,6 +517,16 @@ Returns point or (point-min) if not inside a block."
           (setq result
                 (scala-indent:analyze-syntax-stack stack))
           ;(message (format "result: %s" result))
+          (when (and (not result)
+                     (save-excursion (= (point)
+                                        (scala-syntax:beginning-of-code-line))))
+            (setq stack (cons ?\n stack))
+            ;(message (format "stack: %s" stack))
+            (setq result
+                  (scala-indent:analyze-syntax-stack stack))
+            ;(message (format "result: %s" result))
+            (when result
+              (scala-syntax:backward-sexp-forcing)))
           (unless result
             (while (looking-at-p "\\.") (backward-char))
             (scala-syntax:backward-sexp-forcing))))
@@ -518,11 +537,12 @@ Returns point or (point-min) if not inside a block."
 
 (defun scala-indent:new-calculate-indent-for-line (&optional point)
   "TODO document"
-  (let* ((pointAndIndent (scala-indent:find-analysis-start point))
-         (point (car pointAndIndent))
-         (orig-indent (cadr pointAndIndent))
+  (let* ((initResult (scala-indent:find-analysis-start point))
+         (point (car initResult))
+         (orig-indent (cadr initResult))
+         (stack (caddr initResult))
          (line-no (line-number-at-pos point))
-         (analysis (scala-indent:analyze-context orig-indent point))
+         (analysis (scala-indent:analyze-context orig-indent point stack))
          (syntax-elem (list (car analysis)))
          (ctxt-line (cadr analysis))
          (ctxt-indent (caddr analysis))
@@ -531,6 +551,7 @@ Returns point or (point-min) if not inside a block."
     (while (and (= ctxt-line line-no) (> line-no 1))
       (setq analysis
             (scala-indent:analyze-context
+             orig-indent
              (save-excursion
                (goto-char stopped-point)
                (scala-syntax:backward-sexp-forcing)
@@ -543,7 +564,7 @@ Returns point or (point-min) if not inside a block."
     ;(message "syntax-elem: %s" syntax-elem)
     (when-let ((_ (< ctxt-line line-no))
                (relative (scala-indent:relative-indent-by-elem syntax-elem)))
-      (if (eq 'maintain relative)
+      (if (eq :maintain relative)
           (current-indentation)
         (+ ctxt-indent relative)))))
 

From 0f3e7ea2d38c23593e929f5b5138258caadac38d Mon Sep 17 00:00:00 2001
From: Keith Pinson <keith.pinson@banno.com>
Date: Mon, 4 Oct 2021 20:02:55 -0400
Subject: [PATCH 23/42] get if-then-else working

---
 scala-mode-indent.el | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/scala-mode-indent.el b/scala-mode-indent.el
index 4512e43..9ebf66f 100644
--- a/scala-mode-indent.el
+++ b/scala-mode-indent.el
@@ -430,6 +430,9 @@ Returns point or (point-min) if not inside a block."
     ('(while) 'decl)
     ('(enum) 'decl)
     ('(trait) 'decl)
+    (`(if . ,_) 'if)
+    (`(then . ,_) 'then)
+    (`(else . ,_) 'else)
     (`(for) 'for-comp)
     (`(for . ,_) 'for-body)
     (`(yield . ,_) 'yield-from-comp)
@@ -438,7 +441,7 @@ Returns point or (point-min) if not inside a block."
     (`(val . ,_) 'decl)
     (`(var . ,_) 'decl)
     (`(given . ,_) 'decl)
-    ('(match) 'match)
+    (`(,_ match) 'match)
     ('(with) 'block)
     (`(do . ,_) 'block)
     ((and `(class . ,tail) (guard (memq ': tail))) 'block)
@@ -454,6 +457,8 @@ Returns point or (point-min) if not inside a block."
   (pcase syntax-elem
     (`(dedented decl . ,_) :maintain)
     ('(dedented) 2)
+    ('(dedented then) 0)
+    ('(dedented else) 0)
     ('(dedented dot-chain) 4)
     ('(decl-lhs dot-chain) 4)
     (`(decl-lhs decl . ,_) 0)
@@ -463,17 +468,20 @@ Returns point or (point-min) if not inside a block."
     ('(dedented yield-from-comp) :maintain)
     ('(decl-lhs yield-from-comp) -2)
     (`(for-body . ,_) 2)
-    (`(decl-lhs . ,_) 2)
+    ('(if then) 0)
+    ('(then else) 0)
+    (`(decl-lhs) 2)
+    (`(decl-lhs . ,_) 0)
     (`(match case . ,_) 2)
     ('(arrow-lhs) 2)
     ('(block) 2)
-    ('(decl) 2)
+    (`(decl . ,_) 2)
     ('(generator yield-from-comp) -2)
     (`(generator . ,_) 0)
     (`(block . ,_) 2)
     ('(case case) 0) ;; e.g. in enums
     (`(arrow-lhs case . ,_) 0) ;; within match
-    (`(decl decl . ,_) 2)))
+    ))
 
 (defun scala-indent:find-analysis-start (&optional point)
   "TODO document"

From c5b0f46c1f2c0294bfc8772b29504413e71f661e Mon Sep 17 00:00:00 2001
From: Keith Pinson <keith.pinson@banno.com>
Date: Tue, 5 Oct 2021 10:58:56 -0400
Subject: [PATCH 24/42] role back dedented; if-then-else edgecases; &c

---
 scala-mode-indent.el | 219 +++++++++++++++++++++++++++----------------
 1 file changed, 140 insertions(+), 79 deletions(-)

diff --git a/scala-mode-indent.el b/scala-mode-indent.el
index 9ebf66f..187e507 100644
--- a/scala-mode-indent.el
+++ b/scala-mode-indent.el
@@ -421,66 +421,135 @@ Returns point or (point-min) if not inside a block."
 (defun scala-indent:analyze-syntax-stack (stack)
   "TODO document"
   (pcase stack
+    ;; <dot chaining>
     ('(?\n ?.) 'dot-chain)
+    ;; =
+    (`(= ?\n . ,_) 'decl-lhs)
+    ((and `(= ,_ . ,tail) (guard (memq ?\n tail))) 'after-decl)
+    (`(= ,_ . ,_) 'decl-lhs)
+    ;; =>
+    (`(=> ?\n . ,_) 'arrow-lhs)
+    ((and `(=> ,_ . ,tail) (guard (memq ?\n tail))) 'after-arrow)
+    (`(=> ,_ . ,_) 'arrow-lhs)
+    ;; <-
+    (`(<- . ,_) 'generator)
+    ;; case
+    (`(case . ,_) 'case)
+    ;; class
+    ((and `(class . ,tail) (guard (memq ': tail))) 'block)
+    (`(class . ,_) 'decl)
+    ;; def
+    (`(def . ,_) 'decl)
+    ;; do
+    (`(do ,_ . ,_) 'block)
+    ;; else
+    (`(else ?\n . ,_) 'else-conseq)
+    ('(else) 'else)
+    (`(else . ,_) 'else-inline)
+    ;; enum
+    ((and `(enum . ,tail) (guard (memq ': tail))) 'block)
+    (`(enum . ,_) 'decl)
+    ;; final
     ('(final) 'decl)
-    ('(override) 'decl)
-    ('(class) 'decl)
-    ('(object) 'decl)
-    ('(implicit) 'decl)
-    ('(while) 'decl)
-    ('(enum) 'decl)
-    ('(trait) 'decl)
-    (`(if . ,_) 'if)
-    (`(then . ,_) 'then)
-    (`(else . ,_) 'else)
+    ;; for
     (`(for) 'for-comp)
     (`(for . ,_) 'for-body)
-    (`(yield . ,_) 'yield-from-comp)
-    (`(<- . ,_) 'generator)
-    (`(def . ,_) 'decl)
-    (`(val . ,_) 'decl)
-    (`(var . ,_) 'decl)
+    ;; given
     (`(given . ,_) 'decl)
+    ;; if
+    (`(if ?\n . ,_) 'if-cond)
+    (`(if . ,_) 'if)
+    ;; implicit
+    ('(implicit) 'decl)
+    ;; import
+    ((and `(import . ,tail) (guard (memq ?\n tail))) 'after-decl)
+    (`(import . ,_) 'decl)
+    ;; match
     (`(,_ match) 'match)
-    ('(with) 'block)
-    (`(do . ,_) 'block)
-    ((and `(class . ,tail) (guard (memq ': tail))) 'block)
-    (`(case . ,_) 'case)
-    (`(= ,_ . ,_) 'decl-lhs)
-    (`(=> . ,_) 'arrow-lhs)
-    ((and `(enum . ,tail) (guard (memq ': tail))) 'block)
+    ;; object
+    ((and `(object . ,tail) (guard (memq ': tail))) 'block)
+    (`(object . ,_) 'decl)
+    ;; override
+    ('(override) 'decl)
+    ;; package
+    (`(package . ,_) 'decl)
+    ;; then
+    (`(then ?\n . ,_) 'then-conseq)
+    ('(then) 'then)
+    (`(then . ,_) 'then-inline)
+    ;; trait
     ((and `(trait . ,tail) (guard (memq ': tail))) 'block)
-    ((and `(object . ,tail) (guard (memq ': tail))) 'block)))
+    (`(trait . ,_) 'decl)
+    ;; val
+    (`(val . ,_) 'decl)
+    ;; var
+    (`(var . ,_) 'decl)
+    ;; while
+    ('(while) 'decl)
+    ;; with
+    ('(with) 'block)
+    ;; yield
+    (`(yield . ,_) 'yield-from-comp)
+    ))
 
 (defun scala-indent:relative-indent-by-elem (syntax-elem)
   "TODO document"
   (pcase syntax-elem
-    (`(dedented decl . ,_) :maintain)
-    ('(dedented) 2)
-    ('(dedented then) 0)
-    ('(dedented else) 0)
-    ('(dedented dot-chain) 4)
-    ('(decl-lhs dot-chain) 4)
+    ;; after-decl
+    (`(after-decl else) -2)
+    (`(after-decl) 0)
+    ;; arrow-lhs
+    (`(arrow-lhs) 2)
+    (`(arrow-lhs case . ,_) 0) ;; within match
+    (`(arrow-lhs . ,_) :maintain)
+    ;; block
+    (`(block) 2)
+    (`(block . ,_) 2)
+    ;; case
+    (`(case case) 0) ;; e.g. in enums
+    ;; decl
+    (`(decl decl) 0)
+    (`(decl . ,_) 2)
+    ;; decl-lhs
     (`(decl-lhs decl . ,_) 0)
+    (`(decl-lhs dot-chain) 4)
     (`(decl-lhs for-comp) 0)
-    (`(decl-lhs generator) 0) ;; TODO for comprehensions are problematic
-    ('(for-comp yield-from-comp) 0)
-    ('(dedented yield-from-comp) :maintain)
-    ('(decl-lhs yield-from-comp) -2)
-    (`(for-body . ,_) 2)
-    ('(if then) 0)
-    ('(then else) 0)
+    (`(decl-lhs generator) 0)
+    (`(decl-lhs yield-from-comp) -2)
     (`(decl-lhs) 2)
     (`(decl-lhs . ,_) 0)
-    (`(match case . ,_) 2)
-    ('(arrow-lhs) 2)
-    ('(block) 2)
-    (`(decl . ,_) 2)
-    ('(generator yield-from-comp) -2)
+    ;; else
+    (`(else decl) 2)
+    ;; else-conseq
+    (`(else-conseq) 2)
+    (`(else-conseq . ,_) :maintain)
+    ;; else-inline
+    (`(else-inline . ,_) 0)
+    ;; for-body
+    (`(for-body . ,_) 2)
+    ;; for-comp
+    (`(for-comp yield-from-comp) 0)
+    ;; generator
+    (`(generator yield-from-comp) -2)
     (`(generator . ,_) 0)
-    (`(block . ,_) 2)
-    ('(case case) 0) ;; e.g. in enums
-    (`(arrow-lhs case . ,_) 0) ;; within match
+    ;; if
+    (`(if then) 0)
+    (`(if then-inline) 0)
+    ;; if-cond
+    (`(if-cond then) 0)
+    (`(if-cond) 2)
+    ;; match
+    (`(match case . ,_) 2)
+    ;; then
+    (`(then decl) 2)
+    (`(then else) 0)
+    (`(then else-inline) 0)
+    ;; then-conseq
+    (`(then-conseq else) 0)
+    (`(then-conseq) 2)
+    ;; then-inline
+    (`(then-inline else) 0)
+    (`(then-inline else-inline) 0)
     ))
 
 (defun scala-indent:find-analysis-start (&optional point)
@@ -490,8 +559,7 @@ Returns point or (point-min) if not inside a block."
     (let (stack)
       ;; Always look at a token on the current for starters
       (when (> (current-indentation) (current-column))
-        (scala-syntax:forward-token)
-        (setq stack (cons ?\n stack)))
+        (scala-syntax:forward-token))
       (if (= (line-beginning-position) (line-end-position))
           ;; Handle blank lines
           (progn
@@ -502,42 +570,37 @@ Returns point or (point-min) if not inside a block."
           ;; Avoid double-reading curent symbol
           (beginning-of-thing 'sexp)))
       (list (point)
-            (current-indentation)
             stack))))
 
-(defun scala-indent:analyze-context (orig-indent point &optional init-stack)
+(defun scala-indent:analyze-context (point &optional init-stack)
   "TODO document"
   (save-excursion
     (goto-char point)
     (let (result
           (stack init-stack))
-      ;; TODO probably want to bound this much more tightly than the beginning
-      ;; of the buffer. This means worse case performance could be bad.
       (while (and (not result) (> (point) 1))
-        (if (< orig-indent (current-indentation))
-            (setq result 'dedented)
-          (setq stack
-                (cons (if (looking-at-p "\\.")
-                          ?.
-                        (sexp-at-point))
-                      stack))
-          ;(message (format "stack: %s" stack))
+        (setq stack
+              (cons (if (looking-at-p "\\.")
+                        ?.
+                      (sexp-at-point))
+                    stack))
+        (message (format "stack: %s" stack))
+        (setq result
+              (scala-indent:analyze-syntax-stack stack))
+        (message (format "result: %s" result))
+        (when (and (not result)
+                   (save-excursion (= (point)
+                                      (scala-syntax:beginning-of-code-line))))
+          (setq stack (cons ?\n stack))
+          (message (format "stack: %s" stack))
           (setq result
                 (scala-indent:analyze-syntax-stack stack))
-          ;(message (format "result: %s" result))
-          (when (and (not result)
-                     (save-excursion (= (point)
-                                        (scala-syntax:beginning-of-code-line))))
-            (setq stack (cons ?\n stack))
-            ;(message (format "stack: %s" stack))
-            (setq result
-                  (scala-indent:analyze-syntax-stack stack))
-            ;(message (format "result: %s" result))
-            (when result
-              (scala-syntax:backward-sexp-forcing)))
-          (unless result
-            (while (looking-at-p "\\.") (backward-char))
-            (scala-syntax:backward-sexp-forcing))))
+          (message (format "result: %s" result))
+          (when result
+            (scala-syntax:backward-sexp-forcing)))
+        (unless result
+          (while (looking-at-p "\\.") (backward-char))
+          (scala-syntax:backward-sexp-forcing)))
       (list result
             (line-number-at-pos)
             (current-indentation)
@@ -547,29 +610,27 @@ Returns point or (point-min) if not inside a block."
   "TODO document"
   (let* ((initResult (scala-indent:find-analysis-start point))
          (point (car initResult))
-         (orig-indent (cadr initResult))
-         (stack (caddr initResult))
+         (stack (cadr initResult))
          (line-no (line-number-at-pos point))
-         (analysis (scala-indent:analyze-context orig-indent point stack))
+         (analysis (scala-indent:analyze-context point stack))
          (syntax-elem (list (car analysis)))
          (ctxt-line (cadr analysis))
          (ctxt-indent (caddr analysis))
          (stopped-point (cadddr analysis)))
-    ;(message "analysis: %s" analysis)
+    (message "analysis: %s" analysis)
     (while (and (= ctxt-line line-no) (> line-no 1))
       (setq analysis
             (scala-indent:analyze-context
-             orig-indent
              (save-excursion
                (goto-char stopped-point)
                (scala-syntax:backward-sexp-forcing)
                (point))))
-      ;(message "analysis: %s" analysis)
+      (message "analysis: %s" analysis)
       (setq syntax-elem (cons (car analysis) syntax-elem))
       (setq ctxt-line (cadr analysis))
       (setq ctxt-indent (caddr analysis))
       (setq stopped-point (cadddr analysis)))
-    ;(message "syntax-elem: %s" syntax-elem)
+    (message "syntax-elem: %s" syntax-elem)
     (when-let ((_ (< ctxt-line line-no))
                (relative (scala-indent:relative-indent-by-elem syntax-elem)))
       (if (eq :maintain relative)

From 11d5c114c9c001c72e480511e98a594b65da3572 Mon Sep 17 00:00:00 2001
From: Keith Pinson <keith.pinson@banno.com>
Date: Tue, 5 Oct 2021 12:18:41 -0400
Subject: [PATCH 25/42] hey, this is starting to work pretty well

---
 scala-mode-indent.el | 73 ++++++++++++++++++++++++++++++++++----------
 1 file changed, 57 insertions(+), 16 deletions(-)

diff --git a/scala-mode-indent.el b/scala-mode-indent.el
index 187e507..0102f72 100644
--- a/scala-mode-indent.el
+++ b/scala-mode-indent.el
@@ -439,6 +439,8 @@ Returns point or (point-min) if not inside a block."
     ((and `(class . ,tail) (guard (memq ': tail))) 'block)
     (`(class . ,_) 'decl)
     ;; def
+    ((and `(def . ,tail) (guard (memq '= tail)))
+     (if (memq ?\n tail) 'after-decl 'block))
     (`(def . ,_) 'decl)
     ;; do
     (`(do ,_ . ,_) 'block)
@@ -481,6 +483,8 @@ Returns point or (point-min) if not inside a block."
     ((and `(trait . ,tail) (guard (memq ': tail))) 'block)
     (`(trait . ,_) 'decl)
     ;; val
+    ((and `(val . ,tail) (guard (memq '= tail)))
+     (if (memq ?\n tail) 'after-decl 'block))
     (`(val . ,_) 'decl)
     ;; var
     (`(var . ,_) 'decl)
@@ -506,9 +510,12 @@ Returns point or (point-min) if not inside a block."
     (`(block) 2)
     (`(block . ,_) 2)
     ;; case
+    (`(case) :maintain)
     (`(case case) 0) ;; e.g. in enums
+    (`(case ,_) 2) ;; e.g. in enums
     ;; decl
     (`(decl decl) 0)
+    (`(decl else) -2)
     (`(decl . ,_) 2)
     ;; decl-lhs
     (`(decl-lhs decl . ,_) 0)
@@ -519,7 +526,7 @@ Returns point or (point-min) if not inside a block."
     (`(decl-lhs) 2)
     (`(decl-lhs . ,_) 0)
     ;; else
-    (`(else decl) 2)
+    (`(else ,_) 2)
     ;; else-conseq
     (`(else-conseq) 2)
     (`(else-conseq . ,_) :maintain)
@@ -535,21 +542,28 @@ Returns point or (point-min) if not inside a block."
     ;; if
     (`(if then) 0)
     (`(if then-inline) 0)
+    (`(if . ,_) 2)
     ;; if-cond
     (`(if-cond then) 0)
     (`(if-cond) 2)
     ;; match
     (`(match case . ,_) 2)
     ;; then
-    (`(then decl) 2)
     (`(then else) 0)
     (`(then else-inline) 0)
+    (`(then ,_) 2)
     ;; then-conseq
     (`(then-conseq else) 0)
     (`(then-conseq) 2)
+    (`(then-conseq ,_) 2)
     ;; then-inline
     (`(then-inline else) 0)
     (`(then-inline else-inline) 0)
+    ;; yield-from-comp
+    (`(yield-from-comp) 0)
+    ;; <fallbacks>
+    (`(,_ then) -2)
+    (`(,_ else) -2)
     ))
 
 (defun scala-indent:find-analysis-start (&optional point)
@@ -584,18 +598,18 @@ Returns point or (point-min) if not inside a block."
                         ?.
                       (sexp-at-point))
                     stack))
-        (message (format "stack: %s" stack))
+        ;(message (format "stack: %s" stack))
         (setq result
               (scala-indent:analyze-syntax-stack stack))
-        (message (format "result: %s" result))
+        ;(message (format "result: %s" result))
         (when (and (not result)
                    (save-excursion (= (point)
                                       (scala-syntax:beginning-of-code-line))))
           (setq stack (cons ?\n stack))
-          (message (format "stack: %s" stack))
+          ;(message (format "stack: %s" stack))
           (setq result
                 (scala-indent:analyze-syntax-stack stack))
-          (message (format "result: %s" result))
+          ;(message (format "result: %s" result))
           (when result
             (scala-syntax:backward-sexp-forcing)))
         (unless result
@@ -604,7 +618,20 @@ Returns point or (point-min) if not inside a block."
       (list result
             (line-number-at-pos)
             (current-indentation)
-            (point)))))
+            (point)
+            stack))))
+
+(defun scala-indent:need-more-context (syntax-elem stopped-point)
+  (and
+   (consp syntax-elem)
+   ;; read a full statement
+   (pcase (car syntax-elem)
+     ('after-decl t)
+     ('after-arrow t))
+   (save-excursion
+     (goto-char stopped-point)
+     ;; but that statement took up less than a line
+     (> (current-column) (current-indentation)))))
 
 (defun scala-indent:new-calculate-indent-for-line (&optional point)
   "TODO document"
@@ -613,24 +640,38 @@ Returns point or (point-min) if not inside a block."
          (stack (cadr initResult))
          (line-no (line-number-at-pos point))
          (analysis (scala-indent:analyze-context point stack))
-         (syntax-elem (list (car analysis)))
-         (ctxt-line (cadr analysis))
-         (ctxt-indent (caddr analysis))
-         (stopped-point (cadddr analysis)))
-    (message "analysis: %s" analysis)
-    (while (and (= ctxt-line line-no) (> line-no 1))
+         (syntax-elem (list (nth 0 analysis)))
+         (ctxt-line (nth 1 analysis))
+         (ctxt-indent (nth 2 analysis))
+         (stopped-point (nth 3 analysis))
+         (end-stack (nth 4 analysis))
+         )
+    ;(message "analysis: %s" analysis)
+    (while (or (and (= ctxt-line line-no) (> line-no 1)
+                    ;; If we keep reading for this reason, we've accepted the
+                    ;; existing tokens and so need to clear the stack
+                    (or (setq stack nil)
+                        t))
+               (and (scala-indent:need-more-context syntax-elem stopped-point)
+                    ;; If we read a full statement that was only part of a line,
+                    ;; drop it and try again for more context
+                    (or (setq syntax-elem (cdr syntax-elem))
+                        ;; restart with the existing stack
+                        (setq stack end-stack)
+                        t)))
       (setq analysis
             (scala-indent:analyze-context
              (save-excursion
                (goto-char stopped-point)
                (scala-syntax:backward-sexp-forcing)
-               (point))))
-      (message "analysis: %s" analysis)
+               (point))
+             stack))
+      ;(message "analysis: %s" analysis)
       (setq syntax-elem (cons (car analysis) syntax-elem))
       (setq ctxt-line (cadr analysis))
       (setq ctxt-indent (caddr analysis))
       (setq stopped-point (cadddr analysis)))
-    (message "syntax-elem: %s" syntax-elem)
+    ;(message "syntax-elem: %s" syntax-elem)
     (when-let ((_ (< ctxt-line line-no))
                (relative (scala-indent:relative-indent-by-elem syntax-elem)))
       (if (eq :maintain relative)

From f2a0fbd3d884e370529b4d1f7f86b89201020fbe Mon Sep 17 00:00:00 2001
From: Keith Pinson <keith.pinson@banno.com>
Date: Thu, 23 Dec 2021 16:33:43 -0500
Subject: [PATCH 26/42] Get dot-chaining much more stable; but other things
 unstable

---
 scala-mode-indent.el | 55 ++++++++++++++++++++++++++++++++------------
 1 file changed, 40 insertions(+), 15 deletions(-)

diff --git a/scala-mode-indent.el b/scala-mode-indent.el
index 0102f72..1c5b877 100644
--- a/scala-mode-indent.el
+++ b/scala-mode-indent.el
@@ -419,10 +419,11 @@ Returns point or (point-min) if not inside a block."
       (point))))
 
 (defun scala-indent:analyze-syntax-stack (stack)
-  "TODO document"
+  "A kind of tokenize step of the hand-wavy parse"
   (pcase stack
     ;; <dot chaining>
     ('(?\n ?.) 'dot-chain)
+    (`(?\n ?. . ,_) 'dot-chain)
     ;; =
     (`(= ?\n . ,_) 'decl-lhs)
     ((and `(= ,_ . ,tail) (guard (memq ?\n tail))) 'after-decl)
@@ -512,7 +513,7 @@ Returns point or (point-min) if not inside a block."
     ;; case
     (`(case) :maintain)
     (`(case case) 0) ;; e.g. in enums
-    (`(case ,_) 2) ;; e.g. in enums
+    (`(case ,_) 2)
     ;; decl
     (`(decl decl) 0)
     (`(decl else) -2)
@@ -520,6 +521,7 @@ Returns point or (point-min) if not inside a block."
     ;; decl-lhs
     (`(decl-lhs decl . ,_) 0)
     (`(decl-lhs dot-chain) 4)
+    (`(dot-chain dot-chain) 0)
     (`(decl-lhs for-comp) 0)
     (`(decl-lhs generator) 0)
     (`(decl-lhs yield-from-comp) -2)
@@ -567,7 +569,7 @@ Returns point or (point-min) if not inside a block."
     ))
 
 (defun scala-indent:find-analysis-start (&optional point)
-  "TODO document"
+  "Find a place to start tokenizing in a consistent manner"
   (save-excursion
     (when point (goto-char point))
     (let (stack)
@@ -594,11 +596,19 @@ Returns point or (point-min) if not inside a block."
           (stack init-stack))
       (while (and (not result) (> (point) 1))
         (setq stack
-              (cons (if (looking-at-p "\\.")
-                        ?.
-                      (sexp-at-point))
-                    stack))
         ;(message (format "stack: %s" stack))
+              (if (looking-at-p "\\.")
+                  (cons ?. stack)
+                (let ((s (sexp-at-point)))
+                  (backward-char)
+                  (if (looking-at-p "\\.")
+                      ;; Try hard to notice dot-chaining
+                      (cons ?. (cons s stack))
+                    (if (looking-at-p "\"")
+                        ;; A little hack in case we are inside of a string
+                        stack
+                      (forward-char)
+                      (cons s stack))))))
         (setq result
               (scala-indent:analyze-syntax-stack stack))
         ;(message (format "result: %s" result))
@@ -621,7 +631,7 @@ Returns point or (point-min) if not inside a block."
             (point)
             stack))))
 
-(defun scala-indent:need-more-context (syntax-elem stopped-point)
+(defun scala-indent:full-stmt-less-than-line (syntax-elem stopped-point)
   (and
    (consp syntax-elem)
    ;; read a full statement
@@ -652,13 +662,18 @@ Returns point or (point-min) if not inside a block."
                     ;; existing tokens and so need to clear the stack
                     (or (setq stack nil)
                         t))
-               (and (scala-indent:need-more-context syntax-elem stopped-point)
+               (and (scala-indent:full-stmt-less-than-line syntax-elem stopped-point)
                     ;; If we read a full statement that was only part of a line,
                     ;; drop it and try again for more context
                     (or (setq syntax-elem (cdr syntax-elem))
                         ;; restart with the existing stack
                         (setq stack end-stack)
-                        t)))
+                        t))
+               ;; We know we have a dot-chain, but we need to get more context
+               ;; to know how to position it
+               (when (equal syntax-elem '(dot-chain))
+                 (setq stack end-stack)
+                 t))
       (setq analysis
             (scala-indent:analyze-context
              (save-excursion
@@ -667,16 +682,26 @@ Returns point or (point-min) if not inside a block."
                (point))
              stack))
       ;(message "analysis: %s" analysis)
-      (setq syntax-elem (cons (car analysis) syntax-elem))
-      (setq ctxt-line (cadr analysis))
-      (setq ctxt-indent (caddr analysis))
-      (setq stopped-point (cadddr analysis)))
     ;(message "syntax-elem: %s" syntax-elem)
+      (setq syntax-elem (cons (nth 0 analysis) syntax-elem))
+      (setq ctxt-line (nth 1 analysis))
+      (setq ctxt-indent (nth 2 analysis))
+      (setq stopped-point (nth 3 analysis))
+      (setq end-stack (nth 4 analysis)))
     (when-let ((_ (< ctxt-line line-no))
                (relative (scala-indent:relative-indent-by-elem syntax-elem)))
       (if (eq :maintain relative)
           (current-indentation)
-        (+ ctxt-indent relative)))))
+        (+ (if (eq ?\n (car end-stack))
+               ;; Oops, moved a bit too far back while determining context.
+               ;; Don't really want to determine our indentation based on the
+               ;; line whose newline we are looking at, but rather the next one.
+               (save-excursion
+                 (goto-char stopped-point)
+                 (forward-line)
+                 (current-indentation))
+             ctxt-indent)
+           relative)))))
 
 (defun scala-indent:resolve-block-step (start anchor)
   "Resolves the appropriate indent step for block line at position

From b7ea24ae7f79b1b80ce999a00669c6b6756f6f97 Mon Sep 17 00:00:00 2001
From: Keith Pinson <keith.pinson@banno.com>
Date: Thu, 23 Dec 2021 17:56:14 -0500
Subject: [PATCH 27/42] Run both indentation algorithms; choose result by
 simple heuristic

After a little testing this seems to have massively improved the cooperation of
these algorithms and therefore the stability of the overall result. Am I kidding
myself or am I closing in on something here?
---
 scala-mode-indent.el | 134 ++++++++++++++++++++++++++-----------------
 1 file changed, 80 insertions(+), 54 deletions(-)

diff --git a/scala-mode-indent.el b/scala-mode-indent.el
index 1c5b877..cf2f423 100644
--- a/scala-mode-indent.el
+++ b/scala-mode-indent.el
@@ -281,8 +281,8 @@ and are in fact a sign of run-on. Reserved-symbols not included.")
        ((looking-at scala-indent:mustNotContinue-re)
         nil)
        ;; NO: this line is the start of value body
-       ((scala-indent:body-p)
-        nil)
+       ;; ((scala-indent:body-p) ;; TODO did I delete this function when I shouldn't have?
+       ;;  nil)
        ;; YES: eager strategy can stop here, everything is a run-on if no
        ;; counter evidence
        ((= strategy scala-indent:eager-strategy)
@@ -506,6 +506,7 @@ Returns point or (point-min) if not inside a block."
     ;; arrow-lhs
     (`(arrow-lhs) 2)
     (`(arrow-lhs case . ,_) 0) ;; within match
+    (`(arrow-lhs dot-chain) 4)
     (`(arrow-lhs . ,_) :maintain)
     ;; block
     (`(block) 2)
@@ -593,10 +594,10 @@ Returns point or (point-min) if not inside a block."
   (save-excursion
     (goto-char point)
     (let (result
-          (stack init-stack))
+          (stack init-stack)
+          last-indentation)
       (while (and (not result) (> (point) 1))
         (setq stack
-        ;(message (format "stack: %s" stack))
               (if (looking-at-p "\\.")
                   (cons ?. stack)
                 (let ((s (sexp-at-point)))
@@ -621,13 +622,16 @@ Returns point or (point-min) if not inside a block."
                 (scala-indent:analyze-syntax-stack stack))
           ;(message (format "result: %s" result))
           (when result
+            (setq last-indentation (current-indentation))
             (scala-syntax:backward-sexp-forcing)))
         (unless result
+          (setq last-indentation (current-indentation))
           (while (looking-at-p "\\.") (backward-char))
           (scala-syntax:backward-sexp-forcing)))
       (list result
             (line-number-at-pos)
             (current-indentation)
+            last-indentation
             (point)
             stack))))
 
@@ -643,8 +647,15 @@ Returns point or (point-min) if not inside a block."
      ;; but that statement took up less than a line
      (> (current-column) (current-indentation)))))
 
-(defun scala-indent:new-calculate-indent-for-line (&optional point)
-  "TODO document"
+(defun scala-indent:whitespace-biased-indent (&optional point)
+  "Whitespace-syntax-friendly heuristic indentation engine.
+
+The basic idea is to look back a relatively short distance (one semantic line
+back with some hand-waving) to parse the context based on a two-level
+tokenization. The parser is not anything like well-formalized, but it can start
+at an arbitrary point in the buffer, and except in pathological cases, look at
+relatively few lines in order to make a good guess; and it is tolerant to a
+certain amount of incorrect or in-progress syntactic forms."
   (let* ((initResult (scala-indent:find-analysis-start point))
          (point (car initResult))
          (stack (cadr initResult))
@@ -653,8 +664,9 @@ Returns point or (point-min) if not inside a block."
          (syntax-elem (list (nth 0 analysis)))
          (ctxt-line (nth 1 analysis))
          (ctxt-indent (nth 2 analysis))
-         (stopped-point (nth 3 analysis))
-         (end-stack (nth 4 analysis))
+         (prev-indent (nth 3 analysis))
+         (stopped-point (nth 4 analysis))
+         (end-stack (nth 5 analysis))
          )
     ;(message "analysis: %s" analysis)
     (while (or (and (= ctxt-line line-no) (> line-no 1)
@@ -682,26 +694,25 @@ Returns point or (point-min) if not inside a block."
                (point))
              stack))
       ;(message "analysis: %s" analysis)
-    ;(message "syntax-elem: %s" syntax-elem)
       (setq syntax-elem (cons (nth 0 analysis) syntax-elem))
       (setq ctxt-line (nth 1 analysis))
       (setq ctxt-indent (nth 2 analysis))
-      (setq stopped-point (nth 3 analysis))
-      (setq end-stack (nth 4 analysis)))
+      (setq prev-indent (nth 3 analysis))
+      (setq stopped-point (nth 4 analysis))
+      (setq end-stack (nth 5 analysis)))
     (when-let ((_ (< ctxt-line line-no))
                (relative (scala-indent:relative-indent-by-elem syntax-elem)))
-      (if (eq :maintain relative)
-          (current-indentation)
-        (+ (if (eq ?\n (car end-stack))
-               ;; Oops, moved a bit too far back while determining context.
-               ;; Don't really want to determine our indentation based on the
-               ;; line whose newline we are looking at, but rather the next one.
-               (save-excursion
-                 (goto-char stopped-point)
-                 (forward-line)
-                 (current-indentation))
-             ctxt-indent)
-           relative)))))
+      (list (if (eq :maintain relative)
+                (current-indentation)
+              (+ (if (eq ?\n (car end-stack))
+                     ;; Oops, moved a bit too far back while determining
+                     ;; context. Don't really want to determine our indentation
+                     ;; based on the line whose newline we are looking at, but
+                     ;; rather the next one.
+                     prev-indent
+                   ctxt-indent)
+                 relative))
+            stopped-point))))
 
 (defun scala-indent:resolve-block-step (start anchor)
   "Resolves the appropriate indent step for block line at position
@@ -732,42 +743,57 @@ Returns point or (point-min) if not inside a block."
 ;;; Indentation engine
 ;;;
 
-(defun scala-indent:apply-indent-rules (rule-indents &optional point)
-  "Evaluates each rule, until one returns non-nil value. Returns
-the sum of the value and the respective indent step, or nil if
-nothing was applied."
-  (when rule-indents
-    (save-excursion
-      (when point (goto-char point))
-      (let* ((pos (scala-syntax:beginning-of-code-line))
-             (rule-indent (car rule-indents))
-             (rule-statement (car rule-indent))
-             (indent-statement (cadr rule-indent))
-             (anchor (funcall rule-statement point)))
-        (if anchor
-            (progn
-              (if scala-mode:debug-messages
-                  (message "indenting acording to %s at %d for pos %d for point %s" rule-statement anchor pos point))
-              (when (/= anchor (point))
-                (error (format "Assertion error: anchor=%d, point=%d" anchor (point))))
-              (+ (current-column)
-                 (save-excursion
-                   (if (functionp indent-statement)
-                       (funcall indent-statement pos anchor)
-                     (eval indent-statement)))))
-          (scala-indent:apply-indent-rules (cdr rule-indents)))))))
+(defun scala-indent:block-biased-indent (point)
+  "TODO."
+  (save-excursion
+    (when point (goto-char point))
+    (let* ((pos (scala-syntax:beginning-of-code-line))
+           (anchor (scala-indent:goto-block-anchor point)))
+      (when anchor
+        (when (/= anchor (point))
+          (error (format "Assertion error: anchor=%d, point=%d" anchor (point))))
+        (list
+         (+ (current-column)
+            (save-excursion
+              (scala-indent:resolve-block-step pos anchor)))
+         anchor
+        )
+        ))))
+
+(defun scala-indent:reconcile (whitespace block)
+  (let ((ws-indent (nth 0 whitespace))
+        (ws-lookback-point (nth 1 whitespace))
+        (blk-indent-point (nth 0 block))
+        (blk-lookback (nth 1 block)))
+    (cond
+     ;; Nothing to reconcile
+     ((eq ws-indent blk-indent-point) ws-indent)
+     ;; Counterintuitive as it may be, the algorithm that had to look the
+     ;; farthest back (and so has the smallest lookback point) is least likely
+     ;; to have gotten the answer right. This is because both algorithms have
+     ;; bias toward not giving up; but the more remote they get from their
+     ;; starting point, the more likely it is that they did not understand the
+     ;; local syntax, and are going to suggest a large and unpleasant change in
+     ;; indentation. Or from another perspective: we want to bias toward local
+     ;; correctness. If they stopped on the same character, then we know from
+     ;; the behavior of the block algorithm that it is a parenthetical
+     ;; character; in which case the block algorithm most likely got the right
+     ;; answer.
+     ((> ws-lookback-point blk-lookback) ws-indent)
+     (t blk-indent-point))))
 
 (defun scala-indent:calculate-indent-for-line (&optional point)
   "Calculate the appropriate indent for the current point or POINT.
 
 Returns the new column, or nil if the indent cannot be determined."
-  (or
-   (scala-indent:apply-indent-rules
-    `((scala-indent:goto-block-anchor scala-indent:resolve-block-step)
-      )
-    point)
-   (scala-indent:new-calculate-indent-for-line point)
-   0))
+  (let ((whitespace (ignore-errors
+                      (scala-indent:whitespace-biased-indent point)))
+        (block (scala-indent:block-biased-indent point)))
+    (pcase (cons whitespace block)
+      (`(nil . ,x) (nth 0 x))
+      (`(,x . nil) (nth 0 x))
+      (`(,x . ,y) (scala-indent:reconcile x y)))
+   ))
 
 (defun scala-indent:indent-line-to (column)
   "Indent the line to column and move cursor to the indent

From 442203e7c2d2c9f67bfc8abbf6d59c9b8d44f144 Mon Sep 17 00:00:00 2001
From: Keith Pinson <keith.pinson@banno.com>
Date: Thu, 23 Dec 2021 18:37:38 -0500
Subject: [PATCH 28/42] Still trying to get dot-chaining perfected

But I see that it is certainly not perfected in the case where an infix operator
gets injected into the dot-chain.
---
 scala-mode-indent.el | 49 ++++++++++++++++++++++++--------------------
 1 file changed, 27 insertions(+), 22 deletions(-)

diff --git a/scala-mode-indent.el b/scala-mode-indent.el
index cf2f423..e19e479 100644
--- a/scala-mode-indent.el
+++ b/scala-mode-indent.el
@@ -421,8 +421,11 @@ Returns point or (point-min) if not inside a block."
 (defun scala-indent:analyze-syntax-stack (stack)
   "A kind of tokenize step of the hand-wavy parse"
   (pcase stack
+    ;; <hitting the beginning of a block when starting in the middle> { (
+    (`(?\{ ,_ . ,_) 'left-boundary)
+    (`(?\( ,_ . ,_) 'left-boundary)
     ;; <dot chaining>
-    ('(?\n ?.) 'dot-chain)
+    (`(?\n ?.) 'dot-chain)
     (`(?\n ?. . ,_) 'dot-chain)
     ;; =
     (`(= ?\n . ,_) 'decl-lhs)
@@ -447,13 +450,13 @@ Returns point or (point-min) if not inside a block."
     (`(do ,_ . ,_) 'block)
     ;; else
     (`(else ?\n . ,_) 'else-conseq)
-    ('(else) 'else)
+    (`(else) 'else)
     (`(else . ,_) 'else-inline)
     ;; enum
     ((and `(enum . ,tail) (guard (memq ': tail))) 'block)
     (`(enum . ,_) 'decl)
     ;; final
-    ('(final) 'decl)
+    (`(final) 'decl)
     ;; for
     (`(for) 'for-comp)
     (`(for . ,_) 'for-body)
@@ -463,7 +466,7 @@ Returns point or (point-min) if not inside a block."
     (`(if ?\n . ,_) 'if-cond)
     (`(if . ,_) 'if)
     ;; implicit
-    ('(implicit) 'decl)
+    (`(implicit) 'decl)
     ;; import
     ((and `(import . ,tail) (guard (memq ?\n tail))) 'after-decl)
     (`(import . ,_) 'decl)
@@ -473,12 +476,12 @@ Returns point or (point-min) if not inside a block."
     ((and `(object . ,tail) (guard (memq ': tail))) 'block)
     (`(object . ,_) 'decl)
     ;; override
-    ('(override) 'decl)
+    (`(override) 'decl)
     ;; package
     (`(package . ,_) 'decl)
     ;; then
     (`(then ?\n . ,_) 'then-conseq)
-    ('(then) 'then)
+    (`(then) 'then)
     (`(then . ,_) 'then-inline)
     ;; trait
     ((and `(trait . ,tail) (guard (memq ': tail))) 'block)
@@ -490,9 +493,9 @@ Returns point or (point-min) if not inside a block."
     ;; var
     (`(var . ,_) 'decl)
     ;; while
-    ('(while) 'decl)
+    (`(while) 'decl)
     ;; with
-    ('(with) 'block)
+    (`(with) 'block)
     ;; yield
     (`(yield . ,_) 'yield-from-comp)
     ))
@@ -549,6 +552,8 @@ Returns point or (point-min) if not inside a block."
     ;; if-cond
     (`(if-cond then) 0)
     (`(if-cond) 2)
+    ;; left-boundary
+    (`(left-boundary dot-chain) 4)
     ;; match
     (`(match case . ,_) 2)
     ;; then
@@ -600,7 +605,7 @@ Returns point or (point-min) if not inside a block."
         (setq stack
               (if (looking-at-p "\\.")
                   (cons ?. stack)
-                (let ((s (sexp-at-point)))
+                (let ((s (or (sexp-at-point) (char-after))))
                   (backward-char)
                   (if (looking-at-p "\\.")
                       ;; Try hard to notice dot-chaining
@@ -612,15 +617,12 @@ Returns point or (point-min) if not inside a block."
                       (cons s stack))))))
         (setq result
               (scala-indent:analyze-syntax-stack stack))
-        ;(message (format "result: %s" result))
         (when (and (not result)
                    (save-excursion (= (point)
                                       (scala-syntax:beginning-of-code-line))))
           (setq stack (cons ?\n stack))
-          ;(message (format "stack: %s" stack))
           (setq result
                 (scala-indent:analyze-syntax-stack stack))
-          ;(message (format "result: %s" result))
           (when result
             (setq last-indentation (current-indentation))
             (scala-syntax:backward-sexp-forcing)))
@@ -668,11 +670,15 @@ certain amount of incorrect or in-progress syntactic forms."
          (stopped-point (nth 4 analysis))
          (end-stack (nth 5 analysis))
          )
-    ;(message "analysis: %s" analysis)
     (while (or (and (= ctxt-line line-no) (> line-no 1)
                     ;; If we keep reading for this reason, we've accepted the
                     ;; existing tokens and so need to clear the stack
                     (or (setq stack nil)
+                        (setq point
+                              (save-excursion
+                                (goto-char stopped-point)
+                                (scala-syntax:backward-sexp-forcing)
+                                (point)))
                         t))
                (and (scala-indent:full-stmt-less-than-line syntax-elem stopped-point)
                     ;; If we read a full statement that was only part of a line,
@@ -680,20 +686,19 @@ certain amount of incorrect or in-progress syntactic forms."
                     (or (setq syntax-elem (cdr syntax-elem))
                         ;; restart with the existing stack
                         (setq stack end-stack)
+                        (setq point
+                              (save-excursion
+                                (goto-char stopped-point)
+                                (scala-syntax:backward-sexp-forcing)
+                                (point)))
                         t))
                ;; We know we have a dot-chain, but we need to get more context
                ;; to know how to position it
                (when (equal syntax-elem '(dot-chain))
-                 (setq stack end-stack)
+                 (setq stack nil)
+                 (setq point stopped-point)
                  t))
-      (setq analysis
-            (scala-indent:analyze-context
-             (save-excursion
-               (goto-char stopped-point)
-               (scala-syntax:backward-sexp-forcing)
-               (point))
-             stack))
-      ;(message "analysis: %s" analysis)
+      (setq analysis (scala-indent:analyze-context point stack))
       (setq syntax-elem (cons (nth 0 analysis) syntax-elem))
       (setq ctxt-line (nth 1 analysis))
       (setq ctxt-indent (nth 2 analysis))

From d26a1d23ea9e3bc5e76c64cc5033b3b5014c4d05 Mon Sep 17 00:00:00 2001
From: Keith Pinson <keith.pinson@banno.com>
Date: Thu, 30 Dec 2021 14:04:59 -0500
Subject: [PATCH 29/42] Detect infinite loop and break

---
 scala-mode-indent.el | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/scala-mode-indent.el b/scala-mode-indent.el
index e19e479..f5cc4c4 100644
--- a/scala-mode-indent.el
+++ b/scala-mode-indent.el
@@ -703,7 +703,12 @@ certain amount of incorrect or in-progress syntactic forms."
       (setq ctxt-line (nth 1 analysis))
       (setq ctxt-indent (nth 2 analysis))
       (setq prev-indent (nth 3 analysis))
-      (setq stopped-point (nth 4 analysis))
+      (let ((old-stopped-point stopped-point))
+        (setq stopped-point (nth 4 analysis))
+        (when (eq old-stopped-point stopped-point)
+          (message
+           "Whitespace-friendly indentation algorithm not making progress :(")
+          (error "Got stuck at %s" stopped-point)))
       (setq end-stack (nth 5 analysis)))
     (when-let ((_ (< ctxt-line line-no))
                (relative (scala-indent:relative-indent-by-elem syntax-elem)))

From a6983a52b67e3fb53492118a517ed9b72aa3b4a2 Mon Sep 17 00:00:00 2001
From: Keith Pinson <keith.pinson@banno.com>
Date: Thu, 30 Dec 2021 15:06:39 -0500
Subject: [PATCH 30/42] Seems to fix the infinite loop

---
 scala-mode-indent.el | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scala-mode-indent.el b/scala-mode-indent.el
index f5cc4c4..36fe1fa 100644
--- a/scala-mode-indent.el
+++ b/scala-mode-indent.el
@@ -599,8 +599,8 @@ Returns point or (point-min) if not inside a block."
   (save-excursion
     (goto-char point)
     (let (result
-          (stack init-stack)
-          last-indentation)
+          last-indentation
+          (stack init-stack))
       (while (and (not result) (> (point) 1))
         (setq stack
               (if (looking-at-p "\\.")

From c8a7bbafcd55409d031413bd80148a65dccbbbe1 Mon Sep 17 00:00:00 2001
From: Keith Pinson <keith.pinson@banno.com>
Date: Thu, 30 Dec 2021 15:07:33 -0500
Subject: [PATCH 31/42] Extract function: `scala-indent:continue-lookback?`

---
 scala-mode-indent.el | 66 +++++++++++++++++++++++++-------------------
 1 file changed, 38 insertions(+), 28 deletions(-)

diff --git a/scala-mode-indent.el b/scala-mode-indent.el
index 36fe1fa..21dfb86 100644
--- a/scala-mode-indent.el
+++ b/scala-mode-indent.el
@@ -649,6 +649,37 @@ Returns point or (point-min) if not inside a block."
      ;; but that statement took up less than a line
      (> (current-column) (current-indentation)))))
 
+(defun scala-indent:continue-lookback? (syntax-elem
+                                       ctxt-line
+                                       line-no
+                                       stopped-point
+                                       end-stack)
+  (or (and (= ctxt-line line-no) (> line-no 1)
+           ;; If we keep reading for this reason, we've accepted the
+           ;; existing tokens and so need to clear the stack
+           (list nil ;; syntax-elem
+                 nil ;; stack
+                 (save-excursion ;; point
+                   (goto-char stopped-point)
+                   (scala-syntax:backward-sexp-forcing)
+                   (point))))
+      (when (scala-indent:full-stmt-less-than-line syntax-elem stopped-point)
+        ;; If we read a full statement that was only part of a line,
+        ;; drop it and try again for more context
+        (list (cdr syntax-elem) ;; syntax-elem
+              end-stack ;; restart with the existing stack
+              (save-excursion ;; point
+                (goto-char stopped-point)
+                (scala-syntax:backward-sexp-forcing)
+                (point))))
+      ;; We know we have a dot-chain, but we need to get more context to know
+      ;; how to position it
+      (when (equal syntax-elem '(dot-chain))
+        (list nil ;; syntax-elem
+              nil ;; stack
+              stopped-point ;; point
+              ))))
+
 (defun scala-indent:whitespace-biased-indent (&optional point)
   "Whitespace-syntax-friendly heuristic indentation engine.
 
@@ -670,34 +701,13 @@ certain amount of incorrect or in-progress syntactic forms."
          (stopped-point (nth 4 analysis))
          (end-stack (nth 5 analysis))
          )
-    (while (or (and (= ctxt-line line-no) (> line-no 1)
-                    ;; If we keep reading for this reason, we've accepted the
-                    ;; existing tokens and so need to clear the stack
-                    (or (setq stack nil)
-                        (setq point
-                              (save-excursion
-                                (goto-char stopped-point)
-                                (scala-syntax:backward-sexp-forcing)
-                                (point)))
-                        t))
-               (and (scala-indent:full-stmt-less-than-line syntax-elem stopped-point)
-                    ;; If we read a full statement that was only part of a line,
-                    ;; drop it and try again for more context
-                    (or (setq syntax-elem (cdr syntax-elem))
-                        ;; restart with the existing stack
-                        (setq stack end-stack)
-                        (setq point
-                              (save-excursion
-                                (goto-char stopped-point)
-                                (scala-syntax:backward-sexp-forcing)
-                                (point)))
-                        t))
-               ;; We know we have a dot-chain, but we need to get more context
-               ;; to know how to position it
-               (when (equal syntax-elem '(dot-chain))
-                 (setq stack nil)
-                 (setq point stopped-point)
-                 t))
+    (message "analysis: %s" analysis)
+    (while (when-let ((x (scala-indent:continue-lookback?
+                        syntax-elem ctxt-line line-no stopped-point end-stack)))
+             (setq syntax-elem (or (nth 0 x) syntax-elem))
+             (setq stack (nth 1 x))
+             (setq point (nth 2 x))
+             t)
       (setq analysis (scala-indent:analyze-context point stack))
       (setq syntax-elem (cons (nth 0 analysis) syntax-elem))
       (setq ctxt-line (nth 1 analysis))

From 1ef3ce5dc4750d396276a2331411c42983127906 Mon Sep 17 00:00:00 2001
From: Keith Pinson <keith.pinson@banno.com>
Date: Tue, 4 Jan 2022 14:45:38 -0500
Subject: [PATCH 32/42] Augh, try to fix a bunch of cases without breaking
 others

There are more still that aren't right.
---
 scala-mode-indent.el | 65 +++++++++++++++++++++++++++++++++++---------
 1 file changed, 52 insertions(+), 13 deletions(-)

diff --git a/scala-mode-indent.el b/scala-mode-indent.el
index 21dfb86..2f97274 100644
--- a/scala-mode-indent.el
+++ b/scala-mode-indent.el
@@ -282,6 +282,8 @@ and are in fact a sign of run-on. Reserved-symbols not included.")
         nil)
        ;; NO: this line is the start of value body
        ;; ((scala-indent:body-p) ;; TODO did I delete this function when I shouldn't have?
+       ;; TODO or even if I did, maybe it just doesn't matter because the
+       ;; heuristics that union this algorithm with the other will compensate?
        ;;  nil)
        ;; YES: eager strategy can stop here, everything is a run-on if no
        ;; counter evidence
@@ -422,6 +424,7 @@ Returns point or (point-min) if not inside a block."
   "A kind of tokenize step of the hand-wavy parse"
   (pcase stack
     ;; <hitting the beginning of a block when starting in the middle> { (
+    (`(?\{) 'left-boundary) ;; too aggressive?
     (`(?\{ ,_ . ,_) 'left-boundary)
     (`(?\( ,_ . ,_) 'left-boundary)
     ;; <dot chaining>
@@ -430,7 +433,7 @@ Returns point or (point-min) if not inside a block."
     ;; =
     (`(= ?\n . ,_) 'decl-lhs)
     ((and `(= ,_ . ,tail) (guard (memq ?\n tail))) 'after-decl)
-    (`(= ,_ . ,_) 'decl-lhs)
+    (`(= ,_ . ,_) 'decl-inline-lhs)
     ;; =>
     (`(=> ?\n . ,_) 'arrow-lhs)
     ((and `(=> ,_ . ,tail) (guard (memq ?\n tail))) 'after-arrow)
@@ -479,6 +482,8 @@ Returns point or (point-min) if not inside a block."
     (`(override) 'decl)
     ;; package
     (`(package . ,_) 'decl)
+    ;; sealed
+    (`(sealed) 'decl)
     ;; then
     (`(then ?\n . ,_) 'then-conseq)
     (`(then) 'then)
@@ -520,6 +525,7 @@ Returns point or (point-min) if not inside a block."
     (`(case ,_) 2)
     ;; decl
     (`(decl decl) 0)
+    (`(decl decl decl-inline-lhs) 0)
     (`(decl else) -2)
     (`(decl . ,_) 2)
     ;; decl-lhs
@@ -589,10 +595,19 @@ Returns point or (point-min) if not inside a block."
             (setq stack (cons ?\n stack)))
         ;; (beginning-of-thing 'sexp) gets confused by `.'
         (unless (looking-at-p "\\.")
-          ;; Avoid double-reading curent symbol
+          ;; Avoid double-reading current symbol
           (beginning-of-thing 'sexp)))
-      (list (point)
-            stack))))
+      ;; handle the occurence of case in various contexts
+      (or (save-excursion
+            (when-let ((_ (looking-at-p (concat "case *"
+                                              scala-syntax:class-or-object-re)))
+                     (point (progn (forward-to-word 1) (point)))
+                     (class-or-object (sexp-at-point)))
+              ;; This throws away the stack we've built up above. The assumption
+              ;; here is that this case is mutually exclusive with those above.
+              (scala-indent:skip-back-over-modifiers point
+                                                     (list class-or-object))))
+       (list (point) stack)))))
 
 (defun scala-indent:analyze-context (point &optional init-stack)
   "TODO document"
@@ -629,13 +644,19 @@ Returns point or (point-min) if not inside a block."
         (unless result
           (setq last-indentation (current-indentation))
           (while (looking-at-p "\\.") (backward-char))
+          ;; ")." is a funny case where we actually do want to be on the dot
+          (if (looking-at-p ")") (forwar-char))
           (scala-syntax:backward-sexp-forcing)))
-      (list result
-            (line-number-at-pos)
-            (current-indentation)
-            last-indentation
-            (point)
-            stack))))
+      (let* ((x (or (scala-indent:skip-back-over-modifiers (point) stack)
+                    (list (point) stack)))
+             (point (nth 0 x))
+             (stack (nth 1 x)))
+        (list result
+              (line-number-at-pos)
+              (current-indentation)
+              last-indentation
+              point
+              stack)))))
 
 (defun scala-indent:full-stmt-less-than-line (syntax-elem stopped-point)
   (and
@@ -657,7 +678,7 @@ Returns point or (point-min) if not inside a block."
   (or (and (= ctxt-line line-no) (> line-no 1)
            ;; If we keep reading for this reason, we've accepted the
            ;; existing tokens and so need to clear the stack
-           (list nil ;; syntax-elem
+           (list syntax-elem ;; syntax-elem
                  nil ;; stack
                  (save-excursion ;; point
                    (goto-char stopped-point)
@@ -675,11 +696,29 @@ Returns point or (point-min) if not inside a block."
       ;; We know we have a dot-chain, but we need to get more context to know
       ;; how to position it
       (when (equal syntax-elem '(dot-chain))
-        (list nil ;; syntax-elem
+        (list syntax-elem ;; syntax-elem
               nil ;; stack
               stopped-point ;; point
               ))))
 
+(defun scala-indent:skip-back-over-modifiers (point stack)
+  (if-let* ((head (car stack))
+            (_ (memq head '(trait class object)))
+            (new-point point)
+            (new-sexp t)
+            (new-stack stack))
+      (save-excursion
+        (goto-char new-point)
+        (scala-syntax:backward-sexp-forcing)
+        (setq new-sexp (sexp-at-point))
+        (while (memq new-sexp
+                     '(final sealed case open abstract implicit private))
+          (setq new-point (point))
+          (setq new-stack (cons new-sexp new-stack))
+          (scala-syntax:backward-sexp-forcing)
+          (setq new-sexp (sexp-at-point)))
+        (list new-point new-stack))))
+
 (defun scala-indent:whitespace-biased-indent (&optional point)
   "Whitespace-syntax-friendly heuristic indentation engine.
 
@@ -704,7 +743,7 @@ certain amount of incorrect or in-progress syntactic forms."
     (message "analysis: %s" analysis)
     (while (when-let ((x (scala-indent:continue-lookback?
                         syntax-elem ctxt-line line-no stopped-point end-stack)))
-             (setq syntax-elem (or (nth 0 x) syntax-elem))
+             (setq syntax-elem (nth 0 x))
              (setq stack (nth 1 x))
              (setq point (nth 2 x))
              t)

From c18f2bb9976bcbb140f907e4685c2759aafbb83d Mon Sep 17 00:00:00 2001
From: Jack Viers <jackcviers@Jacks-MacBook-Pro.local>
Date: Wed, 5 Jan 2022 17:02:33 -0600
Subject: [PATCH 33/42] WIP: Adding cycling indent

* Functionality conceptually complete (haven't tested it yet)
* Trying to figure out how to test this with ERT before I QA it.
---
 scala-mode-indent.el    | 81 ++++++++++++++++++++++++++++++++++++++++-
 test/scala-mode-test.el | 26 +++++++++++++
 2 files changed, 105 insertions(+), 2 deletions(-)

diff --git a/scala-mode-indent.el b/scala-mode-indent.el
index 2f97274..a635a8c 100644
--- a/scala-mode-indent.el
+++ b/scala-mode-indent.el
@@ -133,6 +133,14 @@ Scaladoc behavior of indenting comment lines to the second asterisk."
   :safe #'booleanp
   :group 'scala)
 
+(defcustom scala-indent:use-cycle-indent nil
+  "When non-nil, indentation will cycle from the new indent
+  strategy indent, the last known indent, and the left margin on
+  subsequent indent-line calls."
+  :type 'boolean
+  :safe #'booleanp
+  :group 'scala)
+
 (defun scala-indent:run-on-strategy ()
   "Returns the currently effecti run-on strategy"
   (or scala-indent:effective-run-on-strategy
@@ -897,8 +905,65 @@ strings"
           (beginning-of-line)
           (when (looking-at "^\\s +$") (point)))))
 
-(defun scala-indent:indent-line (&optional strategy)
-  "Indents the current line."
+(defvar-local scala-indent:cycle-indent-stack nil
+  "The automatically buffer local scala indent cycle stack.
+
+The stack is initialized as (left-margin, (current-indentation))
+when the custom var \"scala-indent:use-cycle-indent\" is non-nil
+and \"scala-indent:indent-line\" is called. Subsequent
+\"scala-indent:indent-line\" calls pop the indentation value from
+the stack, until it is empty, resetting the indentation cycle.")
+
+
+
+(defun scala-indent:cycle-indent-stack-push (indentation)
+  "Pushes an integer value onto the \"scala-indent:cycle-indent-stack\".
+
+Will fail if INDENTATION is not an integer"
+
+  (if (integerp indentation)
+      (add-to-list 'scala-indent:cycle-indent-stack indentation)
+    (error "\"scala-indent:cycle-indent-stack-push\": Invalid INDENTATION argument %s"
+	   indentation)))
+
+(defun scala-indent:cycle-indent-stack-pop ()
+  "Gets the top value of the \"scala-indent:cycle-indent-stack\" stack.
+
+ Modifies the stack in-place."
+
+  (pop 'scala-indent:cycle-indent-stack))
+
+(defun scala-indent:cycle-indent-stack-depth ()
+  "The current depth of the \"scala-indent:cycle-indent-stack\" stack"
+  
+  (length 'scala-indent:cycle-indent-stack))
+
+(defun scala-indent:cycle-indent-stack-emptyp ()
+  "Check if the \"scala-indent:cycle-indent-stack\" is empty.
+
+Returns t if the \"scala-indent:cycle-indent-stack\" is empty,
+nil otherwise."
+  
+  (eql (length 'scala-indent:cycle-indent-stack) 0))
+
+(defun scala-indent:cycle-indent-line (&optional strategy)
+  "Cycle scala indentation using optionally passed STRATEGY.
+
+When the \"scala-indent:cycle-indent-stack\" is empty, push 0 and
+the current indentation onto the stack, then indent according to
+the optionally passed STRATEGY.  Indent to the top of
+\"scala-indent:cycle-indent-stack\" when non-empty."
+  
+  (interactive "*")
+  (cond (((scala-indent:cycle-indent-stack-emptyp))
+	 (scala-indent:cycle-indent-stack-push (current-indentation))
+	 (scala-indent:cycle-indent-stack-push 0)
+	 (call-interactively 'scala-indent:strategy-indent-line t))
+	(t (scala-indent:indent-line-to (scala-indent:cycle-indent-stack-pop)))))
+
+;; the previously-named scala-indent:indent-line
+(defun scala-indent:strategy-indent-line (&optional strategy)
+  "Indent lines according to the OPTIONAL scala indentation STRATEGY."
   (interactive "*")
   (let ((state (save-excursion (syntax-ppss (line-beginning-position)))))
     (if (nth 8 state) ;; 8 = start pos of comment or string
@@ -918,6 +983,18 @@ strings"
       (scala-indent:indent-code-line strategy)))
   )
 
+(defun scala-indent:indent-line (&optional strategy)
+  "Indent the current line with cycling.
+
+If the custom var \"scala-indent:use-cycle-indent\" is non-nil,
+cycle-indent using the optionally passed STRATEGY.  Indent using
+the optionally passed STRATEGY without cycling otherwise."
+  
+  (interactive "*")
+  (if scala-indent:use-cycle-indent
+      (call-interactively t 'scala-indent:cycle-indent-line)
+    (call-interactively t 'scala-indent:strategy-indent-line)))
+
 (defun scala-indent:indent-with-reluctant-strategy ()
   (interactive "*")
   (scala-indent:indent-line scala-indent:reluctant-strategy))
diff --git a/test/scala-mode-test.el b/test/scala-mode-test.el
index 8a7a195..6d2b63a 100644
--- a/test/scala-mode-test.el
+++ b/test/scala-mode-test.el
@@ -176,3 +176,29 @@ comment. A concrete example may be viewed at https://github.com/scala/scala/blob
    "/* &*/"
    "110111"
    "DDDOOO"))
+
+(ert-deftest scala-indent:scala-indent:use-cycle-indent-test-1 ()
+  "Custom \"scala-indent:use-cycle-indent\" should be nil by default."
+  
+  (should-not scala-indent:use-cycle-indent ))
+
+(ert-deftest scala-indent:scala-indent:use-cycle-indent-test-2 ()
+  "Custom \"scala-indent:use-cycle-indent\" must be a boolean."
+  
+  :expected-result :failed
+  (custom-set-variables '(scala-indent:use-cycle-indent "gobbledygook"))
+  (should (= 'scala-indent:use-cycle-indent "gobbledygook")))
+
+(ert-deftest scala-indent:scala-indent:use-cycle-indent-test-3 ()
+  "Custom \"scala-indent:use-cycle-indent\" should be settable to a boolean value."
+  
+  (custom-set-variables '(scala-indent:use-cycle-indent t))
+  (should 'scala-indent:use-cycle-indent )
+  (custom-set-variables '(scala-indent:use-cycle-indent nil)))
+
+(ert-deftest scala-indent:scala-indent:cycle-indent-stack-test-1 ()
+  "\"scala-indent:cycle-indent-stack\" should be 0 current-indentation after one call."
+  (custom-set-variables '(scala-indent:use-cycle-indent t))
+  (call-interactively (scala-indent:indent-line))
+  (should (buffer-local-value 'scala-indent:cycle-indent-stack))
+  (custom-set-variables '(scala-indent:use-cycle-indent nil)))

From 74eaa2446f55d54d833222930c3bfb72e0068782 Mon Sep 17 00:00:00 2001
From: Jack Viers <jackcviers@gmail.com>
Date: Sun, 15 May 2022 19:47:15 -0500
Subject: [PATCH 34/42] Cycles between current indent and 0 when cycle-indent
 is true

---
 scala-mode-indent.el | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/scala-mode-indent.el b/scala-mode-indent.el
index a635a8c..b3d6c0c 100644
--- a/scala-mode-indent.el
+++ b/scala-mode-indent.el
@@ -905,7 +905,7 @@ strings"
           (beginning-of-line)
           (when (looking-at "^\\s +$") (point)))))
 
-(defvar-local scala-indent:cycle-indent-stack nil
+(defvar-local scala-indent:cycle-indent-stack (list)
   "The automatically buffer local scala indent cycle stack.
 
 The stack is initialized as (left-margin, (current-indentation))
@@ -931,20 +931,21 @@ Will fail if INDENTATION is not an integer"
 
  Modifies the stack in-place."
 
-  (pop 'scala-indent:cycle-indent-stack))
+  (pop (buffer-local-value 'scala-indent:cycle-indent-stack (current-buffer))))
 
 (defun scala-indent:cycle-indent-stack-depth ()
   "The current depth of the \"scala-indent:cycle-indent-stack\" stack"
   
-  (length 'scala-indent:cycle-indent-stack))
+  (length (buffer-local-value 'scala-indent:cycle-indent-stack (current-buffer))))
 
-(defun scala-indent:cycle-indent-stack-emptyp ()
+
+(defun scala-indent:cycle-indent-stack-emptyp (x)
   "Check if the \"scala-indent:cycle-indent-stack\" is empty.
 
 Returns t if the \"scala-indent:cycle-indent-stack\" is empty,
 nil otherwise."
-  
-  (eql (length 'scala-indent:cycle-indent-stack) 0))
+
+  (= (length (buffer-local-value 'scala-indent:cycle-indent-stack (current-buffer))) 0))
 
 (defun scala-indent:cycle-indent-line (&optional strategy)
   "Cycle scala indentation using optionally passed STRATEGY.
@@ -955,7 +956,7 @@ the optionally passed STRATEGY.  Indent to the top of
 \"scala-indent:cycle-indent-stack\" when non-empty."
   
   (interactive "*")
-  (cond (((scala-indent:cycle-indent-stack-emptyp))
+  (cond ((scala-indent:cycle-indent-stack-emptyp nil)
 	 (scala-indent:cycle-indent-stack-push (current-indentation))
 	 (scala-indent:cycle-indent-stack-push 0)
 	 (call-interactively 'scala-indent:strategy-indent-line t))

From 9d77eec3a6537221a0f0b237d565b2995dac5ada Mon Sep 17 00:00:00 2001
From: Jack Viers <jackcviers@gmail.com>
Date: Thu, 19 May 2022 09:25:04 -0500
Subject: [PATCH 35/42] Fixes incorrect argument order in call-interactively

Incorrect order of arguments causes wrong type argument error in
scala-indent:indent-line. Swapping argument order fixes the error reported by
@jj1uzh [here](https://github.com/Kazark/emacs-scala-mode/pull/1#issuecomment-1131735619).
---
 scala-mode-indent.el | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scala-mode-indent.el b/scala-mode-indent.el
index b3d6c0c..0df2ba5 100644
--- a/scala-mode-indent.el
+++ b/scala-mode-indent.el
@@ -993,8 +993,8 @@ the optionally passed STRATEGY without cycling otherwise."
   
   (interactive "*")
   (if scala-indent:use-cycle-indent
-      (call-interactively t 'scala-indent:cycle-indent-line)
-    (call-interactively t 'scala-indent:strategy-indent-line)))
+      (call-interactively 'scala-indent:cycle-indent-line t)
+    (call-interactively 'scala-indent:strategy-indent-line t)))
 
 (defun scala-indent:indent-with-reluctant-strategy ()
   (interactive "*")

From ad122e464e4081465ed4478fba5214ac5fe852e0 Mon Sep 17 00:00:00 2001
From: Mehmet Emre <memre@usfca.edu>
Date: Fri, 31 Mar 2023 19:47:12 -0700
Subject: [PATCH 36/42] fix typo in function name

---
 scala-mode-indent.el | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scala-mode-indent.el b/scala-mode-indent.el
index 0df2ba5..1612720 100644
--- a/scala-mode-indent.el
+++ b/scala-mode-indent.el
@@ -653,7 +653,7 @@ Returns point or (point-min) if not inside a block."
           (setq last-indentation (current-indentation))
           (while (looking-at-p "\\.") (backward-char))
           ;; ")." is a funny case where we actually do want to be on the dot
-          (if (looking-at-p ")") (forwar-char))
+          (if (looking-at-p ")") (forward-char))
           (scala-syntax:backward-sexp-forcing)))
       (let* ((x (or (scala-indent:skip-back-over-modifiers (point) stack)
                     (list (point) stack)))

From 56d1f77e9047cc767d105ffca28d5ec3e326648c Mon Sep 17 00:00:00 2001
From: Mehmet Emre <memre@usfca.edu>
Date: Fri, 31 Mar 2023 19:47:34 -0700
Subject: [PATCH 37/42] indent after declarations

---
 scala-mode-indent.el | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scala-mode-indent.el b/scala-mode-indent.el
index 1612720..694089d 100644
--- a/scala-mode-indent.el
+++ b/scala-mode-indent.el
@@ -537,7 +537,7 @@ Returns point or (point-min) if not inside a block."
     (`(decl else) -2)
     (`(decl . ,_) 2)
     ;; decl-lhs
-    (`(decl-lhs decl . ,_) 0)
+    (`(decl-lhs decl . ,_) 2)
     (`(decl-lhs dot-chain) 4)
     (`(dot-chain dot-chain) 0)
     (`(decl-lhs for-comp) 0)

From 5323a097a7fa06d08f525dc76c27c16cdf516826 Mon Sep 17 00:00:00 2001
From: Mehmet Emre <memre@usfca.edu>
Date: Fri, 31 Mar 2023 19:47:56 -0700
Subject: [PATCH 38/42] do not push nil into the analysis stack

---
 scala-mode-indent.el | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/scala-mode-indent.el b/scala-mode-indent.el
index 694089d..553bb80 100644
--- a/scala-mode-indent.el
+++ b/scala-mode-indent.el
@@ -756,7 +756,10 @@ certain amount of incorrect or in-progress syntactic forms."
              (setq point (nth 2 x))
              t)
       (setq analysis (scala-indent:analyze-context point stack))
-      (setq syntax-elem (cons (nth 0 analysis) syntax-elem))
+      (setq syntax-elem
+	    (if (nth 0 analysis)
+		(cons (nth 0 analysis) syntax-elem)
+	      syntax-elem))
       (setq ctxt-line (nth 1 analysis))
       (setq ctxt-indent (nth 2 analysis))
       (setq prev-indent (nth 3 analysis))

From 9df2ba4b72df4517173177b94b17b30fba1e1879 Mon Sep 17 00:00:00 2001
From: Mehmet Emre <maemre2@gmail.com>
Date: Mon, 3 Apr 2023 01:54:20 -0700
Subject: [PATCH 39/42] disable aggressive paren rule temporarily

---
 scala-mode-indent.el | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scala-mode-indent.el b/scala-mode-indent.el
index 553bb80..2dbff7d 100644
--- a/scala-mode-indent.el
+++ b/scala-mode-indent.el
@@ -434,7 +434,7 @@ Returns point or (point-min) if not inside a block."
     ;; <hitting the beginning of a block when starting in the middle> { (
     (`(?\{) 'left-boundary) ;; too aggressive?
     (`(?\{ ,_ . ,_) 'left-boundary)
-    (`(?\( ,_ . ,_) 'left-boundary)
+    ; (`(?\( ,_ . ,_) 'left-boundary)
     ;; <dot chaining>
     (`(?\n ?.) 'dot-chain)
     (`(?\n ?. . ,_) 'dot-chain)

From be4ef5c8c581803d11ae576b016a2fa30467d297 Mon Sep 17 00:00:00 2001
From: Mehmet Emre <maemre2@gmail.com>
Date: Sun, 4 Jun 2023 19:48:48 -0700
Subject: [PATCH 40/42] preliminary indentation support for `extension`

---
 scala-mode-indent.el | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/scala-mode-indent.el b/scala-mode-indent.el
index 2dbff7d..d1d2461 100644
--- a/scala-mode-indent.el
+++ b/scala-mode-indent.el
@@ -250,7 +250,9 @@ and the empty line")
   (regexp-opt '("abstract" "catch" "case" "class" "def" "do" "else" "final"
                 "finally" "for" "if" "implicit" "import" "lazy" "new" "object"
                 "override" "package" "private" "protected" "return" "sealed"
-                "throw" "trait" "try" "type" "val" "var" "while" "yield" "inline")
+                "throw" "trait" "try" "type" "val" "var" "while" "yield" "inline"
+                "extension"
+                )
               'words)
   "Words that we don't want to continue the previous line")
 
@@ -438,6 +440,13 @@ Returns point or (point-min) if not inside a block."
     ;; <dot chaining>
     (`(?\n ?.) 'dot-chain)
     (`(?\n ?. . ,_) 'dot-chain)
+    ;; extension
+    ;;
+    ;; FIXME: This is a hack that just checks if the previous line contains
+    ;; extension.  The check for extension should check whether this is a
+    ;; single-def extension and for balanced parentheses, etc. to determine
+    ;; whether we emit a block token.
+    ((and `(extension . ,tail) (guard (memq ?\n tail))) 'block)
     ;; =
     (`(= ?\n . ,_) 'decl-lhs)
     ((and `(= ,_ . ,tail) (guard (memq ?\n tail))) 'after-decl)

From 417dd61bbadc30d08e373afe1ab5b48d7103e10c Mon Sep 17 00:00:00 2001
From: Mehmet Emre <maemre2@gmail.com>
Date: Sun, 4 Jun 2023 19:49:04 -0700
Subject: [PATCH 41/42] add support for `do` as an align keyword like `then`
 and `else`.

In Scala 3, `do-while` is dropped, and `do` is used only for separating the body of a `while` or `for` from the guard/comprehension. So, `do` would need to be indented like `then` or `else`. See: https://docs.scala-lang.org/scala3/reference/dropped-features/do-while.html#
---
 scala-mode-indent.el | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scala-mode-indent.el b/scala-mode-indent.el
index d1d2461..32c34b8 100644
--- a/scala-mode-indent.el
+++ b/scala-mode-indent.el
@@ -372,7 +372,7 @@ is not on a run-on line."
           "\\|:\\("  scala-syntax:after-reserved-symbol-re "\\)"))
 
 (defconst scala-indent:forms-align-re
-  (regexp-opt '("yield" "then" "else" "catch" "finally") 'words))
+  (regexp-opt '("do" "yield" "then" "else" "catch" "finally") 'words))
 
 (defun scala-indent:forms-align-p (&optional point)
   "Returns `scala-syntax:beginning-of-code-line' for the line on
@@ -520,6 +520,7 @@ Returns point or (point-min) if not inside a block."
     (`(with) 'block)
     ;; yield
     (`(yield . ,_) 'yield-from-comp)
+    (`(do . ,_) 'yield-from-comp)
     ))
 
 (defun scala-indent:relative-indent-by-elem (syntax-elem)

From 5eef13b295b9d7a9903fa2eb6b73812e0899db2d Mon Sep 17 00:00:00 2001
From: Mehmet Emre <maemre2@gmail.com>
Date: Sun, 4 Jun 2023 19:51:00 -0700
Subject: [PATCH 42/42] Preserve line numbers for blank lines.

This allows differentiating between indenting a blank line and re-indenting the
line before it.  This situation occurs fairly often: it happens every time the
user hits re turn to create a new line, which calls indentation function on the
previous line and the newly created line separately.  Before this fix, these two
calls would incorrectly calculate the same indentation level, e.g. when the
previous line just created a block.
---
 scala-mode-indent.el | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/scala-mode-indent.el b/scala-mode-indent.el
index 32c34b8..2d382c2 100644
--- a/scala-mode-indent.el
+++ b/scala-mode-indent.el
@@ -746,10 +746,17 @@ tokenization. The parser is not anything like well-formalized, but it can start
 at an arbitrary point in the buffer, and except in pathological cases, look at
 relatively few lines in order to make a good guess; and it is tolerant to a
 certain amount of incorrect or in-progress syntactic forms."
-  (let* ((initResult (scala-indent:find-analysis-start point))
+  (let* ((line-no
+          ;; Get the line number while taking blanks into account.  This allows
+          ;; differentiating between indenting at a blank line and re-indenting
+          ;; at the line right before it.
+          (line-number-at-pos
+           (save-excursion
+             (when point (goto-char point))
+             (point))))
+         (initResult (scala-indent:find-analysis-start point))
          (point (car initResult))
          (stack (cadr initResult))
-         (line-no (line-number-at-pos point))
          (analysis (scala-indent:analyze-context point stack))
          (syntax-elem (list (nth 0 analysis)))
          (ctxt-line (nth 1 analysis))