JuliaSyntax parser-based REPL completions overhaul (#57767)

# Overview

As we add REPL features, bugs related to the ad-hoc parsing done by
`REPLCompletions.completions` have crept in. This pull request replaces
most of the manual parsing (regex, `find_start_brace`) with a new
approach that parses the entire input buffer once, before and after the
cursor, using JuliaSyntax. We then query the parsed syntax tree to
determine the kind of completion to be done.

# Changes
- New, JuliaSyntax-based completions mechanism.

- The `complete_line` interface now has the option of replacing
  arbitrary regions of text in the input buffer by returning a `Region`
  (`Pair{Int, Int}` for consistency with the convention in LineEdit, and
  `pos` being a 0-based byte offset).

- Fixes parsing-related bugs:
  - fix #55420
  - fix #55429
  - fix #55518
  - fix #55520
  - fix #55842
  - fix #56389
  - fix #57307
  - fix #57611
  - fix #57624
  - fix #58099

- Fixes some bugs that exist on 28d3bd5 that were found by fuzzing:
  - `x \"` + `TAB` throws a `FieldError` exception
  - String completion would sometimes delete the entire input buffer.
  - Completions should not happen inside comments.

- The duplicate code for path completion in strings, `Cmd`-strings, and
  the shell has been removed, causing paths to complete the same way for
  all three. Now, `~` is expanded in two situations:
  - If `foo` exists, or if `foo` does not exist but there are no
    possible completions:
    ```
    "~/foo/b|"     =TAB=>   "~/foo/bar|"
    "~/foo/bar|"   =TAB=>   "/home/user/foo/bar|"
       OR
    "~/foo/bar"|   =TAB=>   "/home/user/foo/bar"|
    ```

  - If the current path ends with a `/` and you hit TAB again:
    ```
    "~/foo/|"      =TAB=>   "/home/user/foo/|"
       OR
    "~/foo/"|      =TAB=>   "/home/user/foo/"|
    ```

# Future work
- Method completions could be changed to look for methods with exactly
  the given number of arguments if the closing `)` is present, and search
  for signatures with the right prefix otherwise.

- It would be nice to be able to search by type as well as value
  (perhaps by putting `::T` in place of arguments).

- Other REPL features could benefit from JuliaSyntax, so it might be
  worth sharing the parse tree between completions and other features:

    - Emacs-style sexpr navigation: `C-M-f`/`C-M-b`/`C-M-u`, etc.
    - Improved auto-indent.

- It would be nice if hints worked even when the cursor is between text.

- `CursorNode` is a slightly tweaked copy of `SyntaxNode` from
  JuliaSyntax that tracks the parent node but includes all trivia. It is
  used with `seek_pos`, which navigates to the innermost node at a given
  position so we can examine nearby nodes and the parent. This could
  probably duplicate less code from JuliaSyntax.

(cherry picked from commit ff0a931)

Author: xal-0

stdlib/REPL/src/LineEdit.jl

@@ -391,16 +391,21 @@ function complete_line(s::MIState)
     end
 end
 
+# Old complete_line return type: Vector{String},          String, Bool
+# New complete_line return type: NamedCompletion{String}, String, Bool
+#                            OR  NamedCompletion{String}, Region, Bool
+#
 # due to close coupling of the Pkg ReplExt `complete_line` can still return a vector of strings,
 # so we convert those in this helper
-function complete_line_named(args...; kwargs...)::Tuple{Vector{NamedCompletion},String,Bool}
-    result = complete_line(args...; kwargs...)::Union{Tuple{Vector{NamedCompletion},String,Bool},Tuple{Vector{String},String,Bool}}
-    if result isa Tuple{Vector{NamedCompletion},String,Bool}
-        return result
-    else
-        completions, partial, should_complete = result
-        return map(NamedCompletion, completions), partial, should_complete
-    end
+function complete_line_named(c, s, args...; kwargs...)::Tuple{Vector{NamedCompletion},Region,Bool}
+    r1, r2, should_complete = complete_line(c, s, args...; kwargs...)::Union{
+        Tuple{Vector{String}, String, Bool},
+        Tuple{Vector{NamedCompletion}, String, Bool},
+        Tuple{Vector{NamedCompletion}, Region, Bool},
+    }
+    completions = (r1 isa Vector{String} ? map(NamedCompletion, r1) : r1)
+    r = (r2 isa String ? (position(s)-sizeof(r2) => position(s)) : r2)
+    completions, r, should_complete
 end
 
 # checks for a hint and shows it if appropriate.
@@ -426,14 +431,14 @@ function check_show_hint(s::MIState)
         return
     end
     t_completion = Threads.@spawn :default begin
-        named_completions, partial, should_complete = nothing, nothing, nothing
+        named_completions, reg, should_complete = nothing, nothing, nothing
 
         # only allow one task to generate hints at a time and check around lock
         # if the user has pressed a key since the hint was requested, to skip old completions
         next_key_pressed() && return
         @lock s.hint_generation_lock begin
             next_key_pressed() && return
-            named_completions, partial, should_complete = try
+            named_completions, reg, should_complete = try
                 complete_line_named(st.p.complete, st, s.active_module; hint = true)
             catch
                 lock_clear_hint()
@@ -448,21 +453,19 @@ function check_show_hint(s::MIState)
             return
         end
         # Don't complete for single chars, given e.g. `x` completes to `xor`
-        if length(partial) > 1 && should_complete
+        if reg.second - reg.first > 1 && should_complete
             singlecompletion = length(completions) == 1
             p = singlecompletion ? completions[1] : common_prefix(completions)
             if singlecompletion || p in completions # i.e. complete `@time` even though `@time_imports` etc. exists
-                # The completion `p` and the input `partial` may not share the same initial
+                # The completion `p` and the region `reg` may not share the same initial
                 # characters, for instance when completing to subscripts or superscripts.
                 # So, in general, make sure that the hint starts at the correct position by
                 # incrementing its starting position by as many characters as the input.
-                startind = 1 # index of p from which to start providing the hint
-                maxind = ncodeunits(p)
-                for _ in partial
-                    startind = nextind(p, startind)
-                    startind > maxind && break
-                end
+                maxind = lastindex(p)
+                startind = sizeof(content(s, reg))
                 if startind ≤ maxind # completion on a complete name returns itself so check that there's something to hint
+                    # index of p from which to start providing the hint
+                    startind = nextind(p, startind)
                     hint = p[startind:end]
                     next_key_pressed() && return
                     @lock s.line_modify_lock begin
@@ -491,25 +494,24 @@ function clear_hint(s::ModeState)
 end
 
 function complete_line(s::PromptState, repeats::Int, mod::Module; hint::Bool=false)
-    completions, partial, should_complete = complete_line_named(s.p.complete, s, mod; hint)
+    completions, reg, should_complete = complete_line_named(s.p.complete, s, mod; hint)
     isempty(completions) && return false
     if !should_complete
         # should_complete is false for cases where we only want to show
         # a list of possible completions but not complete, e.g. foo(\t
         show_completions(s, completions)
     elseif length(completions) == 1
         # Replace word by completion
-        prev_pos = position(s)
         push_undo(s)
-        edit_splice!(s, (prev_pos - sizeof(partial)) => prev_pos, completions[1].completion)
+        edit_splice!(s, reg, completions[1].completion)
     else
         p = common_prefix(completions)
+        partial = content(s, reg.first => min(bufend(s), reg.first + sizeof(p)))
         if !isempty(p) && p != partial
             # All possible completions share the same prefix, so we might as
-            # well complete that
-            prev_pos = position(s)
+            # well complete that.
             push_undo(s)
-            edit_splice!(s, (prev_pos - sizeof(partial)) => prev_pos, p)
+            edit_splice!(s, reg, p)
         elseif repeats > 0
             show_completions(s, completions)
         end
@@ -830,12 +832,12 @@ function edit_move_right(m::MIState)
         refresh_line(s)
         return true
     else
-        completions, partial, should_complete = complete_line(s.p.complete, s, m.active_module)
-        if should_complete && eof(buf) && length(completions) == 1 && length(partial) > 1
+        completions, reg, should_complete = complete_line(s.p.complete, s, m.active_module)
+        if should_complete && eof(buf) && length(completions) == 1 && reg.second - reg.first > 1
             # Replace word by completion
             prev_pos = position(s)
             push_undo(s)
-            edit_splice!(s, (prev_pos - sizeof(partial)) => prev_pos, completions[1].completion)
+            edit_splice!(s, (prev_pos - reg.second + reg.first) => prev_pos, completions[1].completion)
             refresh_line(state(s))
             return true
         else
@@ -2255,12 +2257,12 @@ setmodifiers!(c) = nothing
 
 # Search Mode completions
 function complete_line(s::SearchState, repeats, mod::Module; hint::Bool=false)
-    completions, partial, should_complete = complete_line(s.histprompt.complete, s, mod; hint)
+    completions, reg, should_complete = complete_line(s.histprompt.complete, s, mod; hint)
     # For now only allow exact completions in search mode
     if length(completions) == 1
         prev_pos = position(s)
         push_undo(s)
-        edit_splice!(s, (prev_pos - sizeof(partial)) => prev_pos, completions[1].completion)
+        edit_splice!(s, (prev_pos - reg.second - reg.first) => prev_pos, completions[1].completion)
         return true
     end
     return false

stdlib/REPL/src/REPL.jl

@@ -86,6 +86,7 @@ import .LineEdit:
     PromptState,
     mode_idx
 
+include("SyntaxUtil.jl")
 include("REPLCompletions.jl")
 using .REPLCompletions
 
@@ -799,27 +800,29 @@ end
 
 beforecursor(buf::IOBuffer) = String(buf.data[1:buf.ptr-1])
 
+# Convert inclusive-inclusive 1-based char indexing to inclusive-exclusive byte Region.
+to_region(s, r) = first(r)-1 => (length(r) > 0 ? nextind(s, last(r))-1 : first(r)-1)
+
 function complete_line(c::REPLCompletionProvider, s::PromptState, mod::Module; hint::Bool=false)
-    partial = beforecursor(s.input_buffer)
     full = LineEdit.input_string(s)
-    ret, range, should_complete = completions(full, lastindex(partial), mod, c.modifiers.shift, hint)
+    ret, range, should_complete = completions(full, thisind(full, position(s)), mod, c.modifiers.shift, hint)
+    range = to_region(full, range)
     c.modifiers = LineEdit.Modifiers()
-    return unique!(LineEdit.NamedCompletion[named_completion(x) for x in ret]), partial[range], should_complete
+    return unique!(LineEdit.NamedCompletion[named_completion(x) for x in ret]), range, should_complete
 end
 
 function complete_line(c::ShellCompletionProvider, s::PromptState; hint::Bool=false)
-    # First parse everything up to the current position
-    partial = beforecursor(s.input_buffer)
     full = LineEdit.input_string(s)
-    ret, range, should_complete = shell_completions(full, lastindex(partial), hint)
-    return unique!(LineEdit.NamedCompletion[named_completion(x) for x in ret]), partial[range], should_complete
+    ret, range, should_complete = shell_completions(full, thisind(full, position(s)), hint)
+    range = to_region(full, range)
+    return unique!(LineEdit.NamedCompletion[named_completion(x) for x in ret]), range, should_complete
 end
 
 function complete_line(c::LatexCompletions, s; hint::Bool=false)
-    partial = beforecursor(LineEdit.buffer(s))
     full = LineEdit.input_string(s)::String
-    ret, range, should_complete = bslash_completions(full, lastindex(partial), hint)[2]
-    return unique!(LineEdit.NamedCompletion[named_completion(x) for x in ret]), partial[range], should_complete
+    ret, range, should_complete = bslash_completions(full, thisind(full, position(s)), hint)[2]
+    range = to_region(full, range)
+    return unique!(LineEdit.NamedCompletion[named_completion(x) for x in ret]), range, should_complete
 end
 
 with_repl_linfo(f, repl) = f(outstream(repl))