From 8e02c2c0218b4e233f14d130f7d339128886f763 Mon Sep 17 00:00:00 2001
From: "Kartik K. Agaram" <vc@akkartik.com>
Date: Sat, 13 May 2023 17:02:10 -0700
Subject: [PATCH] bugfix: searching files containing unicode

Before this change the cursor was moving, but not being highlighted
properly when the cursor line contained unicode before the cursor.
---
 search.lua            | 70 ++++++++++++++++++++++++++-----------------
 source_text_tests.lua | 20 ++++++-------
 text_tests.lua        | 20 ++++++-------
 3 files changed, 62 insertions(+), 48 deletions(-)

diff --git a/search.lua b/search.lua
index 7d3d335..c9af187 100644
--- a/search.lua
+++ b/search.lua
@@ -18,38 +18,45 @@ end
 
 function Text.search_next(State)
   -- search current line from cursor
-  local pos = find(State.lines[State.cursor1.line].data, State.search_term, State.cursor1.pos, --[[literal]] true)
-  if pos then
-    State.cursor1.pos = pos
+  local curr_pos = State.cursor1.pos
+  local curr_line = State.lines[State.cursor1.line].data
+  local curr_offset = Text.offset(curr_line, curr_pos)
+  local offset = find(curr_line, State.search_term, curr_offset, --[[literal]] true)
+  if offset then
+    State.cursor1.pos = utf8.len(curr_line, 1, offset)
   end
-  if pos == nil then
+  if offset == nil then
     -- search lines below cursor
     for i=State.cursor1.line+1,#State.lines do
-      pos = find(State.lines[i].data, State.search_term, --[[from start]] nil, --[[literal]] true)
-      if pos then
-        State.cursor1 = {line=i, pos=pos}
+      local curr_line = State.lines[i].data
+      offset = find(curr_line, State.search_term, --[[from start]] nil, --[[literal]] true)
+      if offset then
+        State.cursor1 = {line=i, pos=utf8.len(curr_line, 1, offset)}
         break
       end
     end
   end
-  if pos == nil then
+  if offset == nil then
     -- wrap around
     for i=1,State.cursor1.line-1 do
-      pos = find(State.lines[i].data, State.search_term, --[[from start]] nil, --[[literal]] true)
-      if pos then
-        State.cursor1 = {line=i, pos=pos}
+      local curr_line = State.lines[i].data
+      offset = find(curr_line, State.search_term, --[[from start]] nil, --[[literal]] true)
+      if offset then
+        State.cursor1 = {line=i, pos=utf8.len(curr_line, 1, offset)}
         break
       end
     end
   end
-  if pos == nil then
+  if offset == nil then
     -- search current line until cursor
-    pos = find(State.lines[State.cursor1.line].data, State.search_term, --[[from start]] nil, --[[literal]] true)
+    local curr_line = State.lines[State.cursor1.line].data
+    offset = find(curr_line, State.search_term, --[[from start]] nil, --[[literal]] true)
+    local pos = utf8.len(curr_line, 1, offset)
     if pos and pos < State.cursor1.pos then
       State.cursor1.pos = pos
     end
   end
-  if pos == nil then
+  if offset == nil then
     State.cursor1.line = State.search_backup.cursor.line
     State.cursor1.pos = State.search_backup.cursor.pos
     State.screen_top1.line = State.search_backup.screen_top.line
@@ -64,38 +71,45 @@ end
 
 function Text.search_previous(State)
   -- search current line before cursor
-  local pos = rfind(State.lines[State.cursor1.line].data, State.search_term, State.cursor1.pos-1, --[[literal]] true)
-  if pos then
-    State.cursor1.pos = pos
+  local curr_pos = State.cursor1.pos
+  local curr_line = State.lines[State.cursor1.line].data
+  local curr_offset = Text.offset(curr_line, curr_pos)
+  local offset = rfind(curr_line, State.search_term, curr_offset-1, --[[literal]] true)
+  if offset then
+    State.cursor1.pos = utf8.len(curr_line, 1, offset)
   end
-  if pos == nil then
+  if offset == nil then
     -- search lines above cursor
     for i=State.cursor1.line-1,1,-1 do
-      pos = rfind(State.lines[i].data, State.search_term, --[[from end]] nil, --[[literal]] true)
-      if pos then
-        State.cursor1 = {line=i, pos=pos}
+      local curr_line = State.lines[i].data
+      offset = rfind(curr_line, State.search_term, --[[from end]] nil, --[[literal]] true)
+      if offset then
+        State.cursor1 = {line=i, pos=utf8.len(curr_line, 1, offset)}
         break
       end
     end
   end
-  if pos == nil then
+  if offset == nil then
     -- wrap around
     for i=#State.lines,State.cursor1.line+1,-1 do
-      pos = rfind(State.lines[i].data, State.search_term, --[[from end]] nil, --[[literal]] true)
-      if pos then
-        State.cursor1 = {line=i, pos=pos}
+      local curr_line = State.lines[i].data
+      offset = rfind(curr_line, State.search_term, --[[from end]] nil, --[[literal]] true)
+      if offset then
+        State.cursor1 = {line=i, pos=utf8.len(curr_line, 1, offset)}
         break
       end
     end
   end
-  if pos == nil then
+  if offset == nil then
     -- search current line after cursor
-    pos = rfind(State.lines[State.cursor1.line].data, State.search_term, --[[from end]] nil, --[[literal]] true)
+    local curr_line = State.lines[State.cursor1.line].data
+    offset = rfind(curr_line, State.search_term, --[[from end]] nil, --[[literal]] true)
+    local pos = utf8.len(curr_line, 1, offset)
     if pos and pos > State.cursor1.pos then
       State.cursor1.pos = pos
     end
   end
-  if pos == nil then
+  if offset == nil then
     State.cursor1.line = State.search_backup.cursor.line
     State.cursor1.pos = State.search_backup.cursor.pos
     State.screen_top1.line = State.search_backup.screen_top.line
diff --git a/source_text_tests.lua b/source_text_tests.lua
index bb3387f..11e7613 100644
--- a/source_text_tests.lua
+++ b/source_text_tests.lua
@@ -1955,7 +1955,7 @@ end
 function test_search()
   App.screen.init{width=120, height=60}
   Editor_state = edit.initialize_test_state()
-  Editor_state.lines = load_array{'```lines', '```', 'def', 'ghi', 'deg'}
+  Editor_state.lines = load_array{'```lines', '```', 'def', 'ghi', '’deg'}  -- contains unicode quote in final line
   Text.redraw_all(Editor_state)
   Editor_state.cursor1 = {line=1, pos=1}
   Editor_state.screen_top1 = {line=1, pos=1}
@@ -1976,15 +1976,15 @@ function test_search()
   edit.run_after_keychord(Editor_state, 'down')
   edit.run_after_keychord(Editor_state, 'return')
   check_eq(Editor_state.cursor1.line, 4, '2/cursor:line')
-  check_eq(Editor_state.cursor1.pos, 1, '2/cursor:pos')
+  check_eq(Editor_state.cursor1.pos, 2, '2/cursor:pos')
 end
 
 function test_search_upwards()
   App.screen.init{width=120, height=60}
   Editor_state = edit.initialize_test_state()
-  Editor_state.lines = load_array{'abc abd'}
+  Editor_state.lines = load_array{'’abc', 'abd'}  -- contains unicode quote
   Text.redraw_all(Editor_state)
-  Editor_state.cursor1 = {line=1, pos=2}
+  Editor_state.cursor1 = {line=2, pos=1}
   Editor_state.screen_top1 = {line=1, pos=1}
   Editor_state.screen_bottom1 = {}
   edit.draw(Editor_state)
@@ -1994,15 +1994,15 @@ function test_search_upwards()
   -- search for previous occurrence
   edit.run_after_keychord(Editor_state, 'up')
   check_eq(Editor_state.cursor1.line, 1, '2/cursor:line')
-  check_eq(Editor_state.cursor1.pos, 1, '2/cursor:pos')
+  check_eq(Editor_state.cursor1.pos, 2, '2/cursor:pos')
 end
 
 function test_search_wrap()
   App.screen.init{width=120, height=60}
   Editor_state = edit.initialize_test_state()
-  Editor_state.lines = load_array{'abc'}
+  Editor_state.lines = load_array{'’abc', 'def'}  -- contains unicode quote in first line
   Text.redraw_all(Editor_state)
-  Editor_state.cursor1 = {line=1, pos=3}
+  Editor_state.cursor1 = {line=2, pos=1}
   Editor_state.screen_top1 = {line=1, pos=1}
   Editor_state.screen_bottom1 = {}
   edit.draw(Editor_state)
@@ -2012,13 +2012,13 @@ function test_search_wrap()
   edit.run_after_keychord(Editor_state, 'return')
   -- cursor wraps
   check_eq(Editor_state.cursor1.line, 1, '1/cursor:line')
-  check_eq(Editor_state.cursor1.pos, 1, '1/cursor:pos')
+  check_eq(Editor_state.cursor1.pos, 2, '1/cursor:pos')
 end
 
 function test_search_wrap_upwards()
   App.screen.init{width=120, height=60}
   Editor_state = edit.initialize_test_state()
-  Editor_state.lines = load_array{'abc abd'}
+  Editor_state.lines = load_array{'abc ’abd'}  -- contains unicode quote
   Text.redraw_all(Editor_state)
   Editor_state.cursor1 = {line=1, pos=1}
   Editor_state.screen_top1 = {line=1, pos=1}
@@ -2030,5 +2030,5 @@ function test_search_wrap_upwards()
   edit.run_after_keychord(Editor_state, 'up')
   -- cursor wraps
   check_eq(Editor_state.cursor1.line, 1, '1/cursor:line')
-  check_eq(Editor_state.cursor1.pos, 5, '1/cursor:pos')
+  check_eq(Editor_state.cursor1.pos, 6, '1/cursor:pos')
 end
diff --git a/text_tests.lua b/text_tests.lua
index cfdffdd..8b646a4 100644
--- a/text_tests.lua
+++ b/text_tests.lua
@@ -1985,7 +1985,7 @@ end
 function test_search()
   App.screen.init{width=120, height=60}
   Editor_state = edit.initialize_test_state()
-  Editor_state.lines = load_array{'```lines', '```', 'def', 'ghi', 'deg'}
+  Editor_state.lines = load_array{'```lines', '```', 'def', 'ghi', '’deg'}  -- contains unicode quote in final line
   Text.redraw_all(Editor_state)
   Editor_state.cursor1 = {line=1, pos=1}
   Editor_state.screen_top1 = {line=1, pos=1}
@@ -2006,15 +2006,15 @@ function test_search()
   edit.run_after_keychord(Editor_state, 'down')
   edit.run_after_keychord(Editor_state, 'return')
   check_eq(Editor_state.cursor1.line, 4, '2/cursor:line')
-  check_eq(Editor_state.cursor1.pos, 1, '2/cursor:pos')
+  check_eq(Editor_state.cursor1.pos, 2, '2/cursor:pos')
 end
 
 function test_search_upwards()
   App.screen.init{width=120, height=60}
   Editor_state = edit.initialize_test_state()
-  Editor_state.lines = load_array{'abc abd'}
+  Editor_state.lines = load_array{'’abc', 'abd'}  -- contains unicode quote
   Text.redraw_all(Editor_state)
-  Editor_state.cursor1 = {line=1, pos=2}
+  Editor_state.cursor1 = {line=2, pos=1}
   Editor_state.screen_top1 = {line=1, pos=1}
   Editor_state.screen_bottom1 = {}
   edit.draw(Editor_state)
@@ -2024,15 +2024,15 @@ function test_search_upwards()
   -- search for previous occurrence
   edit.run_after_keychord(Editor_state, 'up')
   check_eq(Editor_state.cursor1.line, 1, '2/cursor:line')
-  check_eq(Editor_state.cursor1.pos, 1, '2/cursor:pos')
+  check_eq(Editor_state.cursor1.pos, 2, '2/cursor:pos')
 end
 
 function test_search_wrap()
   App.screen.init{width=120, height=60}
   Editor_state = edit.initialize_test_state()
-  Editor_state.lines = load_array{'abc'}
+  Editor_state.lines = load_array{'’abc', 'def'}  -- contains unicode quote in first line
   Text.redraw_all(Editor_state)
-  Editor_state.cursor1 = {line=1, pos=3}
+  Editor_state.cursor1 = {line=2, pos=1}
   Editor_state.screen_top1 = {line=1, pos=1}
   Editor_state.screen_bottom1 = {}
   edit.draw(Editor_state)
@@ -2042,13 +2042,13 @@ function test_search_wrap()
   edit.run_after_keychord(Editor_state, 'return')
   -- cursor wraps
   check_eq(Editor_state.cursor1.line, 1, '1/cursor:line')
-  check_eq(Editor_state.cursor1.pos, 1, '1/cursor:pos')
+  check_eq(Editor_state.cursor1.pos, 2, '1/cursor:pos')
 end
 
 function test_search_wrap_upwards()
   App.screen.init{width=120, height=60}
   Editor_state = edit.initialize_test_state()
-  Editor_state.lines = load_array{'abc abd'}
+  Editor_state.lines = load_array{'abc ’abd'}  -- contains unicode quote
   Text.redraw_all(Editor_state)
   Editor_state.cursor1 = {line=1, pos=1}
   Editor_state.screen_top1 = {line=1, pos=1}
@@ -2060,5 +2060,5 @@ function test_search_wrap_upwards()
   edit.run_after_keychord(Editor_state, 'up')
   -- cursor wraps
   check_eq(Editor_state.cursor1.line, 1, '1/cursor:line')
-  check_eq(Editor_state.cursor1.pos, 5, '1/cursor:pos')
+  check_eq(Editor_state.cursor1.pos, 6, '1/cursor:pos')
 end