From 7c118759c027bf467222f953eec2b2132f73e2cb Mon Sep 17 00:00:00 2001
From: FIameCaster <82079841+FIameCaster@users.noreply.github.com>
Date: Mon, 13 Nov 2023 01:46:27 +0100
Subject: [PATCH 01/11] Fix empty search match causing infinite loop

---
 src/ext/searchbox.js | 4 +++-
 src/search.js        | 7 +++++--
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/ext/searchbox.js b/src/ext/searchbox.js
index 3b766dd68a4..e35d6dc4116 100644
--- a/src/ext/searchbox.js
+++ b/src/ext/searchbox.js
@@ -167,6 +167,7 @@ class SearchBox {
     updateCounter() {
         var editor = this.editor;
         var regex = editor.$search.$options.re;
+        var supportsUnicodeFlag = editor.$search.$options.$supportsUnicodeFlag;
         var all = 0;
         var before = 0;
         if (regex) {
@@ -188,7 +189,8 @@ class SearchBox {
                 if (all > MAX_COUNT)
                     break;
                 if (!m[0]) {
-                    regex.lastIndex = last += 1;
+                    regex.lastIndex = last +=
+                        supportsUnicodeFlag && (value.charCodeAt(last) ^ 0xd800) < 1024 ? 2 : 1;
                     if (last >= value.length)
                         break;
                 }
diff --git a/src/search.js b/src/search.js
index 5a0bd239b1b..9cbf4dba7a8 100644
--- a/src/search.js
+++ b/src/search.js
@@ -270,6 +270,7 @@ class Search {
             return false;
         var backwards = options.backwards == true;
         var skipCurrent = options.skipCurrent != false;
+        var supportsUnicodeFlag = options.$supportsUnicodeFlag
 
         var range = options.range;
         var start = options.start;
@@ -343,7 +344,8 @@ class Search {
                     last = m.index;
                     if (!length) {
                         if (last >= line.length) break;
-                        re.lastIndex = last += 1;
+                        re.lastIndex = last +=
+                            supportsUnicodeFlag && (line.charCodeAt(last) ^ 0xd800) < 1024 ? 2 : 1;
                     }
                     if (m.index + length > endIndex)
                         break;
@@ -369,7 +371,8 @@ class Search {
                     if (callback(row, last, row,last + length))
                         return true;
                     if (!length) {
-                        re.lastIndex = last += 1;
+                        re.lastIndex = last +=
+                            supportsUnicodeFlag && (line.charCodeAt(last) ^ 0xd800) < 1024 ? 2 : 1;
                         if (last >= line.length) return false;
                     }
                 }

From 72eab3efb902f0ca5663200b4c0003084cc22f24 Mon Sep 17 00:00:00 2001
From: FIameCaster <82079841+FIameCaster@users.noreply.github.com>
Date: Mon, 13 Nov 2023 16:27:41 +0100
Subject: [PATCH 02/11] Improve regex u flag feature detection

---
 src/lib/lang.js |  9 ---------
 src/search.js   | 20 +++++++-------------
 2 files changed, 7 insertions(+), 22 deletions(-)

diff --git a/src/lib/lang.js b/src/lib/lang.js
index 27579403639..473bcb0b2c5 100644
--- a/src/lib/lang.js
+++ b/src/lib/lang.js
@@ -180,12 +180,3 @@ exports.supportsLookbehind = function () {
     }
     return true;
 };
-
-exports.supportsUnicodeFlag = function () {
-    try {
-        new RegExp('^.$', 'u');
-    } catch (error) {
-        return false;
-    }
-    return true;
-};
diff --git a/src/search.js b/src/search.js
index 9cbf4dba7a8..af89f83575a 100644
--- a/src/search.js
+++ b/src/search.js
@@ -218,28 +218,22 @@ class Search {
 
         if (!options.needle)
             return options.re = false;
-        
-        if (options.$supportsUnicodeFlag === undefined) {
-            options.$supportsUnicodeFlag = lang.supportsUnicodeFlag();
-        }
+
+        if (!options.regExp)
+            needle = lang.escapeRegExp(needle);
+
+        var modifier = options.caseSensitive ? "gm" : "gmi";
 
         try {
             new RegExp(needle, "u");
+            options.$supportsUnicodeFlag = true;
+            modifier += "u";
         } catch (e) {
             options.$supportsUnicodeFlag = false; //left for backward compatibility with previous versions for cases like /ab\{2}/gu
         }
-        
-        if (!options.regExp)
-            needle = lang.escapeRegExp(needle);
 
         if (options.wholeWord)
             needle = addWordBoundary(needle, options);
-
-        var modifier = options.caseSensitive ? "gm" : "gmi";
-
-        if (options.$supportsUnicodeFlag) {
-            modifier += "u";
-        }
         
         options.$isMultiLine = !$disableFakeMultiline && /[\n\r]/.test(needle);
         if (options.$isMultiLine)

From 6f2b55417b950bedb04b5f3f65032d538c68def1 Mon Sep 17 00:00:00 2001
From: FIameCaster <82079841+FIameCaster@users.noreply.github.com>
Date: Tue, 14 Nov 2023 01:49:21 +0100
Subject: [PATCH 03/11] Use RegExp.unicode instead of $supportsUnicodeFlag

---
 src/ext/searchbox.js | 2 +-
 src/search.js        | 6 ++----
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/src/ext/searchbox.js b/src/ext/searchbox.js
index e35d6dc4116..939133c9847 100644
--- a/src/ext/searchbox.js
+++ b/src/ext/searchbox.js
@@ -167,7 +167,7 @@ class SearchBox {
     updateCounter() {
         var editor = this.editor;
         var regex = editor.$search.$options.re;
-        var supportsUnicodeFlag = editor.$search.$options.$supportsUnicodeFlag;
+        var supportsUnicodeFlag = regex.unicode;
         var all = 0;
         var before = 0;
         if (regex) {
diff --git a/src/search.js b/src/search.js
index af89f83575a..d09e60777ac 100644
--- a/src/search.js
+++ b/src/search.js
@@ -24,7 +24,6 @@ class Search {
      * @property {boolean} [$isMultiLine] - true, if needle has \n or \r\n
      * @property {boolean} [preserveCase]
      * @property {boolean} [preventScroll]
-     * @property {boolean} [$supportsUnicodeFlag] - internal property, determine if browser supports unicode flag
      * @property {any} [re]
      **/
     
@@ -226,10 +225,9 @@ class Search {
 
         try {
             new RegExp(needle, "u");
-            options.$supportsUnicodeFlag = true;
             modifier += "u";
         } catch (e) {
-            options.$supportsUnicodeFlag = false; //left for backward compatibility with previous versions for cases like /ab\{2}/gu
+            // left for backward compatibility with previous versions for cases like /ab\{2}/gu
         }
 
         if (options.wholeWord)
@@ -264,7 +262,7 @@ class Search {
             return false;
         var backwards = options.backwards == true;
         var skipCurrent = options.skipCurrent != false;
-        var supportsUnicodeFlag = options.$supportsUnicodeFlag
+        var supportsUnicodeFlag = re.unicode;
 
         var range = options.range;
         var start = options.start;

From 934adb5762d68663ea82192254ab894e15dd683b Mon Sep 17 00:00:00 2001
From: FIameCaster <82079841+FIameCaster@users.noreply.github.com>
Date: Tue, 14 Nov 2023 02:01:01 +0100
Subject: [PATCH 04/11] Fix mistake in last commit

---
 src/search.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/search.js b/src/search.js
index d09e60777ac..3da5c95f1d1 100644
--- a/src/search.js
+++ b/src/search.js
@@ -385,10 +385,10 @@ function addWordBoundary(needle, options) {
     let supportsLookbehind = lang.supportsLookbehind();
 
     function wordBoundary(c, firstChar = true) {
-        let wordRegExp = supportsLookbehind && options.$supportsUnicodeFlag ? new RegExp("[\\p{L}\\p{N}_]","u") : new RegExp("\\w");
+        let wordRegExp = supportsLookbehind && options.regExp.unicode ? new RegExp("[\\p{L}\\p{N}_]","u") : new RegExp("\\w");
 
         if (wordRegExp.test(c) || options.regExp) {
-            if (supportsLookbehind && options.$supportsUnicodeFlag) {
+            if (supportsLookbehind && options.regExp.unicode) {
                 if (firstChar) return "(?<=^|[^\\p{L}\\p{N}_])";
                 return "(?=[^\\p{L}\\p{N}_]|$)";
             }

From 8b2b6851acd23ebd2e928e0720661d1e9cd325de Mon Sep 17 00:00:00 2001
From: FIameCaster <82079841+FIameCaster@users.noreply.github.com>
Date: Wed, 15 Nov 2023 13:48:53 +0100
Subject: [PATCH 05/11] Fix failing tests

---
 src/search.js | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/search.js b/src/search.js
index 3da5c95f1d1..2d1ead3f8e2 100644
--- a/src/search.js
+++ b/src/search.js
@@ -24,6 +24,7 @@ class Search {
      * @property {boolean} [$isMultiLine] - true, if needle has \n or \r\n
      * @property {boolean} [preserveCase]
      * @property {boolean} [preventScroll]
+     * @property {boolean} [$supportsUnicodeFlag] - internal property, determine if browser supports unicode flag
      * @property {any} [re]
      **/
     
@@ -225,9 +226,10 @@ class Search {
 
         try {
             new RegExp(needle, "u");
+            options.$supportsUnicodeFlag = true;
             modifier += "u";
         } catch (e) {
-            // left for backward compatibility with previous versions for cases like /ab\{2}/gu
+            options.$supportsUnicodeFlag = false; //left for backward compatibility with previous versions for cases like /ab\{2}/gu
         }
 
         if (options.wholeWord)
@@ -385,10 +387,10 @@ function addWordBoundary(needle, options) {
     let supportsLookbehind = lang.supportsLookbehind();
 
     function wordBoundary(c, firstChar = true) {
-        let wordRegExp = supportsLookbehind && options.regExp.unicode ? new RegExp("[\\p{L}\\p{N}_]","u") : new RegExp("\\w");
+        let wordRegExp = supportsLookbehind && options.$supportsUnicodeFlag ? new RegExp("[\\p{L}\\p{N}_]","u") : new RegExp("\\w");
 
         if (wordRegExp.test(c) || options.regExp) {
-            if (supportsLookbehind && options.regExp.unicode) {
+            if (supportsLookbehind && options.$supportsUnicodeFlag) {
                 if (firstChar) return "(?<=^|[^\\p{L}\\p{N}_])";
                 return "(?=[^\\p{L}\\p{N}_]|$)";
             }

From e09786e2d0a612099b10c2f487ed9352a63876ed Mon Sep 17 00:00:00 2001
From: FIameCaster <82079841+FIameCaster@users.noreply.github.com>
Date: Thu, 23 Nov 2023 12:59:41 +0100
Subject: [PATCH 06/11] Add new unicode mode tests

---
 src/ext/searchbox.js |  3 +--
 src/lib/lang.js      |  4 ++++
 src/search.js        |  6 ++----
 src/search_test.js   | 33 +++++++++++++++++++++++++++++++++
 4 files changed, 40 insertions(+), 6 deletions(-)

diff --git a/src/ext/searchbox.js b/src/ext/searchbox.js
index 939133c9847..741768ac4be 100644
--- a/src/ext/searchbox.js
+++ b/src/ext/searchbox.js
@@ -189,8 +189,7 @@ class SearchBox {
                 if (all > MAX_COUNT)
                     break;
                 if (!m[0]) {
-                    regex.lastIndex = last +=
-                        supportsUnicodeFlag && (value.charCodeAt(last) ^ 0xd800) < 1024 ? 2 : 1;
+                    regex.lastIndex = last += lang.skipEmptyMatch(value, last, supportsUnicodeFlag);
                     if (last >= value.length)
                         break;
                 }
diff --git a/src/lib/lang.js b/src/lib/lang.js
index 473bcb0b2c5..85379b582fd 100644
--- a/src/lib/lang.js
+++ b/src/lib/lang.js
@@ -180,3 +180,7 @@ exports.supportsLookbehind = function () {
     }
     return true;
 };
+
+exports.skipEmptyMatch = function(line, last, supportsUnicodeFlag) {
+    return supportsUnicodeFlag && line.codePointAt(last) > 0xffff ? 2 : 1
+}
diff --git a/src/search.js b/src/search.js
index 2d1ead3f8e2..81672e3d931 100644
--- a/src/search.js
+++ b/src/search.js
@@ -338,8 +338,7 @@ class Search {
                     last = m.index;
                     if (!length) {
                         if (last >= line.length) break;
-                        re.lastIndex = last +=
-                            supportsUnicodeFlag && (line.charCodeAt(last) ^ 0xd800) < 1024 ? 2 : 1;
+                        re.lastIndex = last += lang.skipEmptyMatch(line, last, supportsUnicodeFlag);
                     }
                     if (m.index + length > endIndex)
                         break;
@@ -365,8 +364,7 @@ class Search {
                     if (callback(row, last, row,last + length))
                         return true;
                     if (!length) {
-                        re.lastIndex = last +=
-                            supportsUnicodeFlag && (line.charCodeAt(last) ^ 0xd800) < 1024 ? 2 : 1;
+                        re.lastIndex = last += lang.skipEmptyMatch(line, last, supportsUnicodeFlag);
                         if (last >= line.length) return false;
                     }
                 }
diff --git a/src/search_test.js b/src/search_test.js
index 75e6a93e320..51e3b49ec7d 100644
--- a/src/search_test.js
+++ b/src/search_test.js
@@ -9,6 +9,7 @@ var MockRenderer = require("./test/mockrenderer").MockRenderer;
 var Editor = require("./editor").Editor;
 var Search = require("./search").Search;
 var assert = require("./test/assertions");
+var Range = require("./range").Range;
 
 module.exports = {
     "test: configure the search object" : function() {
@@ -172,6 +173,38 @@ module.exports = {
         assert.position(range.end, 1, 6);
     },
 
+    "test: return to unicode mode when possible": function() {
+        var session = new EditSession(["𝓕oo"]);
+
+        var search = new Search().set({
+            needle: "}",
+            regExp: true
+        });
+
+        search.find(session);
+        search.set({
+            needle: "."
+        });
+
+        var range = search.find(session);
+        assert.position(range.start, 0, 0);
+        assert.position(range.end, 0, 2);
+    },
+
+    "test: empty match before surrogate pair": function() {
+        var session = new EditSession(["𝓕oo"]);
+
+        var search = new Search().set({
+            needle: "()",
+            regExp: true,
+            start: new Range(0, 0, 0, 0)
+        });
+
+        var range = search.find(session);
+        assert.position(range.start, 0, 2);
+        assert.position(range.end, 0, 2);
+    },
+
     "test: find backwards": function() {
         var session = new EditSession(["juhu juhu juhu juhu"]);
         session.getSelection().moveCursorTo(0, 10);

From cd251522ffeeca6e1eeeae5fc5a10ebc0b3e3fb1 Mon Sep 17 00:00:00 2001
From: FIameCaster <82079841+FIameCaster@users.noreply.github.com>
Date: Thu, 23 Nov 2023 13:09:36 +0100
Subject: [PATCH 07/11] Add missing semi colons

---
 src/lib/lang.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/lib/lang.js b/src/lib/lang.js
index 85379b582fd..950fb271a54 100644
--- a/src/lib/lang.js
+++ b/src/lib/lang.js
@@ -182,5 +182,5 @@ exports.supportsLookbehind = function () {
 };
 
 exports.skipEmptyMatch = function(line, last, supportsUnicodeFlag) {
-    return supportsUnicodeFlag && line.codePointAt(last) > 0xffff ? 2 : 1
-}
+    return supportsUnicodeFlag && line.codePointAt(last) > 0xffff ? 2 : 1;
+};

From 93f5774db106c8af5fafa0bc04ec9f92930a71f6 Mon Sep 17 00:00:00 2001
From: FIameCaster <82079841+FIameCaster@users.noreply.github.com>
Date: Sun, 18 Aug 2024 01:05:00 +0200
Subject: [PATCH 08/11] Add penalty on case mismatch

---
 src/autocomplete.js | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/src/autocomplete.js b/src/autocomplete.js
index 520c74ec070..588d758b3aa 100644
--- a/src/autocomplete.js
+++ b/src/autocomplete.js
@@ -1029,7 +1029,6 @@ class FilteredList {
     
     filterCompletions(items, needle) {
         var results = [];
-        var upper = needle.toUpperCase();
         var lower = needle.toLowerCase();
         loop: for (var i = 0, item; item = items[i]; i++) {
             var caption = (!this.ignoreCaption && item.caption) || item.value || item.snippet;
@@ -1043,23 +1042,32 @@ class FilteredList {
                 if (needle !== caption.substr(0, needle.length))
                     continue loop;
             } else {
+                var captionLower = caption.toLowerCase();
                 /**
                  * It is for situation then, for example, we find some like 'tab' in item.value="Check the table"
                  * and want to see "Check the TABle" but see "Check The tABle".
                  */
-                var fullMatchIndex = caption.toLowerCase().indexOf(lower);
+                var fullMatchIndex = captionLower.indexOf(lower);
                 if (fullMatchIndex > -1) {
                     penalty = fullMatchIndex;
+                    for (var j = 0; j < needle.length; j++) {
+                        index = fullMatchIndex + j
+                        // Adding a penalty on case mismatch
+                        if ((lower[j] == needle[j]) != (captionLower[index] == caption[index])) {
+                            penalty += 3;
+                        }
+                    }
                 } else {
                     // caption char iteration is faster in Chrome but slower in Firefox, so lets use indexOf
                     for (var j = 0; j < needle.length; j++) {
-                        // TODO add penalty on case mismatch
-                        var i1 = caption.indexOf(lower[j], lastIndex + 1);
-                        var i2 = caption.indexOf(upper[j], lastIndex + 1);
-                        index = (i1 >= 0) ? ((i2 < 0 || i1 < i2) ? i1 : i2) : i2;
+                        index = captionLower.indexOf(lower[j], lastIndex + 1);
                         if (index < 0)
                             continue loop;
                         distance = index - lastIndex - 1;
+                        // Adding a penalty on case mismatch
+                        if ((lower[j] == needle[j]) != (captionLower[index] == caption[index])) {
+                            penalty += 3;
+                        }
                         if (distance > 0) {
                             // first char mismatch should be more sensitive
                             if (lastIndex === -1)

From 2d70b82a348aa4386887aed66040e13fc429a5c8 Mon Sep 17 00:00:00 2001
From: FIameCaster <82079841+FIameCaster@users.noreply.github.com>
Date: Sun, 18 Aug 2024 01:28:06 +0200
Subject: [PATCH 09/11] Add missing semi

---
 src/autocomplete.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/autocomplete.js b/src/autocomplete.js
index 588d758b3aa..1076c157da6 100644
--- a/src/autocomplete.js
+++ b/src/autocomplete.js
@@ -1051,7 +1051,7 @@ class FilteredList {
                 if (fullMatchIndex > -1) {
                     penalty = fullMatchIndex;
                     for (var j = 0; j < needle.length; j++) {
-                        index = fullMatchIndex + j
+                        index = fullMatchIndex + j;
                         // Adding a penalty on case mismatch
                         if ((lower[j] == needle[j]) != (captionLower[index] == caption[index])) {
                             penalty += 3;

From 9518235a85fa9f925b21f0134d20ca3615ca598f Mon Sep 17 00:00:00 2001
From: FIameCaster <82079841+FIameCaster@users.noreply.github.com>
Date: Sun, 18 Aug 2024 17:27:24 +0200
Subject: [PATCH 10/11] Add test with case mismatch

---
 src/autocomplete_test.js | 45 +++++++++++++++++++++++++++++++++++++++-
 1 file changed, 44 insertions(+), 1 deletion(-)

diff --git a/src/autocomplete_test.js b/src/autocomplete_test.js
index b48ec080b21..63b950fdc98 100644
--- a/src/autocomplete_test.js
+++ b/src/autocomplete_test.js
@@ -1542,7 +1542,50 @@ module.exports = {
 
         // Popup should be closed now
         assert.equal(completer.popup.isOpen, false); 
-    }
+    },
+    "test: penalty on case mismatch": function (done) {
+        var editor = initEditor("");
+        editor.completers = [
+            {
+                getCompletions: function (editor, session, pos, prefix, callback) {
+                    var completions = [
+                        {
+                            caption: "array",
+                            value: "array"
+                        }, {
+                            caption: "aRray",
+                            value: "aRray"
+                        }
+                    ];
+                    callback(null, completions);
+                }
+            }
+        ];
+        
+        sendKey("R");
+        var popup = editor.completer.popup;
+
+        check(function() {
+            assert.equal(popup.data.length, 2);
+            assert.equal(popup.container.querySelector(".ace_selected").textContent.trim(), "aRray");
+
+            sendKey("y");
+
+            check(function() {
+                assert.equal(popup.container.querySelector(".ace_selected").textContent.trim(), "aRray");
+
+                editor.destroy();
+                editor.container.remove();
+                done();
+            });
+        });
+    
+        function check(callback) {
+            setTimeout(function wait() {
+                callback();
+            }, 10);
+        }
+    },
 };
 
 if (typeof module !== "undefined" && module === require.main) {

From 3053e0df093cd2a622269f5c7c6649bcc477f476 Mon Sep 17 00:00:00 2001
From: FIameCaster <82079841+FIameCaster@users.noreply.github.com>
Date: Sun, 18 Aug 2024 17:33:48 +0200
Subject: [PATCH 11/11] Remove trailing comma

---
 src/autocomplete_test.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/autocomplete_test.js b/src/autocomplete_test.js
index 63b950fdc98..f5ef2785ce7 100644
--- a/src/autocomplete_test.js
+++ b/src/autocomplete_test.js
@@ -1585,7 +1585,7 @@ module.exports = {
                 callback();
             }, 10);
         }
-    },
+    }
 };
 
 if (typeof module !== "undefined" && module === require.main) {