fix: Crashes when getting tabs into text editor (#2008)

This time it may be for good. 


### Implementation description
As requested I added the preprocessing functionality into the pattern
editor. I had to duplicate a few functions and update the c++ library to
version 20. but now I can make sure the pattern editor doesn't see a tab
again. I also removed the preprocessor from where it was before because
it is not needed anymore. The changes were tested using a file that used
tabs for all its white space. The file was pasted into the pattern
editor and imported as well. I both cases no crashes occurred and the
files had no tabs on them.

lib/third_party/imgui/ColorTextEditor/CMakeLists.txt

@@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.16)
 # https://github.com/BalazsJako/ImGuiColorTextEdit
 project(imgui_color_text_editor)
 
-set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD 20)
 
 if (NOT IMHEX_EXTERNAL_PLUGIN_BUILD)
     add_library(imgui_color_text_editor OBJECT

lib/third_party/imgui/ColorTextEditor/include/TextEditor.h

@@ -343,6 +343,11 @@ public:
     }
     void SetOverwrite(bool aValue) { mOverwrite = aValue; }
 
+    std::string ReplaceStrings(std::string string, const std::string &search, const std::string &replace);
+    std::vector<std::string> SplitString(const std::string &string, const std::string &delimiter, bool removeEmpty);
+    std::string ReplaceTabsWithSpaces(const std::string& string, uint32_t tabSize);
+    std::string PreprocessText(const std::string &code);
+
 	void SetReadOnly(bool aValue);
 	bool IsReadOnly() const { return mReadOnly; }
 	bool IsTextChanged() const { return mTextChanged; }

lib/third_party/imgui/ColorTextEditor/source/TextEditor.cpp

@@ -1228,7 +1228,8 @@ void TextEditor::Render(const char *aTitle, const ImVec2 &aSize, bool aBorder) {
 void TextEditor::SetText(const std::string &aText) {
     mLines.resize(1);
     mLines[0].clear();
-    for (auto chr : aText) {
+    std::string text = PreprocessText(aText);
+    for (auto chr : text) {
         if (chr == '\r') {
             // ignore the carriage return character
         } else if (chr == '\n')
@@ -1256,7 +1257,7 @@ void TextEditor::SetTextLines(const std::vector<std::string> &aLines) {
         mLines.resize(aLines.size());
 
         for (size_t i = 0; i < aLines.size(); ++i) {
-            const std::string &aLine = aLines[i];
+            const std::string &aLine = PreprocessText(aLines[i]);
 
             mLines[i].reserve(aLine.size());
             for (size_t j = 0; j < aLine.size(); ++j)
@@ -1546,8 +1547,9 @@ void TextEditor::InsertText(const char *aValue) {
     auto pos       = GetActualCursorCoordinates();
     auto start     = std::min(pos, mState.mSelectionStart);
     int totalLines = pos.mLine - start.mLine;
+    auto text = PreprocessText(aValue);
 
-    totalLines += InsertTextAt(pos, aValue);
+    totalLines += InsertTextAt(pos, text.c_str());
 
     SetSelection(pos, pos);
     SetCursorPosition(pos);
@@ -2008,12 +2010,82 @@ void TextEditor::Cut() {
     }
 }
 
+std::string TextEditor::ReplaceStrings(std::string string, const std::string &search, const std::string &replace) {
+    if (search.empty())
+        return string;
+
+    std::size_t pos = 0;
+    while ((pos = string.find(search, pos)) != std::string::npos) {
+        string.replace(pos, search.size(), replace);
+        pos += replace.size();
+    }
+
+    return string;
+}
+
+std::vector<std::string> TextEditor::SplitString(const std::string &string, const std::string &delimiter, bool removeEmpty) {
+    if (delimiter.empty()) {
+        return { string };
+    }
+
+    std::vector<std::string> result;
+
+    size_t start = 0, end = 0;
+    while ((end = string.find(delimiter, start)) != std::string::npos) {
+        size_t size = end - start;
+        if (start + size > string.length())
+            break;
+
+        auto token = string.substr(start, end - start);
+        start = end + delimiter.length();
+        result.emplace_back(std::move(token));
+    }
+
+    result.emplace_back(string.substr(start));
+
+    if (removeEmpty)
+        std::erase_if(result, [](const auto &string) { return string.empty(); });
+
+    return result;
+}
+
+
+std::string TextEditor::ReplaceTabsWithSpaces(const std::string& string, uint32_t tabSize) {
+    if (tabSize == 0)
+        return string;
+
+    auto stringVector = SplitString(string, "\n", false);
+    std::string result;
+    for (auto &line : stringVector) {
+        std::size_t pos = 0;
+        while ((pos = line.find('\t', pos)) != std::string::npos) {
+            auto spaces = tabSize - (pos % tabSize);
+            line.replace(pos, 1, std::string(spaces, ' '));
+            pos += tabSize;
+        }
+        result += line + "\n";
+    }
+    return result;
+}
+
+
+std::string TextEditor::PreprocessText(const std::string &code) {
+    std::string result = ReplaceStrings(code, "\r\n", "\n");
+    result = ReplaceStrings(result, "\r", "\n");
+    result = ReplaceTabsWithSpaces(result, 4);
+    while (result.ends_with('\n'))
+        result.pop_back();
+    return result;
+}
+
 void TextEditor::Paste() {
     if (IsReadOnly())
         return;
 
     auto clipText = ImGui::GetClipboardText();
     if (clipText != nullptr && strlen(clipText) > 0) {
+        std::string text = PreprocessText(clipText);
+
         UndoRecord u;
         u.mBefore = mState;
 
@@ -2024,10 +2096,10 @@ void TextEditor::Paste() {
             DeleteSelection();
         }
 
-        u.mAdded      = clipText;
+        u.mAdded      = text;
         u.mAddedStart = GetActualCursorCoordinates();
 
-        InsertText(clipText);
+        InsertText(text);
 
         u.mAddedEnd = GetActualCursorCoordinates();
         u.mAfter    = mState;

plugins/builtin/include/content/views/view_pattern_editor.hpp

@@ -73,7 +73,6 @@ namespace hex::plugin::builtin {
             return ImGuiWindowFlags_NoScrollbar | ImGuiWindowFlags_NoScrollWithMouse;
         }
 
-        std::string preprocessText(const std::string &code);
         void setPopupWindowHeight(u32 height) { m_popupWindowHeight = height; }
         u32 getPopupWindowHeight() const { return m_popupWindowHeight; }
 

plugins/builtin/source/content/views/view_pattern_editor.cpp

@@ -1082,7 +1082,7 @@ namespace hex::plugin::builtin {
                 if (!skipNewLine)
                     m_consoleEditor.InsertText("\n");
                 skipNewLine = false;
-                m_consoleEditor.InsertText(preprocessText(m_console->at(lineCount + i)));
+                m_consoleEditor.InsertText(m_console->at(lineCount + i));
             }
 
             m_consoleNeedsUpdate = false;
@@ -1812,7 +1812,7 @@ namespace hex::plugin::builtin {
     void ViewPatternEditor::loadPatternFile(const std::fs::path &path, prv::Provider *provider) {
         wolv::io::File file(path, wolv::io::File::Mode::Read);
         if (file.isValid()) {
-            auto code = preprocessText(file.readString());
+            auto code = file.readString();
 
             this->evaluatePattern(code, provider);
             m_textEditor.SetText(code);
@@ -1977,15 +1977,6 @@ namespace hex::plugin::builtin {
         });
     }
 
-    std::string ViewPatternEditor::preprocessText(const std::string &code) {
-        std::string result = wolv::util::replaceStrings(code, "\r\n", "\n");
-        result = wolv::util::replaceStrings(result, "\r", "\n");
-        result = wolv::util::replaceTabsWithSpaces(result, 4);
-        while (result.ends_with('\n'))
-            result.pop_back();
-        return result;
-    }
-
     void ViewPatternEditor::registerEvents() {
         RequestPatternEditorSelectionChange::subscribe(this, [this](u32 line, u32 column) {
             if (line == 0)
@@ -2009,9 +2000,8 @@ namespace hex::plugin::builtin {
         });
 
         RequestSetPatternLanguageCode::subscribe(this, [this](const std::string &code) {
-            const std::string preprocessed = preprocessText(code);
+            m_textEditor.SetText(code);
-            m_textEditor.SetText(preprocessed);
+            m_sourceCode.set(ImHexApi::Provider::get(), code);
-            m_sourceCode.set(ImHexApi::Provider::get(), preprocessed);
             m_hasUnevaluatedChanges = true;
         });
 
@@ -2272,7 +2262,7 @@ namespace hex::plugin::builtin {
             .basePath = "pattern_source_code.hexpat",
             .required = false,
             .load = [this](prv::Provider *provider, const std::fs::path &basePath, const Tar &tar) {
-                const auto sourceCode = preprocessText(tar.readString(basePath));
+                const auto sourceCode = tar.readString(basePath);
 
                 m_sourceCode.set(provider, sourceCode);