#5713 MultiEditor now keeps original line separators from the value loaded, while new separators inserted are alwas \n. Also optimized performance for very long text lines in the editor.

Author: Paweł Salawa

ChangeLog.md

@@ -118,6 +118,7 @@
 - BUGFIX: #5628 Fixed PDF export bugs (first page size too small, occasional table overlapping, page numbers cut off).
 - BUGFIX: #5691 Fixed tooltip in Database Dialog with Wayland under Linux.
 - BUGFIX: #5692 Select first inserted row after inserting multiple rows in Grid View.
+- BUGFIX: #5713 MultiEditor now keeps original line separators from the value loaded, while new separators inserted are alwas \n. Also optimized performance for very long text lines in the editor.
 
 ### 3.4.21
 - ADDED: #5466 Support foreign keys with implicit columns (proper values in FK combobox columns).

Letos/gui/dialogs/configdialog.ui

@@ -267,7 +267,7 @@
           </sizepolicy>
          </property>
          <property name="currentIndex">
-          <number>4</number>
+          <number>1</number>
          </property>
          <widget class="QWidget" name="databaseListPage">
           <layout class="QVBoxLayout" name="verticalLayout_36">
@@ -487,8 +487,8 @@
                <rect>
                 <x>0</x>
                 <y>0</y>
-                <width>439</width>
-                <height>882</height>
+                <width>679</width>
+                <height>854</height>
                </rect>
               </property>
               <layout class="QVBoxLayout" name="verticalLayout_34">
@@ -498,36 +498,52 @@
                   <string>Data browsing and editing</string>
                  </property>
                  <layout class="QGridLayout" name="gridLayout_2">
-                  <item row="0" column="0" colspan="2">
-                   <widget class="QLabel" name="rowsPerPageLabel">
+                  <item row="2" column="2">
+                   <widget class="QSpinBox" name="memPopulConfigsCheck">
+                    <property name="toolTip">
+                     <string>&lt;p&gt;Maximum number of configurations of Populate Table dialog stored in configuration. Value of 100 should be sufficient.&lt;/p&gt;</string>
+                    </property>
+                    <property name="maximum">
+                     <number>999999</number>
+                    </property>
+                    <property name="cfg" stdset="0">
+                     <string notr="true">General.PopulateHistorySize</string>
+                    </property>
+                   </widget>
+                  </item>
+                  <item row="2" column="0" colspan="2">
+                   <widget class="QLabel" name="memPopulConfigsLabel">
+                    <property name="toolTip">
+                     <string>&lt;p&gt;Maximum number of configurations of Populate Table dialog stored in configuration. Value of 100 should be sufficient.&lt;/p&gt;</string>
+                    </property>
                     <property name="text">
-                     <string>Number of data rows per page:</string>
+                     <string>Number of memorized table populating configurations</string>
                     </property>
                    </widget>
                   </item>
-                  <item row="6" column="0" colspan="3">
-                   <widget class="QCheckBox" name="useLfForMultilineBox">
+                  <item row="4" column="0" colspan="3">
+                   <widget class="QCheckBox" name="keepNullWhenEmptyCheck">
                     <property name="toolTip">
-                     <string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;Converts Unicode line separators (U+2028, U+2029) to the standard LF (&lt;span style=&quot; font-family:'monospace';&quot;&gt;\n&lt;/span&gt;) character when editing text in form view editor.&lt;/p&gt;&lt;p&gt;Disable to preserve the original characters.&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
+                     <string>&lt;p&gt;When editing a cell which used to have NULL value and entering empty string as new value, then this option determinates whether the new value should remain NULL (have this option enabled), or should it be overwritten with empty string value (have this option disabled).&lt;/p&gt;</string>
                     </property>
                     <property name="text">
-                     <string>Convert Unicode line separators to LF (\n) in form view editors</string>
+                     <string>Keep NULL value when entering empty value</string>
                     </property>
                     <property name="cfg" stdset="0">
-                     <string notr="true">General.UseLfForMultilineEditors</string>
+                     <string notr="true">General.KeepNullWhenEmptyValue</string>
                     </property>
                    </widget>
                   </item>
-                  <item row="5" column="0" colspan="3">
-                   <widget class="QCheckBox" name="useSciFormatForDoublesCheck">
+                  <item row="7" column="0">
+                   <widget class="QCheckBox" name="editCellAfterAdvancingCheck">
                     <property name="toolTip">
-                     <string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;When enabled, Letos automatically switches to scientific notation for very small real numbers (e.g. &lt;span style=&quot; font-style:italic;&quot;&gt;5.3e-21&lt;/span&gt;).&lt;br/&gt;Decimal notation is used for values whose magnitude allows a readable decimal representation.&lt;/p&gt;&lt;p&gt;When disabled, real numbers are always displayed using decimal notation, even if this results in very long fractional parts.&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
+                     <string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;Automatically starts editing the cell in the next row after moving to it with Enter.&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
                     </property>
                     <property name="text">
-                     <string>Use scientific notation for very small real numbers in the grid view</string>
+                     <string>Start editing after moving to the next row</string>
                     </property>
                     <property name="cfg" stdset="0">
-                     <string notr="true">General.UseSciFormatForDoubles</string>
+                     <string notr="true">General.DataEditAutoAdvanceEdit</string>
                     </property>
                    </widget>
                   </item>
@@ -550,7 +566,7 @@
                     </property>
                    </widget>
                   </item>
-                  <item row="7" column="0">
+                  <item row="6" column="0">
                    <widget class="QCheckBox" name="advanceRowOnEnterCheck">
                     <property name="toolTip">
                      <string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;When editing a cell, pressing Enter accepts the value and moves to the same column in the next row.&lt;br/&gt;If disabled, the selection stays on the current cell.&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
@@ -563,39 +579,23 @@
                     </property>
                    </widget>
                   </item>
-                  <item row="3" column="0" colspan="3">
-                   <widget class="QCheckBox" name="tolltipInDataViewCheck">
-                    <property name="toolTip">
-                     <string>&lt;p&gt;When this is enabled and user holds mouse pointer over a cell in any data view (query results, a table data, a view data) a tooltip will appear with details about the cell - it includes details like column data type, constraints, ROWID and others.&lt;/p&gt;</string>
-                    </property>
+                  <item row="0" column="0" colspan="2">
+                   <widget class="QLabel" name="rowsPerPageLabel">
                     <property name="text">
-                     <string>Show column and row details tooltip in data view</string>
-                    </property>
-                    <property name="cfg" stdset="0">
-                     <string notr="true">General.ShowDataViewTooltips</string>
+                     <string>Number of data rows per page:</string>
                     </property>
                    </widget>
                   </item>
-                  <item row="2" column="0" colspan="2">
-                   <widget class="QLabel" name="memPopulConfigsLabel">
+                  <item row="5" column="0" colspan="3">
+                   <widget class="QCheckBox" name="useSciFormatForDoublesCheck">
                     <property name="toolTip">
-                     <string>&lt;p&gt;Maximum number of configurations of Populate Table dialog stored in configuration. Value of 100 should be sufficient.&lt;/p&gt;</string>
+                     <string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;When enabled, Letos automatically switches to scientific notation for very small real numbers (e.g. &lt;span style=&quot; font-style:italic;&quot;&gt;5.3e-21&lt;/span&gt;).&lt;br/&gt;Decimal notation is used for values whose magnitude allows a readable decimal representation.&lt;/p&gt;&lt;p&gt;When disabled, real numbers are always displayed using decimal notation, even if this results in very long fractional parts.&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
                     </property>
                     <property name="text">
-                     <string>Number of memorized table populating configurations</string>
-                    </property>
-                   </widget>
-                  </item>
-                  <item row="2" column="2">
-                   <widget class="QSpinBox" name="memPopulConfigsCheck">
-                    <property name="toolTip">
-                     <string>&lt;p&gt;Maximum number of configurations of Populate Table dialog stored in configuration. Value of 100 should be sufficient.&lt;/p&gt;</string>
-                    </property>
-                    <property name="maximum">
-                     <number>999999</number>
+                     <string>Use scientific notation for very small real numbers in the grid view</string>
                     </property>
                     <property name="cfg" stdset="0">
-                     <string notr="true">General.PopulateHistorySize</string>
+                     <string notr="true">General.UseSciFormatForDoubles</string>
                     </property>
                    </widget>
                   </item>
@@ -612,29 +612,16 @@
                     </property>
                    </widget>
                   </item>
-                  <item row="4" column="0" colspan="3">
-                   <widget class="QCheckBox" name="keepNullWhenEmptyCheck">
-                    <property name="toolTip">
-                     <string>&lt;p&gt;When editing a cell which used to have NULL value and entering empty string as new value, then this option determinates whether the new value should remain NULL (have this option enabled), or should it be overwritten with empty string value (have this option disabled).&lt;/p&gt;</string>
-                    </property>
-                    <property name="text">
-                     <string>Keep NULL value when entering empty value</string>
-                    </property>
-                    <property name="cfg" stdset="0">
-                     <string notr="true">General.KeepNullWhenEmptyValue</string>
-                    </property>
-                   </widget>
-                  </item>
-                  <item row="8" column="0">
-                   <widget class="QCheckBox" name="editCellAfterAdvancingCheck">
+                  <item row="3" column="0" colspan="3">
+                   <widget class="QCheckBox" name="tolltipInDataViewCheck">
                     <property name="toolTip">
-                     <string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;Automatically starts editing the cell in the next row after moving to it with Enter.&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
+                     <string>&lt;p&gt;When this is enabled and user holds mouse pointer over a cell in any data view (query results, a table data, a view data) a tooltip will appear with details about the cell - it includes details like column data type, constraints, ROWID and others.&lt;/p&gt;</string>
                     </property>
                     <property name="text">
-                     <string>Start editing after moving to the next row</string>
+                     <string>Show column and row details tooltip in data view</string>
                     </property>
                     <property name="cfg" stdset="0">
-                     <string notr="true">General.DataEditAutoAdvanceEdit</string>
+                     <string notr="true">General.ShowDataViewTooltips</string>
                     </property>
                    </widget>
                   </item>

Letos/gui/multieditor/multieditorhex.cpp

@@ -1,11 +1,16 @@
 #include "multieditorhex.h"
 #include "qhexedit2/qhexedit.h"
+#include "style.h"
 #include <QVBoxLayout>
+#include <QBrush>
 
 MultiEditorHex::MultiEditorHex()
 {
     setLayout(new QVBoxLayout());
     hexEdit = new QHexEdit();
+    hexEdit->setAddressAreaColor(STYLE->extendedPalette().editorLineNumberBase().color());
+    hexEdit->setHighlightingColor(STYLE->standardPalette().highlight().color());
+    hexEdit->setSelectionColor(STYLE->standardPalette().highlight().color());
     layout()->addWidget(hexEdit);
 
     connect(hexEdit, SIGNAL(dataChanged()), this, SLOT(modificationChanged()));

Letos/gui/multieditor/multieditortext.cpp

@@ -1,4 +1,5 @@
 #include "multieditortext.h"
+#include "diff/diff_match_patch.h"
 #include "iconmanager.h"
 #include "uiconfig.h"
 #include <QPlainTextEdit>
@@ -15,6 +16,7 @@ MultiEditorText::MultiEditorText(QWidget *parent) :
 {
     setLayout(new QVBoxLayout());
     textEdit = new QPlainTextEdit();
+    textEdit->setLineWrapMode(QPlainTextEdit::NoWrap); // Disable line wrapping for performance with long lines (1MB single line freezes app for several seconds)
     layout()->addWidget(textEdit);
     initActions();
     setupMenu();
@@ -30,19 +32,13 @@ MultiEditorText::MultiEditorText(QWidget *parent) :
 
 void MultiEditorText::setValue(const QVariant& value)
 {
-    textEdit->setPlainText(value.toString());
+    originalText = value.toString();
+    textEdit->setPlainText(originalText);
 }
 
 QVariant MultiEditorText::getValue()
 {
-    if (CFG_UI.General.UseLfForMultilineEditors.get())
-    {
-        QString newStr = textEdit->document()->toRawText();
-        newStr.replace(QChar::ParagraphSeparator, '\n');
-        newStr.replace(QChar::LineSeparator, '\n');
-        return newStr;
-    }
-    return textEdit->document()->toRawText();
+    return getValueFromPlainTextEdit(originalText, textEdit);
 }
 
 void MultiEditorText::setReadOnly(bool value)
@@ -76,6 +72,17 @@ SearchTextLocator *MultiEditorText::getTextLocator()
     return textLocator;
 }
 
+QString MultiEditorText::getValueFromPlainTextEdit(const QString& originalText, QPlainTextEdit* textEdit)
+{
+    QString newText = textEdit->document()->toRawText();
+    if (originalText.contains(QChar::ParagraphSeparator) || originalText.contains(QChar::LineSeparator))
+        return restoreOriginalSeparatorsWithDiff(originalText, newText);
+
+    newText.replace(QChar::ParagraphSeparator, '\n');
+    newText.replace(QChar::LineSeparator, '\n');
+    return newText;
+}
+
 void MultiEditorText::modificationChanged(bool changed)
 {
     if (changed)
@@ -157,6 +164,105 @@ void MultiEditorText::setupMenu()
     contextMenu->addAction(actionMap[DELETE]);
 }
 
+bool MultiEditorText::isOriginalOrEditorLineBreak(QChar ch)
+{
+    return ch == QChar('\n') ||
+           ch == QChar::LineSeparator ||
+           ch == QChar::ParagraphSeparator;
+}
+
+bool MultiEditorText::isQTextDocumentSeparator(QChar ch)
+{
+    return ch == QChar::LineSeparator ||
+           ch == QChar::ParagraphSeparator;
+}
+
+QString MultiEditorText::normalizedForDiff(QString text)
+{
+    for (int i = 0; i < text.size(); ++i)
+    {
+        if (isOriginalOrEditorLineBreak(text.at(i)))
+            text[i] = QChar('\n');
+    }
+
+    return text;
+}
+
+QVector<int> MultiEditorText::buildCurrentToOriginalMap(const QString& originalText, const QString& currentText)
+{
+    QString normalizedOriginal = normalizedForDiff(originalText);
+    QString normalizedCurrent = normalizedForDiff(currentText);
+    QVector<int> currentToOriginal(currentText.size(), -1);
+
+    diff_match_patch dmp;
+    int originalPos = 0;
+    int currentPos = 0;
+    auto diffs = dmp.diff_main(normalizedOriginal, normalizedCurrent, false);
+    for (const Diff& diff : std::as_const(diffs))
+    {
+        int len = diff.text.size();
+        switch (diff.operation)
+        {
+            case Operation::EQUAL:
+            {
+                for (int i = 0; i < len; ++i)
+                {
+                    if (currentPos + i < currentToOriginal.size())
+                        currentToOriginal[currentPos + i] = originalPos + i;
+                }
+                originalPos += len;
+                currentPos += len;
+                break;
+            }
+            case Operation::DELETE:
+            {
+                originalPos += len;
+                break;
+            }
+            case Operation::INSERT:
+            {
+                currentPos += len;
+                break;
+            }
+        }
+    }
+
+    return currentToOriginal;
+}
+
+QString MultiEditorText::restoreOriginalSeparatorsWithDiff(const QString& originalText, const QString& currentRawText)
+{
+    QString result;
+
+    QVector<int> currentToOriginal = buildCurrentToOriginalMap(originalText, currentRawText);
+    result.reserve(currentRawText.size());
+    for (int i = 0; i < currentRawText.size(); ++i)
+    {
+        const QChar currentChar = currentRawText.at(i);
+        if (!isQTextDocumentSeparator(currentChar))
+        {
+            result += currentChar;
+            continue;
+        }
+
+        const int originalPos = currentToOriginal.value(i, -1);
+        if (originalPos >= 0 && originalPos < originalText.size())
+        {
+            const QChar originalChar = originalText.at(originalPos);
+            if (originalChar == QChar::LineSeparator)
+                result += QChar::LineSeparator;
+            else if (originalChar == QChar::ParagraphSeparator)
+                result += QChar::ParagraphSeparator;
+            else
+                result += QChar('\n');
+        }
+        else
+            result += QChar('\n');
+    }
+
+    return result;
+}
+
 MultiEditorWidget* MultiEditorTextPlugin::getInstance()
 {
     return new MultiEditorText();

Letos/gui/multieditor/multieditortext.h

@@ -49,16 +49,25 @@ class GUI_API_EXPORT MultiEditorText : public MultiEditorWidget, public ExtActio
         QList<QWidget*> getNoScrollWidgets();
         SearchTextLocator* getTextLocator();
 
+        static QString getValueFromPlainTextEdit(const QString& originalText, QPlainTextEdit* textEdit);
+        static QString restoreOriginalSeparatorsWithDiff(const QString& originalText, const QString& currentRawText);
+
     protected:
         void createActions();
         void setupDefShortcuts();
 
     private:
         void setupMenu();
 
+        static bool isOriginalOrEditorLineBreak(QChar ch);
+        static bool isQTextDocumentSeparator(QChar ch);
+        static QString normalizedForDiff(QString text);
+        static QVector<int> buildCurrentToOriginalMap(const QString& originalText, const QString& currentText);
+
         QPlainTextEdit* textEdit = nullptr;
         QMenu* contextMenu = nullptr;
         SearchTextLocator* textLocator = nullptr;
+        QString originalText;
 
     private slots:
         void modificationChanged(bool changed);

Letos/gui/uiconfig.h

@@ -177,7 +177,6 @@ CFG_UI_CATEGORIES(Ui,
         CFG_ENTRY(bool,                    ShowDataViewTooltips,        true)
         CFG_ENTRY(bool,                    KeepNullWhenEmptyValue,      true)
         CFG_ENTRY(bool,                    UseSciFormatForDoubles,      false)
-        CFG_ENTRY(bool,                    UseLfForMultilineEditors,    false)
         CFG_ENTRY(Cfg::ColorPickerConfig,  ColorPickerConfig,           Cfg::ColorPickerConfig())
         CFG_ENTRY(int,                     ToolBarIconSize,             0)
         CFG_ENTRY(QString,                 LastUsedConfigPage,          QString())