feat(ui): more advanced charset functions

1. support dealing more type of files with different charsets
2. solve cash cause by reading file thread
3. improve ui operations

3 files changed, 136 insertions, 55 deletions

diff --git a/src/ui/widgets/PlainTextEditorPage.cpp b/src/ui/widgets/PlainTextEditorPage.cpp
index c70991b9..c1787c36 100644
--- a/src/ui/widgets/PlainTextEditorPage.cpp
+++ b/src/ui/widgets/PlainTextEditorPage.cpp
@@ -24,13 +24,13 @@
  *
  */
 
-#include "ui/widgets/PlainTextEditorPage.h"
-
-#include <encoding-detect/TextEncodingDetect.h>
+#include "PlainTextEditorPage.h"
 
 #include <boost/format.hpp>
+#include <string>
 #include <utility>
 
+#include "core/function/CharsetOperator.h"
 #include "ui/thread/FileReadThread.h"
 #include "ui_PlainTextEditor.h"
 
@@ -42,8 +42,6 @@ PlainTextEditorPage::PlainTextEditorPage(QString file_path, QWidget *parent)
       full_file_path_(std::move(file_path)) {
   ui_->setupUi(this);
 
-  if (full_file_path_.isEmpty()) read_done_ = true;
-
   ui_->textPage->setFocus();
   ui_->loadingLabel->setHidden(true);
 
@@ -52,21 +50,26 @@ PlainTextEditorPage::PlainTextEditorPage(QString file_path, QWidget *parent)
   this->setAttribute(Qt::WA_DeleteOnClose);
 
   this->ui_->characterLabel->setText(_("0 character"));
-  this->ui_->lfLabel->setText(_("None"));
-  this->ui_->encodingLabel->setText(_("Binary"));
+  this->ui_->lfLabel->setText(_("lf"));
+  this->ui_->encodingLabel->setText(_("utf-8"));
 
   connect(ui_->textPage, &QPlainTextEdit::textChanged, this, [=]() {
+    // if file is loaded
     if (!read_done_) return;
 
     auto text = ui_->textPage->document()->toPlainText();
     auto str = boost::format(_("%1% character(s)")) % text.size();
     this->ui_->characterLabel->setText(str.str().c_str());
-
-    detect_cr_lf(text);
-    detect_encoding(text.toStdString());
   });
 
-  ui_->loadingLabel->setText(_("Loading..."));
+  if (full_file_path_.isEmpty()) {
+    read_done_ = true;
+    ui_->loadingLabel->setHidden(true);
+  } else {
+    read_done_ = false;
+    ui_->loadingLabel->setText(_("Loading..."));
+    ui_->loadingLabel->setHidden(false);
+  }
 }
 
 const QString &PlainTextEditorPage::GetFilePath() const {
@@ -75,6 +78,26 @@ const QString &PlainTextEditorPage::GetFilePath() const {
 
 QPlainTextEdit *PlainTextEditorPage::GetTextPage() { return ui_->textPage; }
 
+bool PlainTextEditorPage::WillCharsetChange() const {
+  // detect if the line-ending will change
+  if (is_crlf_) return true;
+
+  // detect if the charset of the file will change
+  if (charset_name_ != "UTF-8" && charset_name_ != "ISO-8859-1")
+    return true;
+  else
+    return false;
+}
+
+void PlainTextEditorPage::NotifyFileSaved() {
+  this->is_crlf_ = false;
+  this->charset_confidence_ = 100;
+  this->charset_name_ = "UTF-8";
+
+  this->ui_->lfLabel->setText(_("lf"));
+  this->ui_->encodingLabel->setText(_("UTF-8"));
+}
+
 void PlainTextEditorPage::SetFilePath(const QString &filePath) {
   full_file_path_ = filePath;
 }
@@ -150,25 +173,24 @@ void PlainTextEditorPage::ReadFile() {
     if (!binary_mode_) {
       text_page->setReadOnly(false);
     }
-  });
-
-  connect(thread, &FileReadThread::finished, this, [=]() {
-    LOG(INFO) << "thread finished";
-    thread->deleteLater();
-    read_done_ = true;
-    read_thread_ = nullptr;
-    ui_->textPage->setEnabled(true);
+    this->read_done_ = true;
+    this->ui_->textPage->setEnabled(true);
     text_page->document()->setModified(false);
-    ui_->textPage->blockSignals(false);
-    ui_->textPage->document()->blockSignals(false);
-    ui_->loadingLabel->setHidden(true);
+    this->ui_->textPage->blockSignals(false);
+    this->ui_->textPage->document()->blockSignals(false);
+    this->ui_->loadingLabel->setHidden(true);
+
+    // delete thread
+    read_thread_->deleteLater();
   });
 
   connect(this, &PlainTextEditorPage::destroyed, [=]() {
     LOG(INFO) << "request interruption for read thread";
-    if (read_thread_ && thread->isRunning()) thread->requestInterruption();
+    if (read_thread_ != nullptr && read_thread_->isRunning())
+      read_thread_->requestInterruption();
     read_thread_ = nullptr;
   });
+
   this->read_thread_ = thread;
   thread->start();
 }
@@ -185,13 +207,14 @@ void PlainTextEditorPage::slot_insert_text(const std::string &data) {
   LOG(INFO) << "data size" << data.size();
   read_bytes_ += data.size();
   // If binary format is detected, the entire file is converted to binary format
-  // for display
+  // for display.
   bool if_last_binary_mode = binary_mode_;
-  if (!binary_mode_) {
+  if (!binary_mode_ && !read_done_) {
     detect_encoding(data);
   }
 
   if (binary_mode_) {
+    // change formery displayed text to binary format
     if (if_last_binary_mode != binary_mode_) {
       auto text_buffer =
           ui_->textPage->document()->toRawText().toLocal8Bit().toStdString();
@@ -200,17 +223,33 @@ void PlainTextEditorPage::slot_insert_text(const std::string &data) {
           binary_to_string(text_buffer).c_str());
       this->ui_->lfLabel->setText("None");
     }
+
+    // insert new data
     this->GetTextPage()->insertPlainText(binary_to_string(data).c_str());
 
+    // update the size of the file
     auto str = boost::format(_("%1% byte(s)")) % read_bytes_;
     this->ui_->characterLabel->setText(str.str().c_str());
   } else {
-    this->GetTextPage()->insertPlainText(data.c_str());
+    // detect crlf/lf line ending
+    if (!binary_mode_) detect_cr_lf(data);
+
+    // when reding from a text file
+    // try convert the any of thetext to utf8
+    std::string utf8_data;
+    if (!read_done_ && charset_confidence_ > 25) {
+      CharsetOperator::Convert2Utf8(data, utf8_data, charset_name_);
+    } else {
+      // when editing a text file, do nothing.
+      utf8_data = data;
+    }
+
+    // insert the text to the text page
+    this->GetTextPage()->insertPlainText(utf8_data.c_str());
 
     auto text = this->GetTextPage()->toPlainText();
     auto str = boost::format(_("%1% character(s)")) % text.size();
     this->ui_->characterLabel->setText(str.str().c_str());
-    detect_cr_lf(text);
   }
 }
 
@@ -222,38 +261,42 @@ void PlainTextEditorPage::PrepareToDestroy() {
 }
 
 void PlainTextEditorPage::detect_encoding(const std::string &data) {
-  AutoIt::Common::TextEncodingDetect text_detect;
-  AutoIt::Common::TextEncodingDetect::Encoding encoding =
-      text_detect.DetectEncoding((unsigned char *)(data.data()), data.size());
+  // skip the binary data to avoid the false detection of the encoding
+  if (binary_mode_) return;
 
-  if (encoding == AutoIt::Common::TextEncodingDetect::None) {
+  // detect the encoding
+  auto charset = CharsetOperator::Detect(data);
+  this->charset_name_ = std::get<0>(charset).c_str();
+  this->language_name_ = std::get<1>(charset).c_str();
+  this->charset_confidence_ = std::get<2>(charset);
+
+  // probably there is no need to detect the encoding again
+  if (this->charset_confidence_ < 10) {
     binary_mode_ = true;
-    ui_->encodingLabel->setText(_("Binary"));
-  } else if (encoding == AutoIt::Common::TextEncodingDetect::ASCII) {
-    ui_->encodingLabel->setText(_("ASCII(7 bits)"));
-  } else if (encoding == AutoIt::Common::TextEncodingDetect::ANSI) {
-    ui_->encodingLabel->setText(_("ASCII(8 bits)"));
-  } else if (encoding == AutoIt::Common::TextEncodingDetect::UTF8_BOM ||
-             encoding == AutoIt::Common::TextEncodingDetect::UTF8_NOBOM) {
-    ui_->encodingLabel->setText(_("UTF-8"));
-  } else if (encoding == AutoIt::Common::TextEncodingDetect::UTF16_LE_BOM ||
-             encoding == AutoIt::Common::TextEncodingDetect::UTF16_LE_NOBOM) {
-    ui_->encodingLabel->setText(_("UTF-16"));
-  } else if (encoding == AutoIt::Common::TextEncodingDetect::UTF16_BE_BOM ||
-             encoding == AutoIt::Common::TextEncodingDetect::UTF16_BE_NOBOM) {
-    ui_->encodingLabel->setText(_("UTF-16(BE)"));
+  }
+
+  if (binary_mode_) {
+    // hide the line ending label, when the file is binary
+    this->ui_->lfLabel->setHidden(true);
+    this->ui_->encodingLabel->setText(_("binary"));
+  } else {
+    ui_->encodingLabel->setText(this->charset_name_.c_str());
   }
 }
 
-void PlainTextEditorPage::detect_cr_lf(const QString &data) {
+void PlainTextEditorPage::detect_cr_lf(const std::string &data) {
   if (binary_mode_) {
-    this->ui_->lfLabel->setText("None");
     return;
   }
-  if (data.contains("\r\n")) {
-    this->ui_->lfLabel->setText("CRLF");
+
+  // if contain crlf, set the label to crlf
+  if (is_crlf_) return;
+
+  if (data.find("\r\n") != std::string::npos) {
+    this->ui_->lfLabel->setText("crlf");
+    is_crlf_ = true;
   } else {
-    this->ui_->lfLabel->setText("LF");
+    this->ui_->lfLabel->setText("lf");
   }
 }
 
diff --git a/src/ui/widgets/PlainTextEditorPage.h b/src/ui/widgets/PlainTextEditorPage.h
index e76c11e3..0009b7d6 100644
--- a/src/ui/widgets/PlainTextEditorPage.h
+++ b/src/ui/widgets/PlainTextEditorPage.h
@@ -29,6 +29,8 @@
 #ifndef __EDITORPAGE_H__
 #define __EDITORPAGE_H__
 
+#include <string>
+
 #include "core/GpgConstants.h"
 #include "ui/GpgFrontendUI.h"
 
@@ -49,7 +51,7 @@ class PlainTextEditorPage : public QWidget {
    * @param file_path Path of the file handled in this tab
    * @param parent Pointer to the parent widget
    */
-  explicit PlainTextEditorPage(QString file_path = "",
+  explicit PlainTextEditorPage(QString file_path = {},
                                QWidget* parent = nullptr);
 
   /**
@@ -104,6 +106,18 @@ class PlainTextEditorPage : public QWidget {
    */
   void PrepareToDestroy();
 
+  /**
+   * @brief detect if the charset of the file will change
+   *
+   */
+  bool WillCharsetChange() const;
+
+  /**
+   * @brief notify the user that the file has been saved.
+   *
+   */
+  void NotifyFileSaved();
+
  private:
   std::shared_ptr<Ui_PlainTextEditor> ui_;  ///<
   QString full_file_path_;  ///< The path to the file handled in the tab
@@ -112,6 +126,10 @@ class PlainTextEditorPage : public QWidget {
   QThread* read_thread_ = nullptr;  ///<
   bool binary_mode_ = false;        ///<
   size_t read_bytes_ = 0;           ///<
+  std::string charset_name_;        ///<
+  std::string language_name_;       ///<
+  int32_t charset_confidence_;      ///<
+  bool is_crlf_ = false;            ///<
 
   /**
    * @brief
@@ -125,7 +143,7 @@ class PlainTextEditorPage : public QWidget {
    *
    * @param data
    */
-  void detect_cr_lf(const QString& data);
+  void detect_cr_lf(const std::string& data);
 
  private slots:
 
diff --git a/src/ui/widgets/TextEdit.cpp b/src/ui/widgets/TextEdit.cpp
index ecf1a4bd..689f877b 100644
--- a/src/ui/widgets/TextEdit.cpp
+++ b/src/ui/widgets/TextEdit.cpp
@@ -166,11 +166,30 @@ bool TextEdit::save_file(const QString& fileName) {
     return false;
   }
 
+  PlainTextEditorPage* page = SlotCurPageTextEdit();
+  if (page == nullptr) return false;
+
+  if (page->WillCharsetChange()) {
+    auto result = QMessageBox::warning(
+        this, _("Save"),
+        QString("<p>") +
+            _("After saving, the encoding of the current file will be "
+              "converted to UTF-8 and the line endings will be changed to "
+              "LF. ") +
+            "</p>" + "<p>" +
+            _("If this is not the result you expect, please use \"save "
+              "as\".") +
+            "</p>",
+        QMessageBox::Save | QMessageBox::Cancel, QMessageBox::Cancel);
+
+    if (result == QMessageBox::Cancel) {
+      return false;
+    }
+  }
+
   QFile file(fileName);
 
   if (file.open(QIODevice::WriteOnly | QIODevice::Text)) {
-    PlainTextEditorPage* page = SlotCurPageTextEdit();
-
     QTextStream outputStream(&file);
     QApplication::setOverrideCursor(Qt::WaitCursor);
     outputStream << page->GetTextPage()->toPlainText();
@@ -182,7 +201,8 @@ bool TextEdit::save_file(const QString& fileName) {
     int curIndex = tab_widget_->currentIndex();
     tab_widget_->setTabText(curIndex, stripped_name(fileName));
     page->SetFilePath(fileName);
-    //      statusBar()->showMessage(_("File saved"), 2000);
+    page->NotifyFileSaved();
+
     file.close();
     return true;
   } else {