feat: do not load entire data to memory in libarchive operations

1 files changed, 149 insertions, 97 deletions

diff --git a/src/core/function/ArchiveFileOperator.cpp b/src/core/function/ArchiveFileOperator.cpp
index c8cd99f4..b252f3a0 100644
--- a/src/core/function/ArchiveFileOperator.cpp
+++ b/src/core/function/ArchiveFileOperator.cpp
@@ -30,6 +30,7 @@
 
 #include <archive.h>
 #include <archive_entry.h>
+#include <sys/fcntl.h>
 
 #include <fstream>
 
@@ -37,14 +38,6 @@
 
 namespace GpgFrontend {
 
-struct ArchiveStruct {
-  struct archive *archive;
-  struct archive_entry *entry;
-  int fd;
-  bool is_open;
-  std::string name;
-};
-
 auto CopyData(struct archive *ar, struct archive *aw) -> int {
   int r;
   const void *buff;
@@ -68,126 +61,185 @@ auto CopyData(struct archive *ar, struct archive *aw) -> int {
   }
 }
 
-void ArchiveFileOperator::NewArchive2Fd(
-    const std::filesystem::path &target_directory, int fd,
-    const OperationCallback &cb) {
+struct ArchiveReadClientData {
+  GFDataExchanger *ex;
+  std::array<std::byte, 1024> buf;
+  const std::byte *p_buf = buf.data();
+};
+
+auto ArchiveReadCallback(struct archive *, void *client_data,
+                         const void **buffer) -> ssize_t {
+  auto *rdata = static_cast<ArchiveReadClientData *>(client_data);
+  *buffer = reinterpret_cast<const void *>(rdata->p_buf);
+  return rdata->ex->Read(rdata->buf.data(), rdata->buf.size());
+}
+
+auto ArchiveWriteCallback(struct archive *, void *client_data,
+                          const void *buffer, size_t length) -> ssize_t {
+  auto *ex = static_cast<GFDataExchanger *>(client_data);
+  return ex->Write(static_cast<const std::byte *>(buffer), length);
+}
+
+auto ArchiveCloseWriteCallback(struct archive *, void *client_data) -> int {
+  auto *ex = static_cast<GFDataExchanger *>(client_data);
+  ex->CloseWrite();
+  return 0;
+}
+
+void ArchiveFileOperator::NewArchive2DataExchanger(
+    const std::filesystem::path &target_directory,
+    std::shared_ptr<GFDataExchanger> exchanger, const OperationCallback &cb) {
   RunIOOperaAsync(
       [=](const DataObjectPtr &data_object) -> GFError {
-        struct archive *archive;
-        struct archive_entry *entry;
-        std::array<char, 8192> buff{};
+        std::array<char, 1024> buff{};
+        auto ret = 0;
+        const auto base_path = target_directory.parent_path();
 
-        archive = archive_write_new();
+        auto *archive = archive_write_new();
         archive_write_add_filter_none(archive);
         archive_write_set_format_pax_restricted(archive);
-        archive_write_open_fd(archive, fd);
 
-        for (const auto &file :
-             std::filesystem::recursive_directory_iterator(target_directory)) {
-          entry = archive_entry_new();
+        archive_write_open(archive, exchanger.get(), nullptr,
+                           ArchiveWriteCallback, ArchiveCloseWriteCallback);
 
-          auto file_path = file.path().string();
-          archive_entry_set_pathname(entry, file_path.c_str());
-          archive_entry_set_filetype(entry, AE_IFREG);
-          archive_entry_set_perm(entry, 0644);
+        auto *disk = archive_read_disk_new();
+        archive_read_disk_set_standard_lookup(disk);
+        auto r = archive_read_disk_open(disk, target_directory.c_str());
 
-          std::ifstream target_file(file_path, std::ifstream::binary);
-          if (!target_file) {
-            SPDLOG_ERROR("cannot open file: {}, abort...", file_path);
-            archive_entry_free(entry);
-            continue;
+        if (r != ARCHIVE_OK) {
+          SPDLOG_ERROR("archive_read_disk_open() failed: {}, abort...",
+                       archive_error_string(disk));
+          archive_read_free(disk);
+          archive_write_free(archive);
+          return -1;
+        }
+
+        for (;;) {
+          auto *entry = archive_entry_new();
+          r = archive_read_next_header2(disk, entry);
+          if (r == ARCHIVE_EOF) break;
+          if (r != ARCHIVE_OK) {
+            SPDLOG_ERROR(
+                "archive_read_next_header2() failed, ret: {}, explain: {}", r,
+                archive_error_string(disk));
+            ret = -1;
+            break;
           }
 
-          target_file.seekg(0, std::ios::end);
-          auto file_size = target_file.tellg();
-          target_file.seekg(0, std::ios::beg);
-          archive_entry_set_size(entry, file_size);
+          archive_read_disk_descend(disk);
 
-          archive_write_header(archive, entry);
+          // turn absolute path to relative path
+          archive_entry_set_pathname(
+              entry, std::filesystem::relative(
+                         std::filesystem::path(archive_entry_pathname(entry)),
+                         base_path)
+                         .c_str());
 
-          while (!target_file.eof()) {
-            target_file.read(buff.data(), buff.size());
-            std::streamsize const bytes_read = target_file.gcount();
-            archive_write_data(archive, buff.data(), bytes_read);
+          r = archive_write_header(archive, entry);
+          if (r < ARCHIVE_OK) {
+            SPDLOG_ERROR("archive_write_header() failed, ret: {}, explain: {} ",
+                         r, archive_error_string(archive));
+            continue;
           }
 
+          if (r == ARCHIVE_FATAL) {
+            SPDLOG_ERROR(
+                "archive_write_header() failed, ret: {}, explain: {}, "
+                "abort ...",
+                r, archive_error_string(archive));
+            ret = -1;
+            break;
+          }
+
+          if (r > ARCHIVE_FAILED) {
+            auto fd = open(archive_entry_sourcepath(entry), O_RDONLY);
+            auto len = read(fd, buff.data(), buff.size());
+            while (len > 0) {
+              archive_write_data(archive, buff.data(), len);
+              len = read(fd, buff.data(), buff.size());
+            }
+            close(fd);
+          }
           archive_entry_free(entry);
         }
 
-        archive_write_close(archive);
+        archive_read_free(disk);
         archive_write_free(archive);
-        close(fd);
-        return 0;
+        return ret;
       },
       cb, "archive_write_new");
 }
 
-void ArchiveFileOperator::ExtractArchiveFromFd(
-    int fd, const std::filesystem::path &target_path,
-    const OperationCallback &cb) {
-  SPDLOG_DEBUG("extract archive from fd start, cuurent thread: {}",
-               QThread::currentThread()->currentThreadId());
+void ArchiveFileOperator::ExtractArchiveFromDataExchanger(
+    std::shared_ptr<GFDataExchanger> ex,
+    const std::filesystem::path &target_path, const OperationCallback &cb) {
   RunIOOperaAsync(
       [=](const DataObjectPtr &data_object) -> GFError {
-        SPDLOG_DEBUG("extract archive from fd processing, cuurent thread: {}",
-                     QThread::currentThread()->currentThreadId());
-        struct archive *archive;
-        struct archive *ext;
-        struct archive_entry *entry;
-
-        archive = archive_read_new();
-        ext = archive_write_disk_new();
-        archive_write_disk_set_options(ext, 0);
-
-#ifndef NO_BZIP2_EXTRACT
-        archive_read_support_filter_bzip2(archive);
-#endif
-#ifndef NO_GZIP_EXTRACT
-        archive_read_support_filter_gzip(archive);
-#endif
-#ifndef NO_COMPRESS_EXTRACT
-        archive_read_support_filter_compress(archive);
-#endif
-#ifndef NO_TAR_EXTRACT
-        archive_read_support_format_tar(archive);
-#endif
-#ifndef NO_CPIO_EXTRACT
-        archive_read_support_format_cpio(archive);
-#endif
-#ifndef NO_LOOKUP
-        archive_write_disk_set_standard_lookup(ext);
-#endif
-
-        archive_read_open_fd(archive, fd, 8192);
-        SPDLOG_ERROR("archive_read_open_fd() failed: {}",
-                     archive_error_string(archive));
-
-        while (archive_read_next_header(archive, &entry) == ARCHIVE_OK) {
-          SPDLOG_DEBUG("add file: {}, size: {}, bytes: {}, file type: {}",
-                       archive_entry_pathname_utf8(entry),
-                       archive_entry_size(entry),
-                       archive_entry_filetype(entry));
-
-          auto file_path =
-              std::filesystem::path(archive_entry_pathname_utf8(entry));
-          auto target_file_path = target_path / file_path;
-          archive_entry_set_pathname(entry, file_path.c_str());
-
-          auto r = archive_write_header(ext, entry);
+        auto *archive = archive_read_new();
+        auto *ext = archive_write_disk_new();
+
+        auto r = archive_read_support_filter_all(archive);
+        if (r != ARCHIVE_OK) {
+          SPDLOG_ERROR("archive_read_support_filter_all(), ret: {}, reason: {}",
+                       r, archive_error_string(archive));
+          return r;
+        }
+
+        r = archive_read_support_format_all(archive);
+        if (r != ARCHIVE_OK) {
+          SPDLOG_ERROR("archive_read_support_format_all(), ret: {}, reason: {}",
+                       r, archive_error_string(archive));
+          return r;
+        }
+
+        auto rdata = ArchiveReadClientData{};
+        rdata.ex = ex.get();
+
+        r = archive_read_open(archive, &rdata, nullptr, ArchiveReadCallback,
+                              nullptr);
+        if (r != ARCHIVE_OK) {
+          SPDLOG_ERROR("archive_read_open(), ret: {}, reason: {}", r,
+                       archive_error_string(archive));
+          return r;
+        }
+
+        r = archive_write_disk_set_options(ext, 0);
+        if (r != ARCHIVE_OK) {
+          SPDLOG_ERROR("archive_write_disk_set_options(), ret: {}, reason: {}",
+                       r, archive_error_string(archive));
+          return r;
+        }
+
+        for (;;) {
+          struct archive_entry *entry;
+          r = archive_read_next_header(archive, &entry);
+          if (r == ARCHIVE_EOF) break;
           if (r != ARCHIVE_OK) {
-            SPDLOG_ERROR("archive_write_header() failed: {}",
-                         archive_error_string(ext));
+            SPDLOG_ERROR("archive_read_next_header(), ret: {}, reason: {}", r,
+                         archive_error_string(archive));
+            break;
           }
 
-          r = CopyData(archive, ext);
+          r = archive_write_header(ext, entry);
           if (r != ARCHIVE_OK) {
-            SPDLOG_ERROR("copy_data() failed: {}", archive_error_string(ext));
+            SPDLOG_ERROR("archive_write_header(), ret: {}, reason: {}", r,
+                         archive_error_string(archive));
+          } else {
+            r = CopyData(archive, ext);
           }
         }
 
-        archive_read_free(archive);
-        archive_write_free(ext);
-        close(fd);
+        r = archive_read_free(archive);
+        if (r != ARCHIVE_OK) {
+          SPDLOG_ERROR("archive_read_free(), ret: {}, reason: {}", r,
+                       archive_error_string(archive));
+        }
+        r = archive_write_free(ext);
+        if (r != ARCHIVE_OK) {
+          SPDLOG_ERROR("archive_read_free(), ret: {}, reason: {}", r,
+                       archive_error_string(archive));
+        }
+
         return 0;
       },
       cb, "archive_read_new");