From 7c21fdd97132e63572a6aca6fa0492d2ce74e777 Mon Sep 17 00:00:00 2001 From: saturneric Date: Mon, 16 Oct 2023 03:47:52 +0800 Subject: feat: start to add plugin system and do some improve on project configuration --- src/core/function/ArchiveFileOperator.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'src/core/function/ArchiveFileOperator.cpp') diff --git a/src/core/function/ArchiveFileOperator.cpp b/src/core/function/ArchiveFileOperator.cpp index 8aad0500..9b23151f 100644 --- a/src/core/function/ArchiveFileOperator.cpp +++ b/src/core/function/ArchiveFileOperator.cpp @@ -28,6 +28,17 @@ #include "ArchiveFileOperator.h" +#include +#include + +struct ArchiveStruct { + struct archive *archive; + struct archive_entry *entry; + int fd; + bool is_open; + std::string name; +}; + int copy_data(struct archive *ar, struct archive *aw) { int r; const void *buff; -- cgit v1.2.3 From 95997d27106daf91336847f50efaaa32279b7fc7 Mon Sep 17 00:00:00 2001 From: saturneric Date: Mon, 16 Oct 2023 17:54:05 +0800 Subject: fix: check and update copyright at files --- src/core/function/ArchiveFileOperator.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/core/function/ArchiveFileOperator.cpp') diff --git a/src/core/function/ArchiveFileOperator.cpp b/src/core/function/ArchiveFileOperator.cpp index 9b23151f..5c2dea82 100644 --- a/src/core/function/ArchiveFileOperator.cpp +++ b/src/core/function/ArchiveFileOperator.cpp @@ -1,5 +1,5 @@ /** - * Copyright (C) 2021 Saturneric + * Copyright (C) 2021 Saturneric * * This file is part of GpgFrontend. * @@ -20,7 +20,7 @@ * the gpg4usb project, which is under GPL-3.0-or-later. * * All the source code of GpgFrontend was modified and released by - * Saturneric starting on May 12, 2021. + * Saturneric starting on May 12, 2021. * * SPDX-License-Identifier: GPL-3.0-or-later * -- cgit v1.2.3 From 5cbbab71b0ea131a56de0374ccb2c29f26824e39 Mon Sep 17 00:00:00 2001 From: saturneric Date: Wed, 18 Oct 2023 15:17:07 +0800 Subject: chore: remove libarchive from source code 1. it's better to get this library at system PATH --- src/core/function/ArchiveFileOperator.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/core/function/ArchiveFileOperator.cpp') diff --git a/src/core/function/ArchiveFileOperator.cpp b/src/core/function/ArchiveFileOperator.cpp index 5c2dea82..efe824dd 100644 --- a/src/core/function/ArchiveFileOperator.cpp +++ b/src/core/function/ArchiveFileOperator.cpp @@ -28,8 +28,8 @@ #include "ArchiveFileOperator.h" -#include -#include +#include +#include struct ArchiveStruct { struct archive *archive; -- cgit v1.2.3 From 124929609eabff19359caad276a10f1026793c0f Mon Sep 17 00:00:00 2001 From: saturneric Date: Wed, 25 Oct 2023 18:26:26 +0800 Subject: fix: solve some code tidy issues --- src/core/function/ArchiveFileOperator.cpp | 2 -- 1 file changed, 2 deletions(-) (limited to 'src/core/function/ArchiveFileOperator.cpp') diff --git a/src/core/function/ArchiveFileOperator.cpp b/src/core/function/ArchiveFileOperator.cpp index efe824dd..4fc8f5df 100644 --- a/src/core/function/ArchiveFileOperator.cpp +++ b/src/core/function/ArchiveFileOperator.cpp @@ -82,8 +82,6 @@ void GpgFrontend::ArchiveFileOperator::CreateArchive( struct archive *a; struct archive_entry *entry; - ssize_t len; - int fd; SPDLOG_DEBUG("compress: {}", compress); -- cgit v1.2.3 From 889cb8092381b073a0f4a0411a4ede04cd7bdd14 Mon Sep 17 00:00:00 2001 From: saturneric Date: Mon, 6 Nov 2023 20:49:44 +0800 Subject: refactor: improve the code structure of core --- src/core/function/ArchiveFileOperator.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src/core/function/ArchiveFileOperator.cpp') diff --git a/src/core/function/ArchiveFileOperator.cpp b/src/core/function/ArchiveFileOperator.cpp index 4fc8f5df..27d18803 100644 --- a/src/core/function/ArchiveFileOperator.cpp +++ b/src/core/function/ArchiveFileOperator.cpp @@ -170,10 +170,10 @@ void GpgFrontend::ArchiveFileOperator::CreateArchive( if (r > ARCHIVE_FAILED) { QByteArray buff; #ifdef WINDOWS - FileOperator::ReadFile( - QString::fromStdWString(archive_entry_sourcepath_w(entry)), buff); + ReadFile(QString::fromStdWString(archive_entry_sourcepath_w(entry)), + buff); #else - FileOperator::ReadFile(archive_entry_sourcepath(entry), buff); + ReadFile(archive_entry_sourcepath(entry), buff); #endif archive_write_data(a, buff.data(), buff.size()); } -- cgit v1.2.3 From 3ad7fecdb6458fdd6f146bed19fe643c7f93e905 Mon Sep 17 00:00:00 2001 From: saturneric Date: Tue, 7 Nov 2023 15:18:06 +0800 Subject: refactor: remove CommonUtils at core --- src/core/function/ArchiveFileOperator.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'src/core/function/ArchiveFileOperator.cpp') diff --git a/src/core/function/ArchiveFileOperator.cpp b/src/core/function/ArchiveFileOperator.cpp index 27d18803..18920e75 100644 --- a/src/core/function/ArchiveFileOperator.cpp +++ b/src/core/function/ArchiveFileOperator.cpp @@ -39,7 +39,7 @@ struct ArchiveStruct { std::string name; }; -int copy_data(struct archive *ar, struct archive *aw) { +auto copy_data(struct archive *ar, struct archive *aw) -> int { int r; const void *buff; size_t size; @@ -112,8 +112,9 @@ void GpgFrontend::ArchiveFileOperator::CreateArchive( auto u8_filename = relative_archive_path.u8string(); - if (!u8_filename.empty() && u8_filename == u8"-") + if (!u8_filename.empty() && u8_filename == u8"-") { throw std::runtime_error("cannot write to stdout"); + } #ifdef WINDOWS archive_write_open_filename_w(a, relative_archive_path.wstring().c_str()); -- cgit v1.2.3 From 300e55bf5bddc393de050c2ca9a0356fce9a8a9d Mon Sep 17 00:00:00 2001 From: saturneric Date: Thu, 28 Dec 2023 14:32:49 +0800 Subject: feat: add simple archiving functions for encrypt and decrypt --- src/core/function/ArchiveFileOperator.cpp | 310 +++++++++++------------------- 1 file changed, 117 insertions(+), 193 deletions(-) (limited to 'src/core/function/ArchiveFileOperator.cpp') diff --git a/src/core/function/ArchiveFileOperator.cpp b/src/core/function/ArchiveFileOperator.cpp index 18920e75..c8cd99f4 100644 --- a/src/core/function/ArchiveFileOperator.cpp +++ b/src/core/function/ArchiveFileOperator.cpp @@ -31,6 +31,12 @@ #include #include +#include + +#include "core/utils/AsyncUtils.h" + +namespace GpgFrontend { + struct ArchiveStruct { struct archive *archive; struct archive_entry *entry; @@ -39,7 +45,7 @@ struct ArchiveStruct { std::string name; }; -auto copy_data(struct archive *ar, struct archive *aw) -> int { +auto CopyData(struct archive *ar, struct archive *aw) -> int { int r; const void *buff; size_t size; @@ -62,216 +68,132 @@ auto copy_data(struct archive *ar, struct archive *aw) -> int { } } -void GpgFrontend::ArchiveFileOperator::CreateArchive( - const std::filesystem::path &base_path, - const std::filesystem::path &archive_path, int compress, - const std::vector &files) { - SPDLOG_DEBUG("CreateArchive: {}", archive_path.u8string()); - - auto current_base_path_backup = QDir::currentPath(); - QDir::setCurrent(base_path.u8string().c_str()); - - auto relative_archive_path = - std::filesystem::relative(archive_path, base_path); - - std::vector relative_files; - relative_files.reserve(files.size()); - for (const auto &file : files) { - relative_files.push_back(std::filesystem::relative(file, base_path)); - } - - struct archive *a; - struct archive_entry *entry; - - SPDLOG_DEBUG("compress: {}", compress); - - a = archive_write_new(); - switch (compress) { -#ifndef NO_BZIP2_CREATE - case 'j': - case 'y': - archive_write_add_filter_bzip2(a); - break; -#endif -#ifndef NO_COMPRESS_CREATE - case 'Z': - archive_write_add_filter_compress(a); - break; -#endif -#ifndef NO_GZIP_CREATE - case 'z': - archive_write_add_filter_gzip(a); - break; -#endif - default: - archive_write_add_filter_none(a); - break; - } - archive_write_set_format_ustar(a); - archive_write_set_format_pax_restricted(a); - - auto u8_filename = relative_archive_path.u8string(); - - if (!u8_filename.empty() && u8_filename == u8"-") { - throw std::runtime_error("cannot write to stdout"); - } - -#ifdef WINDOWS - archive_write_open_filename_w(a, relative_archive_path.wstring().c_str()); -#else - archive_write_open_filename(a, u8_filename.c_str()); -#endif - - for (const auto &file : relative_files) { - struct archive *disk = archive_read_disk_new(); -#ifndef NO_LOOKUP - archive_read_disk_set_standard_lookup(disk); -#endif - int r; - - SPDLOG_DEBUG("reading file: {}", file.u8string()); - -#ifdef WINDOWS - r = archive_read_disk_open_w(disk, file.wstring().c_str()); -#else - r = archive_read_disk_open(disk, file.u8string().c_str()); -#endif - - SPDLOG_DEBUG("read file done: {}", file.u8string()); - - if (r != ARCHIVE_OK) { - SPDLOG_ERROR("{archive_read_disk_open() failed: {}", - archive_error_string(disk)); - throw std::runtime_error("archive_read_disk_open() failed"); - } - - for (;;) { - entry = archive_entry_new(); - r = archive_read_next_header2(disk, entry); - - if (r == ARCHIVE_EOF) break; - if (r != ARCHIVE_OK) { - SPDLOG_ERROR("archive_read_next_header2() failed: {}", - archive_error_string(disk)); - throw std::runtime_error("archive_read_next_header2() failed"); - } - archive_read_disk_descend(disk); - - SPDLOG_DEBUG("Adding: {} size: {} bytes: {} file type: {}", - archive_entry_pathname_utf8(entry), - archive_entry_size(entry), archive_entry_filetype(entry)); - - r = archive_write_header(a, entry); - if (r < ARCHIVE_OK) { - SPDLOG_ERROR("archive_write_header() failed: {}", - archive_error_string(a)); - throw std::runtime_error("archive_write_header() failed"); - } - if (r == ARCHIVE_FATAL) throw std::runtime_error("archive fatal"); - if (r > ARCHIVE_FAILED) { - QByteArray buff; -#ifdef WINDOWS - ReadFile(QString::fromStdWString(archive_entry_sourcepath_w(entry)), - buff); -#else - ReadFile(archive_entry_sourcepath(entry), buff); -#endif - archive_write_data(a, buff.data(), buff.size()); - } - archive_entry_free(entry); - } - archive_read_close(disk); - archive_read_free(disk); - } - archive_write_close(a); - archive_write_free(a); - - QDir::setCurrent(current_base_path_backup); +void ArchiveFileOperator::NewArchive2Fd( + const std::filesystem::path &target_directory, int fd, + const OperationCallback &cb) { + RunIOOperaAsync( + [=](const DataObjectPtr &data_object) -> GFError { + struct archive *archive; + struct archive_entry *entry; + std::array buff{}; + + archive = archive_write_new(); + archive_write_add_filter_none(archive); + archive_write_set_format_pax_restricted(archive); + archive_write_open_fd(archive, fd); + + for (const auto &file : + std::filesystem::recursive_directory_iterator(target_directory)) { + entry = archive_entry_new(); + + auto file_path = file.path().string(); + archive_entry_set_pathname(entry, file_path.c_str()); + archive_entry_set_filetype(entry, AE_IFREG); + archive_entry_set_perm(entry, 0644); + + std::ifstream target_file(file_path, std::ifstream::binary); + if (!target_file) { + SPDLOG_ERROR("cannot open file: {}, abort...", file_path); + archive_entry_free(entry); + continue; + } + + target_file.seekg(0, std::ios::end); + auto file_size = target_file.tellg(); + target_file.seekg(0, std::ios::beg); + archive_entry_set_size(entry, file_size); + + archive_write_header(archive, entry); + + while (!target_file.eof()) { + target_file.read(buff.data(), buff.size()); + std::streamsize const bytes_read = target_file.gcount(); + archive_write_data(archive, buff.data(), bytes_read); + } + + archive_entry_free(entry); + } + + archive_write_close(archive); + archive_write_free(archive); + close(fd); + return 0; + }, + cb, "archive_write_new"); } -void GpgFrontend::ArchiveFileOperator::ExtractArchive( - const std::filesystem::path &archive_path, - const std::filesystem::path &base_path) { - SPDLOG_DEBUG("ExtractArchive: {}", archive_path.u8string()); +void ArchiveFileOperator::ExtractArchiveFromFd( + int fd, const std::filesystem::path &target_path, + const OperationCallback &cb) { + SPDLOG_DEBUG("extract archive from fd start, cuurent thread: {}", + QThread::currentThread()->currentThreadId()); + RunIOOperaAsync( + [=](const DataObjectPtr &data_object) -> GFError { + SPDLOG_DEBUG("extract archive from fd processing, cuurent thread: {}", + QThread::currentThread()->currentThreadId()); + struct archive *archive; + struct archive *ext; + struct archive_entry *entry; + + archive = archive_read_new(); + ext = archive_write_disk_new(); + archive_write_disk_set_options(ext, 0); - auto current_base_path_backup = QDir::currentPath(); - QDir::setCurrent(base_path.u8string().c_str()); - - struct archive *a; - struct archive *ext; - struct archive_entry *entry; - - a = archive_read_new(); - ext = archive_write_disk_new(); - archive_write_disk_set_options(ext, 0); #ifndef NO_BZIP2_EXTRACT - archive_read_support_filter_bzip2(a); + archive_read_support_filter_bzip2(archive); #endif #ifndef NO_GZIP_EXTRACT - archive_read_support_filter_gzip(a); + archive_read_support_filter_gzip(archive); #endif #ifndef NO_COMPRESS_EXTRACT - archive_read_support_filter_compress(a); + archive_read_support_filter_compress(archive); #endif #ifndef NO_TAR_EXTRACT - archive_read_support_format_tar(a); + archive_read_support_format_tar(archive); #endif #ifndef NO_CPIO_EXTRACT - archive_read_support_format_cpio(a); + archive_read_support_format_cpio(archive); #endif #ifndef NO_LOOKUP - archive_write_disk_set_standard_lookup(ext); + archive_write_disk_set_standard_lookup(ext); #endif - auto filename = archive_path.u8string(); - - if (!filename.empty() && filename == u8"-") { - SPDLOG_ERROR("cannot read from stdin"); - } -#ifdef WINDOWS - if (archive_read_open_filename_w(a, archive_path.wstring().c_str(), 10240) != - ARCHIVE_OK) { -#else - if (archive_read_open_filename(a, archive_path.u8string().c_str(), 10240) != - ARCHIVE_OK) { -#endif - SPDLOG_ERROR("archive_read_open_filename() failed: {}", - archive_error_string(a)); - throw std::runtime_error("archive_read_open_filename() failed"); - } - - for (;;) { - int r = archive_read_next_header(a, &entry); - if (r == ARCHIVE_EOF) break; - if (r != ARCHIVE_OK) { - SPDLOG_ERROR("archive_read_next_header() failed: {}", - archive_error_string(a)); - throw std::runtime_error("archive_read_next_header() failed"); - } - SPDLOG_DEBUG("Adding: {} size: {} bytes: {} file type: {}", - archive_entry_pathname_utf8(entry), archive_entry_size(entry), - archive_entry_filetype(entry)); - r = archive_write_header(ext, entry); - if (r != ARCHIVE_OK) { - SPDLOG_ERROR("archive_write_header() failed: {}", - archive_error_string(ext)); - } else { - r = copy_data(a, ext); - if (r != ARCHIVE_OK) { - SPDLOG_ERROR("copy_data() failed: {}", archive_error_string(ext)); - } - } - } - archive_read_close(a); - archive_read_free(a); - - archive_write_close(ext); - archive_write_free(ext); - - QDir::setCurrent(current_base_path_backup); + archive_read_open_fd(archive, fd, 8192); + SPDLOG_ERROR("archive_read_open_fd() failed: {}", + archive_error_string(archive)); + + while (archive_read_next_header(archive, &entry) == ARCHIVE_OK) { + SPDLOG_DEBUG("add file: {}, size: {}, bytes: {}, file type: {}", + archive_entry_pathname_utf8(entry), + archive_entry_size(entry), + archive_entry_filetype(entry)); + + auto file_path = + std::filesystem::path(archive_entry_pathname_utf8(entry)); + auto target_file_path = target_path / file_path; + archive_entry_set_pathname(entry, file_path.c_str()); + + auto r = archive_write_header(ext, entry); + if (r != ARCHIVE_OK) { + SPDLOG_ERROR("archive_write_header() failed: {}", + archive_error_string(ext)); + } + + r = CopyData(archive, ext); + if (r != ARCHIVE_OK) { + SPDLOG_ERROR("copy_data() failed: {}", archive_error_string(ext)); + } + } + + archive_read_free(archive); + archive_write_free(ext); + close(fd); + return 0; + }, + cb, "archive_read_new"); } -void GpgFrontend::ArchiveFileOperator::ListArchive( +void ArchiveFileOperator::ListArchive( const std::filesystem::path &archive_path) { struct archive *a; struct archive_entry *entry; @@ -291,3 +213,5 @@ void GpgFrontend::ArchiveFileOperator::ListArchive( r = archive_read_free(a); // Note 3 if (r != ARCHIVE_OK) return; } + +} // namespace GpgFrontend \ No newline at end of file -- cgit v1.2.3 From 9f7593eab9b2d6f6fa6d76a303975b355b6ab458 Mon Sep 17 00:00:00 2001 From: saturneric Date: Sat, 30 Dec 2023 19:55:20 +0800 Subject: feat: do not load entire data to memory in libarchive operations --- src/core/function/ArchiveFileOperator.cpp | 246 ++++++++++++++++++------------ 1 file changed, 149 insertions(+), 97 deletions(-) (limited to 'src/core/function/ArchiveFileOperator.cpp') diff --git a/src/core/function/ArchiveFileOperator.cpp b/src/core/function/ArchiveFileOperator.cpp index c8cd99f4..b252f3a0 100644 --- a/src/core/function/ArchiveFileOperator.cpp +++ b/src/core/function/ArchiveFileOperator.cpp @@ -30,6 +30,7 @@ #include #include +#include #include @@ -37,14 +38,6 @@ namespace GpgFrontend { -struct ArchiveStruct { - struct archive *archive; - struct archive_entry *entry; - int fd; - bool is_open; - std::string name; -}; - auto CopyData(struct archive *ar, struct archive *aw) -> int { int r; const void *buff; @@ -68,126 +61,185 @@ auto CopyData(struct archive *ar, struct archive *aw) -> int { } } -void ArchiveFileOperator::NewArchive2Fd( - const std::filesystem::path &target_directory, int fd, - const OperationCallback &cb) { +struct ArchiveReadClientData { + GFDataExchanger *ex; + std::array buf; + const std::byte *p_buf = buf.data(); +}; + +auto ArchiveReadCallback(struct archive *, void *client_data, + const void **buffer) -> ssize_t { + auto *rdata = static_cast(client_data); + *buffer = reinterpret_cast(rdata->p_buf); + return rdata->ex->Read(rdata->buf.data(), rdata->buf.size()); +} + +auto ArchiveWriteCallback(struct archive *, void *client_data, + const void *buffer, size_t length) -> ssize_t { + auto *ex = static_cast(client_data); + return ex->Write(static_cast(buffer), length); +} + +auto ArchiveCloseWriteCallback(struct archive *, void *client_data) -> int { + auto *ex = static_cast(client_data); + ex->CloseWrite(); + return 0; +} + +void ArchiveFileOperator::NewArchive2DataExchanger( + const std::filesystem::path &target_directory, + std::shared_ptr exchanger, const OperationCallback &cb) { RunIOOperaAsync( [=](const DataObjectPtr &data_object) -> GFError { - struct archive *archive; - struct archive_entry *entry; - std::array buff{}; + std::array buff{}; + auto ret = 0; + const auto base_path = target_directory.parent_path(); - archive = archive_write_new(); + auto *archive = archive_write_new(); archive_write_add_filter_none(archive); archive_write_set_format_pax_restricted(archive); - archive_write_open_fd(archive, fd); - for (const auto &file : - std::filesystem::recursive_directory_iterator(target_directory)) { - entry = archive_entry_new(); + archive_write_open(archive, exchanger.get(), nullptr, + ArchiveWriteCallback, ArchiveCloseWriteCallback); - auto file_path = file.path().string(); - archive_entry_set_pathname(entry, file_path.c_str()); - archive_entry_set_filetype(entry, AE_IFREG); - archive_entry_set_perm(entry, 0644); + auto *disk = archive_read_disk_new(); + archive_read_disk_set_standard_lookup(disk); + auto r = archive_read_disk_open(disk, target_directory.c_str()); - std::ifstream target_file(file_path, std::ifstream::binary); - if (!target_file) { - SPDLOG_ERROR("cannot open file: {}, abort...", file_path); - archive_entry_free(entry); - continue; + if (r != ARCHIVE_OK) { + SPDLOG_ERROR("archive_read_disk_open() failed: {}, abort...", + archive_error_string(disk)); + archive_read_free(disk); + archive_write_free(archive); + return -1; + } + + for (;;) { + auto *entry = archive_entry_new(); + r = archive_read_next_header2(disk, entry); + if (r == ARCHIVE_EOF) break; + if (r != ARCHIVE_OK) { + SPDLOG_ERROR( + "archive_read_next_header2() failed, ret: {}, explain: {}", r, + archive_error_string(disk)); + ret = -1; + break; } - target_file.seekg(0, std::ios::end); - auto file_size = target_file.tellg(); - target_file.seekg(0, std::ios::beg); - archive_entry_set_size(entry, file_size); + archive_read_disk_descend(disk); - archive_write_header(archive, entry); + // turn absolute path to relative path + archive_entry_set_pathname( + entry, std::filesystem::relative( + std::filesystem::path(archive_entry_pathname(entry)), + base_path) + .c_str()); - while (!target_file.eof()) { - target_file.read(buff.data(), buff.size()); - std::streamsize const bytes_read = target_file.gcount(); - archive_write_data(archive, buff.data(), bytes_read); + r = archive_write_header(archive, entry); + if (r < ARCHIVE_OK) { + SPDLOG_ERROR("archive_write_header() failed, ret: {}, explain: {} ", + r, archive_error_string(archive)); + continue; } + if (r == ARCHIVE_FATAL) { + SPDLOG_ERROR( + "archive_write_header() failed, ret: {}, explain: {}, " + "abort ...", + r, archive_error_string(archive)); + ret = -1; + break; + } + + if (r > ARCHIVE_FAILED) { + auto fd = open(archive_entry_sourcepath(entry), O_RDONLY); + auto len = read(fd, buff.data(), buff.size()); + while (len > 0) { + archive_write_data(archive, buff.data(), len); + len = read(fd, buff.data(), buff.size()); + } + close(fd); + } archive_entry_free(entry); } - archive_write_close(archive); + archive_read_free(disk); archive_write_free(archive); - close(fd); - return 0; + return ret; }, cb, "archive_write_new"); } -void ArchiveFileOperator::ExtractArchiveFromFd( - int fd, const std::filesystem::path &target_path, - const OperationCallback &cb) { - SPDLOG_DEBUG("extract archive from fd start, cuurent thread: {}", - QThread::currentThread()->currentThreadId()); +void ArchiveFileOperator::ExtractArchiveFromDataExchanger( + std::shared_ptr ex, + const std::filesystem::path &target_path, const OperationCallback &cb) { RunIOOperaAsync( [=](const DataObjectPtr &data_object) -> GFError { - SPDLOG_DEBUG("extract archive from fd processing, cuurent thread: {}", - QThread::currentThread()->currentThreadId()); - struct archive *archive; - struct archive *ext; - struct archive_entry *entry; - - archive = archive_read_new(); - ext = archive_write_disk_new(); - archive_write_disk_set_options(ext, 0); - -#ifndef NO_BZIP2_EXTRACT - archive_read_support_filter_bzip2(archive); -#endif -#ifndef NO_GZIP_EXTRACT - archive_read_support_filter_gzip(archive); -#endif -#ifndef NO_COMPRESS_EXTRACT - archive_read_support_filter_compress(archive); -#endif -#ifndef NO_TAR_EXTRACT - archive_read_support_format_tar(archive); -#endif -#ifndef NO_CPIO_EXTRACT - archive_read_support_format_cpio(archive); -#endif -#ifndef NO_LOOKUP - archive_write_disk_set_standard_lookup(ext); -#endif - - archive_read_open_fd(archive, fd, 8192); - SPDLOG_ERROR("archive_read_open_fd() failed: {}", - archive_error_string(archive)); - - while (archive_read_next_header(archive, &entry) == ARCHIVE_OK) { - SPDLOG_DEBUG("add file: {}, size: {}, bytes: {}, file type: {}", - archive_entry_pathname_utf8(entry), - archive_entry_size(entry), - archive_entry_filetype(entry)); - - auto file_path = - std::filesystem::path(archive_entry_pathname_utf8(entry)); - auto target_file_path = target_path / file_path; - archive_entry_set_pathname(entry, file_path.c_str()); - - auto r = archive_write_header(ext, entry); + auto *archive = archive_read_new(); + auto *ext = archive_write_disk_new(); + + auto r = archive_read_support_filter_all(archive); + if (r != ARCHIVE_OK) { + SPDLOG_ERROR("archive_read_support_filter_all(), ret: {}, reason: {}", + r, archive_error_string(archive)); + return r; + } + + r = archive_read_support_format_all(archive); + if (r != ARCHIVE_OK) { + SPDLOG_ERROR("archive_read_support_format_all(), ret: {}, reason: {}", + r, archive_error_string(archive)); + return r; + } + + auto rdata = ArchiveReadClientData{}; + rdata.ex = ex.get(); + + r = archive_read_open(archive, &rdata, nullptr, ArchiveReadCallback, + nullptr); + if (r != ARCHIVE_OK) { + SPDLOG_ERROR("archive_read_open(), ret: {}, reason: {}", r, + archive_error_string(archive)); + return r; + } + + r = archive_write_disk_set_options(ext, 0); + if (r != ARCHIVE_OK) { + SPDLOG_ERROR("archive_write_disk_set_options(), ret: {}, reason: {}", + r, archive_error_string(archive)); + return r; + } + + for (;;) { + struct archive_entry *entry; + r = archive_read_next_header(archive, &entry); + if (r == ARCHIVE_EOF) break; if (r != ARCHIVE_OK) { - SPDLOG_ERROR("archive_write_header() failed: {}", - archive_error_string(ext)); + SPDLOG_ERROR("archive_read_next_header(), ret: {}, reason: {}", r, + archive_error_string(archive)); + break; } - r = CopyData(archive, ext); + r = archive_write_header(ext, entry); if (r != ARCHIVE_OK) { - SPDLOG_ERROR("copy_data() failed: {}", archive_error_string(ext)); + SPDLOG_ERROR("archive_write_header(), ret: {}, reason: {}", r, + archive_error_string(archive)); + } else { + r = CopyData(archive, ext); } } - archive_read_free(archive); - archive_write_free(ext); - close(fd); + r = archive_read_free(archive); + if (r != ARCHIVE_OK) { + SPDLOG_ERROR("archive_read_free(), ret: {}, reason: {}", r, + archive_error_string(archive)); + } + r = archive_write_free(ext); + if (r != ARCHIVE_OK) { + SPDLOG_ERROR("archive_read_free(), ret: {}, reason: {}", r, + archive_error_string(archive)); + } + return 0; }, cb, "archive_read_new"); -- cgit v1.2.3 From 5030380bc5e155252655e2ad71e63c15271054c4 Mon Sep 17 00:00:00 2001 From: saturneric Date: Sun, 31 Dec 2023 14:10:31 +0800 Subject: feat: fully test basical operations of archive and file --- src/core/function/ArchiveFileOperator.cpp | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'src/core/function/ArchiveFileOperator.cpp') diff --git a/src/core/function/ArchiveFileOperator.cpp b/src/core/function/ArchiveFileOperator.cpp index b252f3a0..d186fee2 100644 --- a/src/core/function/ArchiveFileOperator.cpp +++ b/src/core/function/ArchiveFileOperator.cpp @@ -220,6 +220,11 @@ void ArchiveFileOperator::ExtractArchiveFromDataExchanger( break; } + archive_entry_set_pathname( + entry, (target_path / + std::filesystem::path(archive_entry_pathname(entry))) + .c_str()); + r = archive_write_header(ext, entry); if (r != ARCHIVE_OK) { SPDLOG_ERROR("archive_write_header(), ret: {}, reason: {}", r, -- cgit v1.2.3 From 644aa4397b03dbef73f8bfedc13925b51cad836b Mon Sep 17 00:00:00 2001 From: saturneric Date: Fri, 5 Jan 2024 20:55:15 +0800 Subject: feat: integrate logging api to core --- src/core/function/ArchiveFileOperator.cpp | 60 ++++++++++++++++--------------- 1 file changed, 32 insertions(+), 28 deletions(-) (limited to 'src/core/function/ArchiveFileOperator.cpp') diff --git a/src/core/function/ArchiveFileOperator.cpp b/src/core/function/ArchiveFileOperator.cpp index d186fee2..af311f3f 100644 --- a/src/core/function/ArchiveFileOperator.cpp +++ b/src/core/function/ArchiveFileOperator.cpp @@ -48,14 +48,14 @@ auto CopyData(struct archive *ar, struct archive *aw) -> int { r = archive_read_data_block(ar, &buff, &size, &offset); if (r == ARCHIVE_EOF) return (ARCHIVE_OK); if (r != ARCHIVE_OK) { - SPDLOG_ERROR("archive_read_data_block() failed: {}", - archive_error_string(ar)); + GF_CORE_LOG_ERROR("archive_read_data_block() failed: {}", + archive_error_string(ar)); return (r); } r = archive_write_data_block(aw, buff, size, offset); if (r != ARCHIVE_OK) { - SPDLOG_ERROR("archive_write_data_block() failed: {}", - archive_error_string(aw)); + GF_CORE_LOG_ERROR("archive_write_data_block() failed: {}", + archive_error_string(aw)); return (r); } } @@ -107,8 +107,8 @@ void ArchiveFileOperator::NewArchive2DataExchanger( auto r = archive_read_disk_open(disk, target_directory.c_str()); if (r != ARCHIVE_OK) { - SPDLOG_ERROR("archive_read_disk_open() failed: {}, abort...", - archive_error_string(disk)); + GF_CORE_LOG_ERROR("archive_read_disk_open() failed: {}, abort...", + archive_error_string(disk)); archive_read_free(disk); archive_write_free(archive); return -1; @@ -119,7 +119,7 @@ void ArchiveFileOperator::NewArchive2DataExchanger( r = archive_read_next_header2(disk, entry); if (r == ARCHIVE_EOF) break; if (r != ARCHIVE_OK) { - SPDLOG_ERROR( + GF_CORE_LOG_ERROR( "archive_read_next_header2() failed, ret: {}, explain: {}", r, archive_error_string(disk)); ret = -1; @@ -137,13 +137,14 @@ void ArchiveFileOperator::NewArchive2DataExchanger( r = archive_write_header(archive, entry); if (r < ARCHIVE_OK) { - SPDLOG_ERROR("archive_write_header() failed, ret: {}, explain: {} ", - r, archive_error_string(archive)); + GF_CORE_LOG_ERROR( + "archive_write_header() failed, ret: {}, explain: {} ", r, + archive_error_string(archive)); continue; } if (r == ARCHIVE_FATAL) { - SPDLOG_ERROR( + GF_CORE_LOG_ERROR( "archive_write_header() failed, ret: {}, explain: {}, " "abort ...", r, archive_error_string(archive)); @@ -180,15 +181,17 @@ void ArchiveFileOperator::ExtractArchiveFromDataExchanger( auto r = archive_read_support_filter_all(archive); if (r != ARCHIVE_OK) { - SPDLOG_ERROR("archive_read_support_filter_all(), ret: {}, reason: {}", - r, archive_error_string(archive)); + GF_CORE_LOG_ERROR( + "archive_read_support_filter_all(), ret: {}, reason: {}", r, + archive_error_string(archive)); return r; } r = archive_read_support_format_all(archive); if (r != ARCHIVE_OK) { - SPDLOG_ERROR("archive_read_support_format_all(), ret: {}, reason: {}", - r, archive_error_string(archive)); + GF_CORE_LOG_ERROR( + "archive_read_support_format_all(), ret: {}, reason: {}", r, + archive_error_string(archive)); return r; } @@ -198,15 +201,16 @@ void ArchiveFileOperator::ExtractArchiveFromDataExchanger( r = archive_read_open(archive, &rdata, nullptr, ArchiveReadCallback, nullptr); if (r != ARCHIVE_OK) { - SPDLOG_ERROR("archive_read_open(), ret: {}, reason: {}", r, - archive_error_string(archive)); + GF_CORE_LOG_ERROR("archive_read_open(), ret: {}, reason: {}", r, + archive_error_string(archive)); return r; } r = archive_write_disk_set_options(ext, 0); if (r != ARCHIVE_OK) { - SPDLOG_ERROR("archive_write_disk_set_options(), ret: {}, reason: {}", - r, archive_error_string(archive)); + GF_CORE_LOG_ERROR( + "archive_write_disk_set_options(), ret: {}, reason: {}", r, + archive_error_string(archive)); return r; } @@ -215,8 +219,8 @@ void ArchiveFileOperator::ExtractArchiveFromDataExchanger( r = archive_read_next_header(archive, &entry); if (r == ARCHIVE_EOF) break; if (r != ARCHIVE_OK) { - SPDLOG_ERROR("archive_read_next_header(), ret: {}, reason: {}", r, - archive_error_string(archive)); + GF_CORE_LOG_ERROR("archive_read_next_header(), ret: {}, reason: {}", + r, archive_error_string(archive)); break; } @@ -227,8 +231,8 @@ void ArchiveFileOperator::ExtractArchiveFromDataExchanger( r = archive_write_header(ext, entry); if (r != ARCHIVE_OK) { - SPDLOG_ERROR("archive_write_header(), ret: {}, reason: {}", r, - archive_error_string(archive)); + GF_CORE_LOG_ERROR("archive_write_header(), ret: {}, reason: {}", r, + archive_error_string(archive)); } else { r = CopyData(archive, ext); } @@ -236,13 +240,13 @@ void ArchiveFileOperator::ExtractArchiveFromDataExchanger( r = archive_read_free(archive); if (r != ARCHIVE_OK) { - SPDLOG_ERROR("archive_read_free(), ret: {}, reason: {}", r, - archive_error_string(archive)); + GF_CORE_LOG_ERROR("archive_read_free(), ret: {}, reason: {}", r, + archive_error_string(archive)); } r = archive_write_free(ext); if (r != ARCHIVE_OK) { - SPDLOG_ERROR("archive_read_free(), ret: {}, reason: {}", r, - archive_error_string(archive)); + GF_CORE_LOG_ERROR("archive_read_free(), ret: {}, reason: {}", r, + archive_error_string(archive)); } return 0; @@ -263,8 +267,8 @@ void ArchiveFileOperator::ListArchive( 10240); // Note 1 if (r != ARCHIVE_OK) return; while (archive_read_next_header(a, &entry) == ARCHIVE_OK) { - SPDLOG_DEBUG("File: {}", archive_entry_pathname(entry)); - SPDLOG_DEBUG("File Path: {}", archive_entry_pathname(entry)); + GF_CORE_LOG_DEBUG("File: {}", archive_entry_pathname(entry)); + GF_CORE_LOG_DEBUG("File Path: {}", archive_entry_pathname(entry)); archive_read_data_skip(a); // Note 2 } r = archive_read_free(a); // Note 3 -- cgit v1.2.3 From 6983b5c1dd82d159236ebd06cf17f071cc9c1ee9 Mon Sep 17 00:00:00 2001 From: saturneric Date: Fri, 12 Jan 2024 23:08:38 +0800 Subject: refactor: remove boost and use QString instead of std::filesystem::path --- src/core/function/ArchiveFileOperator.cpp | 35 +++++++++++++++++-------------- 1 file changed, 19 insertions(+), 16 deletions(-) (limited to 'src/core/function/ArchiveFileOperator.cpp') diff --git a/src/core/function/ArchiveFileOperator.cpp b/src/core/function/ArchiveFileOperator.cpp index af311f3f..60b8b004 100644 --- a/src/core/function/ArchiveFileOperator.cpp +++ b/src/core/function/ArchiveFileOperator.cpp @@ -87,13 +87,13 @@ auto ArchiveCloseWriteCallback(struct archive *, void *client_data) -> int { } void ArchiveFileOperator::NewArchive2DataExchanger( - const std::filesystem::path &target_directory, - std::shared_ptr exchanger, const OperationCallback &cb) { + const QString &target_directory, std::shared_ptr exchanger, + const OperationCallback &cb) { RunIOOperaAsync( [=](const DataObjectPtr &data_object) -> GFError { std::array buff{}; auto ret = 0; - const auto base_path = target_directory.parent_path(); + const auto base_path = QDir(QDir(target_directory).absolutePath()); auto *archive = archive_write_new(); archive_write_add_filter_none(archive); @@ -104,7 +104,13 @@ void ArchiveFileOperator::NewArchive2DataExchanger( auto *disk = archive_read_disk_new(); archive_read_disk_set_standard_lookup(disk); - auto r = archive_read_disk_open(disk, target_directory.c_str()); + +#ifdef WINDOWS + auto r = archive_read_disk_open_w( + disk, target_directory.toStdWString().c_str()); +#else + auto r = archive_read_disk_open(disk, target_directory.toUtf8()); +#endif if (r != ARCHIVE_OK) { GF_CORE_LOG_ERROR("archive_read_disk_open() failed: {}, abort...", @@ -130,10 +136,9 @@ void ArchiveFileOperator::NewArchive2DataExchanger( // turn absolute path to relative path archive_entry_set_pathname( - entry, std::filesystem::relative( - std::filesystem::path(archive_entry_pathname(entry)), - base_path) - .c_str()); + entry, + base_path.relativeFilePath(QString(archive_entry_pathname(entry))) + .toUtf8()); r = archive_write_header(archive, entry); if (r < ARCHIVE_OK) { @@ -172,8 +177,8 @@ void ArchiveFileOperator::NewArchive2DataExchanger( } void ArchiveFileOperator::ExtractArchiveFromDataExchanger( - std::shared_ptr ex, - const std::filesystem::path &target_path, const OperationCallback &cb) { + std::shared_ptr ex, const QString &target_path, + const OperationCallback &cb) { RunIOOperaAsync( [=](const DataObjectPtr &data_object) -> GFError { auto *archive = archive_read_new(); @@ -225,9 +230,8 @@ void ArchiveFileOperator::ExtractArchiveFromDataExchanger( } archive_entry_set_pathname( - entry, (target_path / - std::filesystem::path(archive_entry_pathname(entry))) - .c_str()); + entry, + (target_path + "/" + archive_entry_pathname(entry)).toUtf8()); r = archive_write_header(ext, entry); if (r != ARCHIVE_OK) { @@ -254,8 +258,7 @@ void ArchiveFileOperator::ExtractArchiveFromDataExchanger( cb, "archive_read_new"); } -void ArchiveFileOperator::ListArchive( - const std::filesystem::path &archive_path) { +void ArchiveFileOperator::ListArchive(const QString &archive_path) { struct archive *a; struct archive_entry *entry; int r; @@ -263,7 +266,7 @@ void ArchiveFileOperator::ListArchive( a = archive_read_new(); archive_read_support_filter_all(a); archive_read_support_format_all(a); - r = archive_read_open_filename(a, archive_path.u8string().c_str(), + r = archive_read_open_filename(a, archive_path.toUtf8(), 10240); // Note 1 if (r != ARCHIVE_OK) return; while (archive_read_next_header(a, &entry) == ARCHIVE_OK) { -- cgit v1.2.3 From e352e8e6b8d03a24ef5d52eef3e4d370807b5bbd Mon Sep 17 00:00:00 2001 From: saturneric Date: Wed, 17 Jan 2024 19:39:47 +0800 Subject: fix: find and slove some bugs --- src/core/function/ArchiveFileOperator.cpp | 2 -- 1 file changed, 2 deletions(-) (limited to 'src/core/function/ArchiveFileOperator.cpp') diff --git a/src/core/function/ArchiveFileOperator.cpp b/src/core/function/ArchiveFileOperator.cpp index 60b8b004..a4b90698 100644 --- a/src/core/function/ArchiveFileOperator.cpp +++ b/src/core/function/ArchiveFileOperator.cpp @@ -32,8 +32,6 @@ #include #include -#include - #include "core/utils/AsyncUtils.h" namespace GpgFrontend { -- cgit v1.2.3 From 37848811b6c8ca6c2a707431ec077c8098918afe Mon Sep 17 00:00:00 2001 From: saturneric Date: Thu, 18 Jan 2024 00:14:13 +0800 Subject: fix: improve code quality, docs and ci files --- src/core/function/ArchiveFileOperator.cpp | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/core/function/ArchiveFileOperator.cpp') diff --git a/src/core/function/ArchiveFileOperator.cpp b/src/core/function/ArchiveFileOperator.cpp index a4b90698..54c78628 100644 --- a/src/core/function/ArchiveFileOperator.cpp +++ b/src/core/function/ArchiveFileOperator.cpp @@ -158,9 +158,11 @@ void ArchiveFileOperator::NewArchive2DataExchanger( if (r > ARCHIVE_FAILED) { auto fd = open(archive_entry_sourcepath(entry), O_RDONLY); auto len = read(fd, buff.data(), buff.size()); + assert(len <= buff.size() && len > 0); while (len > 0) { archive_write_data(archive, buff.data(), len); len = read(fd, buff.data(), buff.size()); + assert(len <= buff.size() && len > 0); } close(fd); } -- cgit v1.2.3 From 38e4a656dbdd14cf7a6a91cda6c814b5290acd93 Mon Sep 17 00:00:00 2001 From: saturneric Date: Thu, 18 Jan 2024 13:32:25 +0800 Subject: fix: slove codacy issues --- src/core/function/ArchiveFileOperator.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'src/core/function/ArchiveFileOperator.cpp') diff --git a/src/core/function/ArchiveFileOperator.cpp b/src/core/function/ArchiveFileOperator.cpp index 54c78628..aad8b525 100644 --- a/src/core/function/ArchiveFileOperator.cpp +++ b/src/core/function/ArchiveFileOperator.cpp @@ -159,10 +159,22 @@ void ArchiveFileOperator::NewArchive2DataExchanger( auto fd = open(archive_entry_sourcepath(entry), O_RDONLY); auto len = read(fd, buff.data(), buff.size()); assert(len <= buff.size() && len > 0); + if (len == -1) { + GF_CORE_LOG_ERROR("read() failed, ret: {}, abort ...", r); + archive_entry_free(entry); + close(fd); + break; + } while (len > 0) { archive_write_data(archive, buff.data(), len); len = read(fd, buff.data(), buff.size()); assert(len <= buff.size() && len > 0); + if (len == -1) { + GF_CORE_LOG_ERROR("read() failed, ret: {}, abort ...", r); + archive_entry_free(entry); + close(fd); + break; + } } close(fd); } -- cgit v1.2.3 From 2b2b79b968ec832f1092dce5d78d407244e0d0da Mon Sep 17 00:00:00 2001 From: saturneric Date: Thu, 18 Jan 2024 13:34:39 +0800 Subject: fix: correct a fault --- src/core/function/ArchiveFileOperator.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'src/core/function/ArchiveFileOperator.cpp') diff --git a/src/core/function/ArchiveFileOperator.cpp b/src/core/function/ArchiveFileOperator.cpp index aad8b525..ef04d69f 100644 --- a/src/core/function/ArchiveFileOperator.cpp +++ b/src/core/function/ArchiveFileOperator.cpp @@ -160,19 +160,18 @@ void ArchiveFileOperator::NewArchive2DataExchanger( auto len = read(fd, buff.data(), buff.size()); assert(len <= buff.size() && len > 0); if (len == -1) { - GF_CORE_LOG_ERROR("read() failed, ret: {}, abort ...", r); + GF_CORE_LOG_ERROR("read() failed, ret: {}, abort ...", len); archive_entry_free(entry); close(fd); break; } + while (len > 0) { archive_write_data(archive, buff.data(), len); len = read(fd, buff.data(), buff.size()); assert(len <= buff.size() && len > 0); if (len == -1) { - GF_CORE_LOG_ERROR("read() failed, ret: {}, abort ...", r); - archive_entry_free(entry); - close(fd); + GF_CORE_LOG_ERROR("read() failed, ret: {}, abort ...", len); break; } } -- cgit v1.2.3 From e0d2c4021483e6e1366c1050ccb5a30e53e172bb Mon Sep 17 00:00:00 2001 From: saturneric Date: Thu, 18 Jan 2024 20:58:37 +0800 Subject: fix: add test cases and solve discovered issues --- src/core/function/ArchiveFileOperator.cpp | 11 ----------- 1 file changed, 11 deletions(-) (limited to 'src/core/function/ArchiveFileOperator.cpp') diff --git a/src/core/function/ArchiveFileOperator.cpp b/src/core/function/ArchiveFileOperator.cpp index ef04d69f..54c78628 100644 --- a/src/core/function/ArchiveFileOperator.cpp +++ b/src/core/function/ArchiveFileOperator.cpp @@ -159,21 +159,10 @@ void ArchiveFileOperator::NewArchive2DataExchanger( auto fd = open(archive_entry_sourcepath(entry), O_RDONLY); auto len = read(fd, buff.data(), buff.size()); assert(len <= buff.size() && len > 0); - if (len == -1) { - GF_CORE_LOG_ERROR("read() failed, ret: {}, abort ...", len); - archive_entry_free(entry); - close(fd); - break; - } - while (len > 0) { archive_write_data(archive, buff.data(), len); len = read(fd, buff.data(), buff.size()); assert(len <= buff.size() && len > 0); - if (len == -1) { - GF_CORE_LOG_ERROR("read() failed, ret: {}, abort ...", len); - break; - } } close(fd); } -- cgit v1.2.3 From f22ceca734868a4cb946c232f661aad72da01ded Mon Sep 17 00:00:00 2001 From: saturneric Date: Fri, 19 Jan 2024 20:10:17 +0800 Subject: fix: slove discovered faults and bugs --- src/core/function/ArchiveFileOperator.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'src/core/function/ArchiveFileOperator.cpp') diff --git a/src/core/function/ArchiveFileOperator.cpp b/src/core/function/ArchiveFileOperator.cpp index 54c78628..f1345f87 100644 --- a/src/core/function/ArchiveFileOperator.cpp +++ b/src/core/function/ArchiveFileOperator.cpp @@ -158,11 +158,9 @@ void ArchiveFileOperator::NewArchive2DataExchanger( if (r > ARCHIVE_FAILED) { auto fd = open(archive_entry_sourcepath(entry), O_RDONLY); auto len = read(fd, buff.data(), buff.size()); - assert(len <= buff.size() && len > 0); while (len > 0) { archive_write_data(archive, buff.data(), len); len = read(fd, buff.data(), buff.size()); - assert(len <= buff.size() && len > 0); } close(fd); } @@ -179,6 +177,7 @@ void ArchiveFileOperator::NewArchive2DataExchanger( void ArchiveFileOperator::ExtractArchiveFromDataExchanger( std::shared_ptr ex, const QString &target_path, const OperationCallback &cb) { + GF_CORE_LOG_INFO("target path: {}", target_path); RunIOOperaAsync( [=](const DataObjectPtr &data_object) -> GFError { auto *archive = archive_read_new(); -- cgit v1.2.3