From 9d2703c376d2ee7501ed78be8c962203fbfd098f Mon Sep 17 00:00:00 2001 From: Vincent Richard Date: Mon, 25 Mar 2013 12:32:48 +0100 Subject: Added support for charset conversion with ICU (thanks to Mehmet Bozkurt). --- src/charsetConverter.cpp | 3 +- src/charsetConverter_iconv.cpp | 18 ++++ src/charsetConverter_icu.cpp | 202 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 221 insertions(+), 2 deletions(-) create mode 100755 src/charsetConverter_icu.cpp (limited to 'src') diff --git a/src/charsetConverter.cpp b/src/charsetConverter.cpp index c2041476..5d3ab626 100644 --- a/src/charsetConverter.cpp +++ b/src/charsetConverter.cpp @@ -23,7 +23,6 @@ #include "vmime/charsetConverter.hpp" -#include "vmime/charsetConverter_iconv.hpp" #include "vmime/charsetConverter_idna.hpp" @@ -39,7 +38,7 @@ ref charsetConverter::create if (source == "idna" || dest == "idna") return vmime::create (source, dest, opts); else - return vmime::create (source, dest, opts); + return createGenericConverter(source, dest, opts); } diff --git a/src/charsetConverter_iconv.cpp b/src/charsetConverter_iconv.cpp index c5d3557e..43556298 100644 --- a/src/charsetConverter_iconv.cpp +++ b/src/charsetConverter_iconv.cpp @@ -21,6 +21,12 @@ // the GNU General Public License cover the whole combination. // +#include "vmime/config.hpp" + + +#if VMIME_CHARSETCONV_LIB_IS_ICONV + + #include "vmime/charsetConverter_iconv.hpp" #include "vmime/exception.hpp" @@ -82,6 +88,15 @@ namespace vmime { +// static +ref charsetConverter::createGenericConverter + (const charset& source, const charset& dest, + const charsetConverterOptions& opts) +{ + return vmime::create (source, dest, opts); +} + + charsetConverter_iconv::charsetConverter_iconv (const charset& source, const charset& dest, const charsetConverterOptions& opts) : m_desc(NULL), m_source(source), m_dest(dest), m_options(opts) @@ -433,3 +448,6 @@ void charsetFilteredOutputStream_iconv::flush() } // vmime + + +#endif // VMIME_CHARSETCONV_LIB_IS_ICONV diff --git a/src/charsetConverter_icu.cpp b/src/charsetConverter_icu.cpp new file mode 100755 index 00000000..c5a19dc6 --- /dev/null +++ b/src/charsetConverter_icu.cpp @@ -0,0 +1,202 @@ +// +// VMime library (http://www.vmime.org) +// Copyright (C) 2002-2013 Vincent Richard +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU General Public License as +// published by the Free Software Foundation; either version 3 of +// the License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +// +// Linking this library statically or dynamically with other modules is making +// a combined work based on this library. Thus, the terms and conditions of +// the GNU General Public License cover the whole combination. +// + +#include "vmime/config.hpp" + + +#if VMIME_CHARSETCONV_LIB_IS_ICU + + +#include "vmime/charsetConverter_icu.hpp" + +#include "vmime/exception.hpp" +#include "vmime/utility/inputStreamStringAdapter.hpp" +#include "vmime/utility/outputStreamStringAdapter.hpp" + + +extern "C" +{ +#ifndef VMIME_BUILDING_DOC + + #include + #include + +#endif // VMIME_BUILDING_DOC +} + + +#include + + +namespace vmime +{ + + +// static +ref charsetConverter::createGenericConverter + (const charset& source, const charset& dest, + const charsetConverterOptions& opts) +{ + return vmime::create (source, dest, opts); +} + + +charsetConverter_icu::charsetConverter_icu + (const charset& source, const charset& dest, const charsetConverterOptions& opts) + : m_from(NULL), m_to(NULL), m_source(source), m_dest(dest), m_options(opts) +{ + UErrorCode err = U_ZERO_ERROR; + m_from = ucnv_open(source.getName().c_str(), &err); + + if (err != U_ZERO_ERROR) + throw exceptions::charset_conv_error("Cannot initialize ICU [from] converter."); + + m_to = ucnv_open(dest.getName().c_str(), &err); + + if (err != U_ZERO_ERROR) + throw exceptions::charset_conv_error("Cannot initialize ICU [to] converter."); +} + + +charsetConverter_icu::~charsetConverter_icu() +{ + ucnv_close(m_from); + ucnv_close(m_to); +} + + +void charsetConverter_icu::convert(utility::inputStream& in, utility::outputStream& out) +{ + UErrorCode err = U_ZERO_ERROR; + + // From buffers + char cpInBuffer[16]; // stream data put here + size_t outSize = ucnv_getMinCharSize(m_from) * sizeof(cpInBuffer) * sizeof(UChar); + UChar* uOutBuffer = new UChar[outSize]; // Unicode chars end up here + + // Auto delete Unicode char buffer + vmime::utility::auto_ptr cleanup(uOutBuffer); + + // To buffers + // converted (char) data end up here + size_t cpOutBufferSz = ucnv_getMaxCharSize(m_to) * outSize; + char* cpOutBuffer = new char[cpOutBufferSz]; + vmime::utility::auto_ptr cleanupOut(cpOutBuffer); + + // Set replacement chars for when converting from Unicode to codepage + icu::UnicodeString substString(m_options.invalidSequence.c_str()); + ucnv_setSubstString(m_to, substString.getTerminatedBuffer(), -1, &err); + + if (U_FAILURE(err)) + throw exceptions::charset_conv_error("[ICU] Error setting replacement char."); + + // Input data available + while (!in.eof()) + { + // Read input data into buffer + size_t inLength = static_cast(in.read(cpInBuffer, sizeof(cpInBuffer))); + + // Beginning of read data + const char* source = &cpInBuffer[0]; + const char* sourceLimit = source + inLength; // end + 1 + + UBool flush = in.eof(); // is this last run? + + UErrorCode toErr; + + // Loop until all source has been processed + do + { + // Set up target pointers + UChar* target = uOutBuffer; + UChar* targetLimit = target + outSize; + + toErr = U_ZERO_ERROR; + ucnv_toUnicode(m_from, &target, targetLimit, + &source, sourceLimit, NULL, flush, &toErr); + + if (toErr != U_BUFFER_OVERFLOW_ERROR && U_FAILURE(toErr)) + throw exceptions::charset_conv_error("[ICU] Error converting to Unicode from " + m_source.getName()); + + // The Unicode source is the buffer just written and the limit + // is where the previous conversion stopped (target is moved in the conversion) + const UChar* uSource = uOutBuffer; + UChar* uSourceLimit = target; + UErrorCode fromErr; + + // Loop until converted chars are fully written + do + { + char* cpTarget = &cpOutBuffer[0]; + const char* cpTargetLimit = cpOutBuffer + cpOutBufferSz; + + fromErr = U_ZERO_ERROR; + + // Write converted bytes (Unicode) to destination codepage + ucnv_fromUnicode(m_to, &cpTarget, cpTargetLimit, + &uSource, uSourceLimit, NULL, flush, &fromErr); + + if (fromErr != U_BUFFER_OVERFLOW_ERROR && U_FAILURE(fromErr)) + throw exceptions::charset_conv_error("[ICU] Error converting from Unicode to " + m_dest.getName()); + + // Write to destination stream + out.write(cpOutBuffer, (cpTarget - cpOutBuffer)); + + } while (fromErr == U_BUFFER_OVERFLOW_ERROR); + + } while (toErr == U_BUFFER_OVERFLOW_ERROR); + } +} + + +void charsetConverter_icu::convert(const string& in, string& out) +{ + if (m_source == m_dest) + { + // No conversion needed + out = in; + return; + } + + out.clear(); + + utility::inputStreamStringAdapter is(in); + utility::outputStreamStringAdapter os(out); + + convert(is, os); + + os.flush(); +} + + +ref charsetConverter_icu::getFilteredOutputStream(utility::outputStream& os) +{ + // TODO: implement charsetFilteredOutputStream for ICU + return NULL; +} + + +} // vmime + + +#endif // VMIME_CHARSETCONV_LIB_IS_ICU -- cgit