diff --git a/SConstruct b/SConstruct index 5c5fad75..60ee5159 100644 --- a/SConstruct +++ b/SConstruct @@ -357,6 +357,7 @@ libvmimetest_sources = [ 'tests/parser/attachmentHelperTest.cpp', 'tests/parser/bodyPartTest.cpp', 'tests/parser/charsetTest.cpp', + 'tests/parser/charsetFilteredOutputStreamTest.cpp', 'tests/parser/datetimeTest.cpp', 'tests/parser/dispositionTest.cpp', 'tests/parser/emailAddressTest.cpp', diff --git a/tests/charset/Makefile b/tests/charset/Makefile deleted file mode 100644 index e46fafa5..00000000 --- a/tests/charset/Makefile +++ /dev/null @@ -1,5 +0,0 @@ - - -main: main.cpp ../../libvmime-debug.a - g++ -g -o main main.cpp -I../.. ../../libvmime-debug.a -lgnutls -lgsasl - diff --git a/tests/charset/main.cpp b/tests/charset/main.cpp deleted file mode 100644 index d8523461..00000000 --- a/tests/charset/main.cpp +++ /dev/null @@ -1,44 +0,0 @@ -// -// VMime library (http://www.vmime.org) -// Copyright (C) 2002-2013 Vincent Richard -// -// This program is free software; you can redistribute it and/or -// modify it under the terms of the GNU General Public License as -// published by the Free Software Foundation; either version 3 of -// the License, or (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// General Public License for more details. -// -// You should have received a copy of the GNU General Public License along -// with this program; if not, write to the Free Software Foundation, Inc., -// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -// -// Linking this library statically or dynamically with other modules is making -// a combined work based on this library. Thus, the terms and conditions of -// the GNU General Public License cover the whole combination. -// - -#include -#include - -#include "vmime/vmime.hpp" -#include "vmime/platforms/posix/posixHandler.hpp" - - -int main(int argc, char* argv[]) -{ - // VMime initialization - vmime::platform::setHandler(); - - - const vmime::string from(argv[1]); - const vmime::string to(argv[2]); - - vmime::utility::inputStreamAdapter in(std::cin); - vmime::utility::outputStreamAdapter out(std::cout); - - vmime::charset::convert(in, out, from, to); -} diff --git a/tests/charset/run-test.sh b/tests/charset/run-test.sh deleted file mode 100755 index 95846c06..00000000 --- a/tests/charset/run-test.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/sh - -TEST_DIR="./test-suites" -TEMP_DIR="/tmp" -PROGRAM="./main" - - -testFiles=`cd $TEST_DIR ; find . -maxdepth 1 -regex '\./[^\.]*\.in\..*' -type f` - -echo -echo Testing charset conversions -echo ===================================================================== - -for testFile in $testFiles ; do - - testName=`echo $testFile | sed 's/^\.\/\([^\.]*\).*/\1/'` - sourceCharset=`echo $testFile | sed 's/^\.\/[^\.]*\.[^\.]*\.\(.*\)/\1/'` - - testOutFiles=`cd $TEST_DIR ; find . -maxdepth 1 -regex "\./$testName\.out\..*" -type f` - - for testOutFile in $testOutFiles ; do - - destCharset=`echo $testOutFile | sed 's/^\.\/[^\.]*\.[^\.]*\.\(.*\)/\1/'` - - printf %20s "$testName " - printf %30s "$sourceCharset --> $destCharset : " - - $PROGRAM $sourceCharset $destCharset < $TEST_DIR/$testFile > $TEMP_DIR/vmime_result - - diff="diff $TEMP_DIR/vmime_result $TEST_DIR/$testOutFile" - res=`$diff` - - if [ "$res" = "" ] - then - echo "[OK]" - else - diffFile=$TEMP_DIR/vmime.charset.$testName.$sourceCharset.$destCharset.diff - echo "[NO: diff file is $diffFile]" - $diff > $diffFile - fi - - done - -done - -echo - diff --git a/tests/charset/test-suites/gnu.in.utf-8 b/tests/charset/test-suites/gnu.in.utf-8 deleted file mode 100644 index 782bb84e..00000000 --- a/tests/charset/test-suites/gnu.in.utf-8 +++ /dev/null @@ -1,48 +0,0 @@ -Korean Part. -GNU 프로젝트의 웹서버인 www.gnu.org에 오신 것을 환영합니다. GNU 프로젝트는 -GNU 시스템이라고 불리는 유닉스 형태의 완벽한 자유 소프트웨어 운영체제를 -개발하기 위해서 1984년부터 시작되었습니다. (GNU라는 단어는 ``GNU's Not -Unix''를 의미하는 재귀적 약어이며 ``그-뉴"라고 발음합니다.) GNU -시스템으로부터 연유한 다양한 종류의 운영체제들이 현재 ``리눅스''라는 -이름으로 사용되고 있지만, 시스템 커널로 리눅스를 탑재한 운영체제의 보다 -정확한 이름은 GNU/리눅스 시스템입니다. - -Japanese Part - -GNU プロジェクトのウェブサーバ、www.gnu.org へようこそ。1984 年のプロジェクト -開始以来、GNU プロジェクトでは、Unix に似た フリーソフトウェアの完全なオペレー -ティングシステム、GNU システムを開発して来ました(GNU とは「GNU's Not Unix(GNU -は Unix ではない)」の再帰頭字語であり、「グニュー」と発音されます)。現在、カーネ -ルとして Linux を用いた GNU システムのさまざまな変種が広く使われています。これ -らのシステムは「Linux」と呼ばれることが多いのですが、より正確には GNU/Linux -システム と呼ばれるものなのです。 - -Chinese Part. - -歡迎來到GNU 專案的伺服主機，www.gnu.org。 GNU 專案 開始於1984年，旨在發展一 -個 Unix-like 且為 自由軟體 的作業系統： GNU 系統。（GNU 是由``GNU's Not -Unix''所遞迴定義出的頭字語）；它的發音為"guh-NEW"。各種使用 Linux 作為核心的 GNU -作業系統正被廣泛的使用著；雖然這些系統通常被稱作 ``Linux''，但是它們應該更精 -確地被稱為 GNU/Linux 系統。 - -English Part. - -Welcome to the GNU Project web server, www.gnu.org. The GNU Project was -launched in 1984 to develop a complete Unix-like operating system which is -free software: the GNU system. (GNU is a recursive acronym for ``GNU's Not -Unix''; it is pronounced "guh-NEW".) Variants of the GNU operating system, -which use the kernel Linux, are now widely used; though these systems are -often referred to as ``Linux'', they are more accurately called GNU/Linux -systems. - -French Part. -Bienvenue sur le serveur web du projet GNU, www.gnu.org. Le projet GNU a été -lancé en 1984 afin de développer un système d'exploitation complet, -semblable à Unix et qui soit un logiciel libre: le système GNU. (« GNU » -est l'acronyme récursif the « GNU's Not Unix »; on le prononce « gnou » -avec un G audible) Des variantes du système d'exploitation GNU, basées sur -le noyau « Linux », sont utilisées largement à présent; bien que ces -systèmes soient communément appelés par le terme « Linux », ils le -seraient plus exactement par « GNU/Linux ». - - diff --git a/tests/charset/test-suites/gnu.out.iso-8859-1 b/tests/charset/test-suites/gnu.out.iso-8859-1 deleted file mode 100644 index bc11399e..00000000 --- a/tests/charset/test-suites/gnu.out.iso-8859-1 +++ /dev/null @@ -1,48 +0,0 @@ -Korean Part. -GNU ??????????????? ???????????? www.gnu.org??? ?????? ?????? ???????????????. GNU ??????????????? -GNU ?????????????????? ????????? ????????? ????????? ????????? ?????? ??????????????? ??????????????? -???????????? ????????? 1984????????? ?????????????????????. (GNU?????? ????????? ``GNU's Not -Unix''??? ???????????? ????????? ???????????? ``???-???"?????? ???????????????.) GNU -????????????????????? ????????? ????????? ????????? ?????????????????? ?????? ``?????????''?????? -???????????? ???????????? ?????????, ????????? ????????? ???????????? ????????? ??????????????? ?????? -????????? ????????? GNU/????????? ??????????????????. - -Japanese Part - -GNU ??????????????????????????????????????????www.gnu.org ??????????????????1984 ???????????????????????? -???????????????GNU ???????????????????????????Unix ????????? ??????????????????????????????????????????????????? -???????????????????????????GNU ???????????????????????????????????????(GNU ?????????GNU's Not Unix(GNU -??? Unix ????????????)????????????????????????????????????????????????????????????????????????)????????????????????? -???????????? Linux ???????????? GNU ??????????????????????????????????????????????????????????????????????????? -????????????????????????Linux?????????????????????????????????????????????????????????????????? GNU/Linux -???????????? ???????????????????????????????????? - -Chinese Part. - -????????????GNU ????????????????????????www.gnu.org??? GNU ?????? ?????????1984????????????????????? -??? Unix-like ?????? ???????????? ?????????????????? GNU ????????????GNU ??????``GNU's Not -Unix''???????????????????????????????????????????????????"guh-NEW"??????????????? Linux ??????????????? GNU -???????????????????????????????????????????????????????????????????????? ``Linux''??????????????????????????? -??????????????? GNU/Linux ????????? - -English Part. - -Welcome to the GNU Project web server, www.gnu.org. The GNU Project was -launched in 1984 to develop a complete Unix-like operating system which is -free software: the GNU system. (GNU is a recursive acronym for ``GNU's Not -Unix''; it is pronounced "guh-NEW".) Variants of the GNU operating system, -which use the kernel Linux, are now widely used; though these systems are -often referred to as ``Linux'', they are more accurately called GNU/Linux -systems. - -French Part. -Bienvenue sur le serveur web du projet GNU, www.gnu.org. Le projet GNU a t -lanc en 1984 afin de dvelopper un systme d'exploitation complet, -semblable Unix et qui soit un logiciel libre: le systme GNU. ( GNU -est l'acronyme rcursif the GNU's Not Unix ; on le prononce gnou -avec un G audible) Des variantes du systme d'exploitation GNU, bases sur -le noyau Linux , sont utilises largement prsent; bien que ces -systmes soient communment appels par le terme Linux , ils le -seraient plus exactement par GNU/Linux . - - diff --git a/tests/parser/charsetFilteredOutputStreamTest.cpp b/tests/parser/charsetFilteredOutputStreamTest.cpp new file mode 100644 index 00000000..fdfed2c6 --- /dev/null +++ b/tests/parser/charsetFilteredOutputStreamTest.cpp @@ -0,0 +1,205 @@ +// +// VMime library (http://www.vmime.org) +// Copyright (C) 2002-2013 Vincent Richard +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU General Public License as +// published by the Free Software Foundation; either version 3 of +// the License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +// +// Linking this library statically or dynamically with other modules is making +// a combined work based on this library. Thus, the terms and conditions of +// the GNU General Public License cover the whole combination. +// + +#include "tests/testUtils.hpp" + +#include "vmime/charset.hpp" +#include "vmime/charsetConverter.hpp" + +#include "charsetTestSuites.hpp" + + +VMIME_TEST_SUITE_BEGIN(charsetFilteredOutputStreamTest) + + VMIME_TEST_LIST_BEGIN + VMIME_TEST(testInputBufferUnderflow) + VMIME_TEST(testInvalidInput1) + VMIME_TEST(testStreamCopy) + VMIME_TEST(testOneByteAtTime) + VMIME_TEST(testVariableInputChunk) + VMIME_TEST_LIST_END + + + void testInputBufferUnderflow() + { + vmime::ref cc = + vmime::charsetConverter::create("utf-8", "iso-8859-1"); + + vmime::string output; + vmime::utility::outputStreamStringAdapter os(output); + vmime::ref cfos = cc->getFilteredOutputStream(os); + + // føo = 66 c3 b8 6f [UTF8] + // føo = 66 c3 b8 6f [latin1] + + cfos->write("\x66\xc3", 2); + + // Incomplete UTF-8 sequence was not converted + VASSERT_EQ("chunk 1", toHex("f"), toHex(output)); + + // Write second byte of UTF-8 sequence + cfos->write("\xb8\x6f", 2); + + VASSERT_EQ("chunk 2", toHex("f\xf8o"), toHex(output)); + } + + void testInvalidInput1() + { + vmime::string in("foo\xab\xcd\xef bar"); + vmime::string expectedOut("foo??? bar"); + + vmime::string actualOut; + vmime::utility::outputStreamStringAdapter osa(actualOut); + + vmime::ref conv = + vmime::charsetConverter::create + (vmime::charset("utf-8"), + vmime::charset("iso-8859-1")); + + vmime::ref os = + conv->getFilteredOutputStream(osa); + + vmime::utility::inputStreamStringAdapter is(in); + + vmime::utility::stream::value_type buffer[16]; + + for (int i = 0 ; !is.eof() ; ++i) + os->write(buffer, is.read(buffer, 1)); + + os->flush(); + + VASSERT_EQ("1", toHex(expectedOut), toHex(actualOut)); + } + + // Using 'bufferedStreamCopy' + void testStreamCopy() + { + for (unsigned int i = 0 ; i < charsetTestSuitesCount ; ++i) + { + const charsetTestSuiteStruct& entry = charsetTestSuites[i]; + + std::ostringstream testName; + testName << i << ": " << entry.fromCharset << " -> " << entry.toCharset; + + const unsigned int inLength = (entry.fromLength == 0 ? strlen(entry.fromBytes) : entry.fromLength); + vmime::string in(entry.fromBytes, entry.fromBytes + inLength); + + const unsigned int outLength = (entry.toLength == 0 ? strlen(entry.toBytes) : entry.toLength); + vmime::string expectedOut(entry.toBytes, entry.toBytes + outLength); + + vmime::string actualOut; + vmime::utility::outputStreamStringAdapter osa(actualOut); + + vmime::ref conv = + vmime::charsetConverter::create(entry.fromCharset, entry.toCharset); + + vmime::ref os = + conv->getFilteredOutputStream(osa); + + vmime::utility::inputStreamStringAdapter is(in); + + vmime::utility::bufferedStreamCopy(is, *os); + + os->flush(); + + VASSERT_EQ(testName.str(), toHex(expectedOut), toHex(actualOut)); + } + } + + // One byte at a time + void testOneByteAtTime() + { + for (unsigned int i = 0 ; i < charsetTestSuitesCount ; ++i) + { + const charsetTestSuiteStruct& entry = charsetTestSuites[i]; + + std::ostringstream testName; + testName << i << ": " << entry.fromCharset << " -> " << entry.toCharset; + + const unsigned int inLength = (entry.fromLength == 0 ? strlen(entry.fromBytes) : entry.fromLength); + vmime::string in(entry.fromBytes, entry.fromBytes + inLength); + + const unsigned int outLength = (entry.toLength == 0 ? strlen(entry.toBytes) : entry.toLength); + vmime::string expectedOut(entry.toBytes, entry.toBytes + outLength); + + vmime::string actualOut; + vmime::utility::outputStreamStringAdapter osa(actualOut); + + vmime::ref conv = + vmime::charsetConverter::create(entry.fromCharset, entry.toCharset); + + vmime::ref os = + conv->getFilteredOutputStream(osa); + + vmime::utility::inputStreamStringAdapter is(in); + + vmime::utility::stream::value_type buffer[16]; + + for (int i = 0 ; !is.eof() ; ++i) + os->write(buffer, is.read(buffer, 1)); + + os->flush(); + + VASSERT_EQ(testName.str(), toHex(expectedOut), toHex(actualOut)); + } + } + + // Variable chunks + void testVariableInputChunk() + { + for (unsigned int i = 0 ; i < charsetTestSuitesCount ; ++i) + { + const charsetTestSuiteStruct& entry = charsetTestSuites[i]; + + std::ostringstream testName; + testName << i << ": " << entry.fromCharset << " -> " << entry.toCharset; + + const unsigned int inLength = (entry.fromLength == 0 ? strlen(entry.fromBytes) : entry.fromLength); + vmime::string in(entry.fromBytes, entry.fromBytes + inLength); + + const unsigned int outLength = (entry.toLength == 0 ? strlen(entry.toBytes) : entry.toLength); + vmime::string expectedOut(entry.toBytes, entry.toBytes + outLength); + + vmime::string actualOut; + vmime::utility::outputStreamStringAdapter osa(actualOut); + + vmime::ref conv = + vmime::charsetConverter::create(entry.fromCharset, entry.toCharset); + + vmime::ref os = + conv->getFilteredOutputStream(osa); + + vmime::utility::inputStreamStringAdapter is(in); + + vmime::utility::stream::value_type buffer[16]; + + for (int i = 0 ; !is.eof() ; ++i) + os->write(buffer, is.read(buffer, (i % 5) + 1)); + + os->flush(); + + VASSERT_EQ(testName.str(), toHex(expectedOut), toHex(actualOut)); + } + } + +VMIME_TEST_SUITE_END diff --git a/tests/parser/charsetTest.cpp b/tests/parser/charsetTest.cpp index 678dad39..8725c530 100644 --- a/tests/parser/charsetTest.cpp +++ b/tests/parser/charsetTest.cpp @@ -23,68 +23,7 @@ #include "tests/testUtils.hpp" - -// Excerpt from http://www.gnu.org/ -static const vmime::charset inputCharset("gb2312"); -static const char inputBytes[] = - "\xbb\xb6\xd3\xad\xc0\xb4\xb5\xbd\x20\x47\x4e\x55\x20\xb9\xa4\xb3" - "\xcc\xb5\xc4\xcd\xf8\xd2\xb3\xcb\xc5\xb7\xfe\xd6\xf7\xbb\xfa\x20" - "\x77\x77\x77\x2e\x67\x6e\x75\x2e\x6f\x72\x67\x20\xa1\xa3\x20\x47" - "\x4e\x55\x20\xb9\xa4\xb3\xcc\x20\xbf\xaa\xca\xbc\xec\xb6\xd2\xbb" - "\xbe\xc5\xb0\xcb\xcb\xc4\xc4\xea\xa3\xac\xd6\xbc\xd4\xda\xb7\xa2" - "\xd5\xb9\xd2\xbb\xb8\xf6\xc0\xe0\xcb\xc6\x20\x55\x6e\x69\x78\x20" - "\xa3\xac\xc7\xd2\xce\xaa\x20\xd7\xd4\xd3\xc9\xc8\xed\xbc\xfe\x20" - "\xb5\xc4\xcd\xea\xd5\xfb\xb2\xd9\xd7\xf7\xcf\xb5\xcd\xb3\xa3\xba" - "\x20\x47\x4e\x55\x20\xcf\xb5\xcd\xb3\xa1\xa3\xa3\xa8\x47\x4e\x55" - "\x20\xca\xc7\xd3\xc9\xa1\xb0\x47\x4e\x55\x27\x73\x20\x4e\x6f\x74" - "\x20\x55\x6e\x69\x78\xa1\xb1\xcb\xf9\xb5\xdd\xbb\xd8\xb6\xa8\xd2" - "\xe5\xb3\xf6\xb5\xc4\xca\xd7\xd7\xd6\xc4\xb8\xcb\xf5\xd0\xb4\xd3" - "\xef\xa3\xbb\xcb\xfc\xb5\xc4\xb7\xa2\xd2\xf4\xce\xaa\xa1\xb0\x67" - "\x75\x68\x2d\x4e\x45\x57\xa1\xb1\xa3\xa9\xa1\xa3\xb8\xf7\xd6\xd6" - "\xca\xb9\xd3\xc3\x20\x4c\x69\x6e\x75\x78\x20\xd7\xf7\xce\xaa\xc4" - "\xda\xba\xcb\xb5\xc4\x20\x47\x4e\x55\x20\xb2\xd9\xd7\xf7\xcf\xb5" - "\xcd\xb3\xd5\xfd\xb1\xbb\xb9\xe3\xb7\xba\xb5\xd8\xca\xb9\xd3\xc3" - "\xd6\xf8\xa3\xbb\xcb\xe4\xc8\xbb\xd5\xe2\xd0\xa9\xcf\xb5\xcd\xb3" - "\xcd\xa8\xb3\xa3\xb1\xbb\xb3\xc6\xd7\xf7\xce\xaa\xa1\xb0\x4c\x69" - "\x6e\x75\x78\xa1\xb1\xa3\xac\xb5\xab\xca\xc7\xcb\xfc\xc3\xc7\xd3" - "\xa6\xb8\xc3\xb8\xfc\xbe\xab\xc8\xb7\xb5\xd8\xb1\xbb\xb3\xc6\xce" - "\xaa\x20\x47\x4e\x55\x2f\x4c\x69\x6e\x75\x78\x20\xcf\xb5\xcd\xb3" - "\x20\xa1\xa3\x0a"; - -static const vmime::charset outputCharset("utf-8"); -static const char outputBytes[] = - "\xe6\xac\xa2\xe8\xbf\x8e\xe6\x9d\xa5\xe5\x88\xb0\x20\x47\x4e\x55" - "\x20\xe5\xb7\xa5\xe7\xa8\x8b\xe7\x9a\x84\xe7\xbd\x91\xe9\xa1\xb5" - "\xe4\xbc\xba\xe6\x9c\x8d\xe4\xb8\xbb\xe6\x9c\xba\x20\x77\x77\x77" - "\x2e\x67\x6e\x75\x2e\x6f\x72\x67\x20\xe3\x80\x82\x20\x47\x4e\x55" - "\x20\xe5\xb7\xa5\xe7\xa8\x8b\x20\xe5\xbc\x80\xe5\xa7\x8b\xe6\x96" - "\xbc\xe4\xb8\x80\xe4\xb9\x9d\xe5\x85\xab\xe5\x9b\x9b\xe5\xb9\xb4" - "\xef\xbc\x8c\xe6\x97\xa8\xe5\x9c\xa8\xe5\x8f\x91\xe5\xb1\x95\xe4" - "\xb8\x80\xe4\xb8\xaa\xe7\xb1\xbb\xe4\xbc\xbc\x20\x55\x6e\x69\x78" - "\x20\xef\xbc\x8c\xe4\xb8\x94\xe4\xb8\xba\x20\xe8\x87\xaa\xe7\x94" - "\xb1\xe8\xbd\xaf\xe4\xbb\xb6\x20\xe7\x9a\x84\xe5\xae\x8c\xe6\x95" - "\xb4\xe6\x93\x8d\xe4\xbd\x9c\xe7\xb3\xbb\xe7\xbb\x9f\xef\xbc\x9a" - "\x20\x47\x4e\x55\x20\xe7\xb3\xbb\xe7\xbb\x9f\xe3\x80\x82\xef\xbc" - "\x88\x47\x4e\x55\x20\xe6\x98\xaf\xe7\x94\xb1\xe2\x80\x9c\x47\x4e" - "\x55\x27\x73\x20\x4e\x6f\x74\x20\x55\x6e\x69\x78\xe2\x80\x9d\xe6" - "\x89\x80\xe9\x80\x92\xe5\x9b\x9e\xe5\xae\x9a\xe4\xb9\x89\xe5\x87" - "\xba\xe7\x9a\x84\xe9\xa6\x96\xe5\xad\x97\xe6\xaf\x8d\xe7\xbc\xa9" - "\xe5\x86\x99\xe8\xaf\xad\xef\xbc\x9b\xe5\xae\x83\xe7\x9a\x84\xe5" - "\x8f\x91\xe9\x9f\xb3\xe4\xb8\xba\xe2\x80\x9c\x67\x75\x68\x2d\x4e" - "\x45\x57\xe2\x80\x9d\xef\xbc\x89\xe3\x80\x82\xe5\x90\x84\xe7\xa7" - "\x8d\xe4\xbd\xbf\xe7\x94\xa8\x20\x4c\x69\x6e\x75\x78\x20\xe4\xbd" - "\x9c\xe4\xb8\xba\xe5\x86\x85\xe6\xa0\xb8\xe7\x9a\x84\x20\x47\x4e" - "\x55\x20\xe6\x93\x8d\xe4\xbd\x9c\xe7\xb3\xbb\xe7\xbb\x9f\xe6\xad" - "\xa3\xe8\xa2\xab\xe5\xb9\xbf\xe6\xb3\x9b\xe5\x9c\xb0\xe4\xbd\xbf" - "\xe7\x94\xa8\xe8\x91\x97\xef\xbc\x9b\xe8\x99\xbd\xe7\x84\xb6\xe8" - "\xbf\x99\xe4\xba\x9b\xe7\xb3\xbb\xe7\xbb\x9f\xe9\x80\x9a\xe5\xb8" - "\xb8\xe8\xa2\xab\xe7\xa7\xb0\xe4\xbd\x9c\xe4\xb8\xba\xe2\x80\x9c" - "\x4c\x69\x6e\x75\x78\xe2\x80\x9d\xef\xbc\x8c\xe4\xbd\x86\xe6\x98" - "\xaf\xe5\xae\x83\xe4\xbb\xac\xe5\xba\x94\xe8\xaf\xa5\xe6\x9b\xb4" - "\xe7\xb2\xbe\xe7\xa1\xae\xe5\x9c\xb0\xe8\xa2\xab\xe7\xa7\xb0\xe4" - "\xb8\xba\x20\x47\x4e\x55\x2f\x4c\x69\x6e\x75\x78\x20\xe7\xb3\xbb" - "\xe7\xbb\x9f\x20\xe3\x80\x82\x0a"; - +#include "charsetTestSuites.hpp" VMIME_TEST_SUITE_BEGIN(charsetTest) @@ -93,156 +32,67 @@ VMIME_TEST_SUITE_BEGIN(charsetTest) // Test valid input VMIME_TEST(testConvertStringValid) VMIME_TEST(testConvertStreamValid) - VMIME_TEST(testFilterValid1) - VMIME_TEST(testFilterValid2) - VMIME_TEST(testFilterValid3) VMIME_TEST(testEncodingHebrew1255) - // Test invalid input - VMIME_TEST(testFilterInvalid1) - // IDNA VMIME_TEST(testEncodeIDNA) VMIME_TEST(testDecodeIDNA) - // TODO: more tests + VMIME_TEST(testUTF7Support) VMIME_TEST_LIST_END void testConvertStringValid() { - vmime::string in(inputBytes, sizeof(inputBytes) - 1); - vmime::string expectedOut(outputBytes, sizeof(outputBytes) - 1); - vmime::string actualOut; + for (unsigned int i = 0 ; i < charsetTestSuitesCount ; ++i) + { + const charsetTestSuiteStruct& entry = charsetTestSuites[i]; - vmime::charset::convert - (in, actualOut, inputCharset, outputCharset); + std::ostringstream testName; + testName << i << ": " << entry.fromCharset << " -> " << entry.toCharset; - VASSERT_EQ("1", toHex(expectedOut), toHex(actualOut)); + const unsigned int inLength = (entry.fromLength == 0 ? strlen(entry.fromBytes) : entry.fromLength); + vmime::string in(entry.fromBytes, entry.fromBytes + inLength); + + const unsigned int outLength = (entry.toLength == 0 ? strlen(entry.toBytes) : entry.toLength); + vmime::string expectedOut(entry.toBytes, entry.toBytes + outLength); + + vmime::string actualOut; + + vmime::charset::convert + (in, actualOut, entry.fromCharset, entry.toCharset); + + VASSERT_EQ(testName.str(), toHex(expectedOut), toHex(actualOut)); + } } void testConvertStreamValid() { - vmime::string in(inputBytes, sizeof(inputBytes) - 1); - vmime::string expectedOut(outputBytes, sizeof(outputBytes) - 1); + for (unsigned int i = 0 ; i < charsetTestSuitesCount ; ++i) + { + const charsetTestSuiteStruct& entry = charsetTestSuites[i]; - vmime::string actualOut; - vmime::utility::outputStreamStringAdapter os(actualOut); + std::ostringstream testName; + testName << i << ": " << entry.fromCharset << " -> " << entry.toCharset; - vmime::utility::inputStreamStringAdapter is(in); + const unsigned int inLength = (entry.fromLength == 0 ? strlen(entry.fromBytes) : entry.fromLength); + vmime::string in(entry.fromBytes, entry.fromBytes + inLength); - vmime::charset::convert - (is, os, inputCharset, outputCharset); + const unsigned int outLength = (entry.toLength == 0 ? strlen(entry.toBytes) : entry.toLength); + vmime::string expectedOut(entry.toBytes, entry.toBytes + outLength); - os.flush(); + vmime::string actualOut; + vmime::utility::outputStreamStringAdapter os(actualOut); - VASSERT_EQ("1", toHex(expectedOut), toHex(actualOut)); - } + vmime::utility::inputStreamStringAdapter is(in); - // Using 'bufferedStreamCopy' - void testFilterValid1() - { - vmime::string in(inputBytes, sizeof(inputBytes) - 1); - vmime::string expectedOut(outputBytes, sizeof(outputBytes) - 1); + vmime::charset::convert + (is, os, entry.fromCharset, entry.toCharset); - vmime::string actualOut; - vmime::utility::outputStreamStringAdapter osa(actualOut); + os.flush(); - vmime::ref conv = - vmime::charsetConverter::create(inputCharset, outputCharset); - - vmime::ref os = - conv->getFilteredOutputStream(osa); - - vmime::utility::inputStreamStringAdapter is(in); - - vmime::utility::bufferedStreamCopy(is, *os); - - os->flush(); - - VASSERT_EQ("1", toHex(expectedOut), toHex(actualOut)); - } - - // One byte at a time - void testFilterValid2() - { - vmime::string in(inputBytes, sizeof(inputBytes) - 1); - vmime::string expectedOut(outputBytes, sizeof(outputBytes) - 1); - - vmime::string actualOut; - vmime::utility::outputStreamStringAdapter osa(actualOut); - - vmime::ref conv = - vmime::charsetConverter::create(inputCharset, outputCharset); - - vmime::ref os = - conv->getFilteredOutputStream(osa); - - vmime::utility::inputStreamStringAdapter is(in); - - vmime::utility::stream::value_type buffer[16]; - - for (int i = 0 ; !is.eof() ; ++i) - os->write(buffer, is.read(buffer, 1)); - - os->flush(); - - VASSERT_EQ("1", toHex(expectedOut), toHex(actualOut)); - } - - // Variable chunks - void testFilterValid3() - { - vmime::string in(inputBytes, sizeof(inputBytes) - 1); - vmime::string expectedOut(outputBytes, sizeof(outputBytes) - 1); - - vmime::string actualOut; - vmime::utility::outputStreamStringAdapter osa(actualOut); - - vmime::ref conv = - vmime::charsetConverter::create(inputCharset, outputCharset); - - vmime::ref os = - conv->getFilteredOutputStream(osa); - - vmime::utility::inputStreamStringAdapter is(in); - - vmime::utility::stream::value_type buffer[16]; - - for (int i = 0 ; !is.eof() ; ++i) - os->write(buffer, is.read(buffer, (i % 5) + 1)); - - os->flush(); - - VASSERT_EQ("1", toHex(expectedOut), toHex(actualOut)); - } - - void testFilterInvalid1() - { - vmime::string in("foo\xab\xcd\xef bar"); - vmime::string expectedOut("foo??? bar"); - - vmime::string actualOut; - vmime::utility::outputStreamStringAdapter osa(actualOut); - - vmime::ref conv = - vmime::charsetConverter::create - (vmime::charset("utf-8"), - vmime::charset("iso-8859-1")); - - vmime::ref os = - conv->getFilteredOutputStream(osa); - - vmime::utility::inputStreamStringAdapter is(in); - - vmime::utility::stream::value_type buffer[16]; - - for (int i = 0 ; !is.eof() ; ++i) - os->write(buffer, is.read(buffer, 1)); - - os->flush(); - - VASSERT_EQ("1", toHex(expectedOut), toHex(actualOut)); + VASSERT_EQ(testName.str(), toHex(expectedOut), toHex(actualOut)); + } } void testEncodingHebrew1255() @@ -255,44 +105,6 @@ VMIME_TEST_SUITE_BEGIN(charsetTest) VASSERT_EQ("1", "=?windows-1255?B?6fn3+On5+Pfp6fk=?=", encoded); } - // Conversion to hexadecimal for easier debugging - static const vmime::string toHex(const vmime::string str) - { - static const char hexChars[] = "0123456789abcdef"; - - vmime::string res = "\n"; - - for (unsigned int i = 0 ; i < str.length() ; i += 16) - { - unsigned int r = std::min - (static_cast (16), str.length() - i); - - vmime::string hex; - vmime::string chr; - - for (unsigned int j = 0 ; j < r ; ++j) - { - const unsigned char c = str[i + j]; - - hex += hexChars[c / 16]; - hex += hexChars[c % 16]; - hex += " "; - - if (c >= 32 && c <= 127) - chr += c; - else - chr += '.'; - } - - for (unsigned int j = r ; j < 16 ; ++j) - hex += " "; - - res += hex + " " + chr + "\n"; - } - - return res; - } - static const vmime::string convertHelper (const vmime::string& in, const vmime::charset& csrc, const vmime::charset& cdest) { @@ -326,5 +138,11 @@ VMIME_TEST_SUITE_BEGIN(charsetTest) VASSERT_EQ("3.2", "إختبار", convertHelper("xn--kgbechtv", "idna", "utf-8")); } + void testUTF7Support() + { + // Ensure UTF-7 is supported, because it is used for IMAP + VASSERT_EQ("1", "VMime +ACY UTF-7 encoding", convertHelper("VMime & UTF-7 encoding", "utf-8", "utf-7")); + } + VMIME_TEST_SUITE_END diff --git a/tests/parser/charsetTestSuites.hpp b/tests/parser/charsetTestSuites.hpp new file mode 100644 index 00000000..b12b3e37 --- /dev/null +++ b/tests/parser/charsetTestSuites.hpp @@ -0,0 +1,103 @@ +// +// VMime library (http://www.vmime.org) +// Copyright (C) 2002-2013 Vincent Richard +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU General Public License as +// published by the Free Software Foundation; either version 3 of +// the License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +// +// Linking this library statically or dynamically with other modules is making +// a combined work based on this library. Thus, the terms and conditions of +// the GNU General Public License cover the whole combination. +// + + +struct charsetTestSuiteStruct +{ + const char* fromCharset; + const char* toCharset; + const char* fromBytes; + const unsigned int fromLength; + const char* toBytes; + const unsigned int toLength; +}; + + +static const charsetTestSuiteStruct charsetTestSuites[] = +{ + // Test data 1 (excerpt from http://www.gnu.org) + { + "gb2312", "utf-8", + + "\xbb\xb6\xd3\xad\xc0\xb4\xb5\xbd\x20\x47\x4e\x55\x20\xb9\xa4\xb3" + "\xcc\xb5\xc4\xcd\xf8\xd2\xb3\xcb\xc5\xb7\xfe\xd6\xf7\xbb\xfa\x20" + "\x77\x77\x77\x2e\x67\x6e\x75\x2e\x6f\x72\x67\x20\xa1\xa3\x20\x47" + "\x4e\x55\x20\xb9\xa4\xb3\xcc\x20\xbf\xaa\xca\xbc\xec\xb6\xd2\xbb" + "\xbe\xc5\xb0\xcb\xcb\xc4\xc4\xea\xa3\xac\xd6\xbc\xd4\xda\xb7\xa2" + "\xd5\xb9\xd2\xbb\xb8\xf6\xc0\xe0\xcb\xc6\x20\x55\x6e\x69\x78\x20" + "\xa3\xac\xc7\xd2\xce\xaa\x20\xd7\xd4\xd3\xc9\xc8\xed\xbc\xfe\x20" + "\xb5\xc4\xcd\xea\xd5\xfb\xb2\xd9\xd7\xf7\xcf\xb5\xcd\xb3\xa3\xba" + "\x20\x47\x4e\x55\x20\xcf\xb5\xcd\xb3\xa1\xa3\xa3\xa8\x47\x4e\x55" + "\x20\xca\xc7\xd3\xc9\xa1\xb0\x47\x4e\x55\x27\x73\x20\x4e\x6f\x74" + "\x20\x55\x6e\x69\x78\xa1\xb1\xcb\xf9\xb5\xdd\xbb\xd8\xb6\xa8\xd2" + "\xe5\xb3\xf6\xb5\xc4\xca\xd7\xd7\xd6\xc4\xb8\xcb\xf5\xd0\xb4\xd3" + "\xef\xa3\xbb\xcb\xfc\xb5\xc4\xb7\xa2\xd2\xf4\xce\xaa\xa1\xb0\x67" + "\x75\x68\x2d\x4e\x45\x57\xa1\xb1\xa3\xa9\xa1\xa3\xb8\xf7\xd6\xd6" + "\xca\xb9\xd3\xc3\x20\x4c\x69\x6e\x75\x78\x20\xd7\xf7\xce\xaa\xc4" + "\xda\xba\xcb\xb5\xc4\x20\x47\x4e\x55\x20\xb2\xd9\xd7\xf7\xcf\xb5" + "\xcd\xb3\xd5\xfd\xb1\xbb\xb9\xe3\xb7\xba\xb5\xd8\xca\xb9\xd3\xc3" + "\xd6\xf8\xa3\xbb\xcb\xe4\xc8\xbb\xd5\xe2\xd0\xa9\xcf\xb5\xcd\xb3" + "\xcd\xa8\xb3\xa3\xb1\xbb\xb3\xc6\xd7\xf7\xce\xaa\xa1\xb0\x4c\x69" + "\x6e\x75\x78\xa1\xb1\xa3\xac\xb5\xab\xca\xc7\xcb\xfc\xc3\xc7\xd3" + "\xa6\xb8\xc3\xb8\xfc\xbe\xab\xc8\xb7\xb5\xd8\xb1\xbb\xb3\xc6\xce" + "\xaa\x20\x47\x4e\x55\x2f\x4c\x69\x6e\x75\x78\x20\xcf\xb5\xcd\xb3" + "\x20\xa1\xa3\x0a", + 0, + + "\xe6\xac\xa2\xe8\xbf\x8e\xe6\x9d\xa5\xe5\x88\xb0\x20\x47\x4e\x55" + "\x20\xe5\xb7\xa5\xe7\xa8\x8b\xe7\x9a\x84\xe7\xbd\x91\xe9\xa1\xb5" + "\xe4\xbc\xba\xe6\x9c\x8d\xe4\xb8\xbb\xe6\x9c\xba\x20\x77\x77\x77" + "\x2e\x67\x6e\x75\x2e\x6f\x72\x67\x20\xe3\x80\x82\x20\x47\x4e\x55" + "\x20\xe5\xb7\xa5\xe7\xa8\x8b\x20\xe5\xbc\x80\xe5\xa7\x8b\xe6\x96" + "\xbc\xe4\xb8\x80\xe4\xb9\x9d\xe5\x85\xab\xe5\x9b\x9b\xe5\xb9\xb4" + "\xef\xbc\x8c\xe6\x97\xa8\xe5\x9c\xa8\xe5\x8f\x91\xe5\xb1\x95\xe4" + "\xb8\x80\xe4\xb8\xaa\xe7\xb1\xbb\xe4\xbc\xbc\x20\x55\x6e\x69\x78" + "\x20\xef\xbc\x8c\xe4\xb8\x94\xe4\xb8\xba\x20\xe8\x87\xaa\xe7\x94" + "\xb1\xe8\xbd\xaf\xe4\xbb\xb6\x20\xe7\x9a\x84\xe5\xae\x8c\xe6\x95" + "\xb4\xe6\x93\x8d\xe4\xbd\x9c\xe7\xb3\xbb\xe7\xbb\x9f\xef\xbc\x9a" + "\x20\x47\x4e\x55\x20\xe7\xb3\xbb\xe7\xbb\x9f\xe3\x80\x82\xef\xbc" + "\x88\x47\x4e\x55\x20\xe6\x98\xaf\xe7\x94\xb1\xe2\x80\x9c\x47\x4e" + "\x55\x27\x73\x20\x4e\x6f\x74\x20\x55\x6e\x69\x78\xe2\x80\x9d\xe6" + "\x89\x80\xe9\x80\x92\xe5\x9b\x9e\xe5\xae\x9a\xe4\xb9\x89\xe5\x87" + "\xba\xe7\x9a\x84\xe9\xa6\x96\xe5\xad\x97\xe6\xaf\x8d\xe7\xbc\xa9" + "\xe5\x86\x99\xe8\xaf\xad\xef\xbc\x9b\xe5\xae\x83\xe7\x9a\x84\xe5" + "\x8f\x91\xe9\x9f\xb3\xe4\xb8\xba\xe2\x80\x9c\x67\x75\x68\x2d\x4e" + "\x45\x57\xe2\x80\x9d\xef\xbc\x89\xe3\x80\x82\xe5\x90\x84\xe7\xa7" + "\x8d\xe4\xbd\xbf\xe7\x94\xa8\x20\x4c\x69\x6e\x75\x78\x20\xe4\xbd" + "\x9c\xe4\xb8\xba\xe5\x86\x85\xe6\xa0\xb8\xe7\x9a\x84\x20\x47\x4e" + "\x55\x20\xe6\x93\x8d\xe4\xbd\x9c\xe7\xb3\xbb\xe7\xbb\x9f\xe6\xad" + "\xa3\xe8\xa2\xab\xe5\xb9\xbf\xe6\xb3\x9b\xe5\x9c\xb0\xe4\xbd\xbf" + "\xe7\x94\xa8\xe8\x91\x97\xef\xbc\x9b\xe8\x99\xbd\xe7\x84\xb6\xe8" + "\xbf\x99\xe4\xba\x9b\xe7\xb3\xbb\xe7\xbb\x9f\xe9\x80\x9a\xe5\xb8" + "\xb8\xe8\xa2\xab\xe7\xa7\xb0\xe4\xbd\x9c\xe4\xb8\xba\xe2\x80\x9c" + "\x4c\x69\x6e\x75\x78\xe2\x80\x9d\xef\xbc\x8c\xe4\xbd\x86\xe6\x98" + "\xaf\xe5\xae\x83\xe4\xbb\xac\xe5\xba\x94\xe8\xaf\xa5\xe6\x9b\xb4" + "\xe7\xb2\xbe\xe7\xa1\xae\xe5\x9c\xb0\xe8\xa2\xab\xe7\xa7\xb0\xe4" + "\xb8\xba\x20\x47\x4e\x55\x2f\x4c\x69\x6e\x75\x78\x20\xe7\xb3\xbb" + "\xe7\xbb\x9f\x20\xe3\x80\x82\x0a", + 0 + } + +}; + +static const unsigned int charsetTestSuitesCount = sizeof(charsetTestSuites) / sizeof(charsetTestSuites[0]); diff --git a/tests/testUtils.cpp b/tests/testUtils.cpp index 56117e07..d008ed94 100644 --- a/tests/testUtils.cpp +++ b/tests/testUtils.cpp @@ -270,3 +270,39 @@ std::ostream& operator<<(std::ostream& os, const vmime::exception& e) } +const vmime::string toHex(const vmime::string str) +{ + static const char hexChars[] = "0123456789abcdef"; + + vmime::string res = "\n"; + + for (unsigned int i = 0 ; i < str.length() ; i += 16) + { + unsigned int r = std::min + (static_cast (16), str.length() - i); + + vmime::string hex; + vmime::string chr; + + for (unsigned int j = 0 ; j < r ; ++j) + { + const unsigned char c = str[i + j]; + + hex += hexChars[c / 16]; + hex += hexChars[c % 16]; + hex += " "; + + if (c >= 32 && c <= 127) + chr += c; + else + chr += '.'; + } + + for (unsigned int j = r ; j < 16 ; ++j) + hex += " "; + + res += hex + " " + chr + "\n"; + } + + return res; +} diff --git a/tests/testUtils.hpp b/tests/testUtils.hpp index 2e5b161f..4fa6b457 100644 --- a/tests/testUtils.hpp +++ b/tests/testUtils.hpp @@ -348,3 +348,6 @@ public: // Exception helper std::ostream& operator<<(std::ostream& os, const vmime::exception& e); + +// Conversion to hexadecimal for easier debugging +const vmime::string toHex(const vmime::string str);