GpgFrontend/vmime
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed #149: don't loose charset when fixing invalid broken words.

Browse Source
This commit is contained in:
Vincent Richard 2016-11-05 13:31:54 +01:00
parent 9a1753dc6c
commit 5424aa2381
2 changed files with 97 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
src/vmime/text.cpp
View File

@ -417,10 +417,9 @@ void text::fixBrokenWords(std::vector <shared_ptr <word> >& words)
// Fix words which encode a non-integral number of characters.
// This is not RFC-compliant, but we should be able to recover from it.
for (size_t i = 0, n = words.size() - 1 ; i < n ; ++i)
for (size_t i = 0, n = words.size() ; i < n - 1 ; ++i)
{
shared_ptr <word> w1 = words[i];
shared_ptr <word> w2 = words[i + 1];
// Check whether the word is valid
bool valid = false;
@ -436,22 +435,46 @@ void text::fixBrokenWords(std::vector <shared_ptr <word> >& words)
}
// If the current word is not valid, try to grab some bytes
// from the next word, to see whether it becomes valid.
// from the next words, to see whether it becomes valid.
if (!valid)
{
string buffer(w1->getBuffer());
buffer += w2->getBuffer();
size_t mergeWords = 1; // number of adjacent words to merge
for (size_t j = i + 1 ; j < n ; ++j)
{
shared_ptr <word> nextWord = words[j];
if (nextWord->getCharset() != w1->getCharset())
break;
buffer += nextWord->getBuffer();
++mergeWords;
}
if (mergeWords == 1)
{
// No adjacent word with same charset found
continue;
}
string::size_type firstInvalidByte;
valid = w1->getCharset().isValidText(buffer, &firstInvalidByte);
// Current word with additional bytes from the next word
// is now valid: adjust buffers of both words.
// Current word with additional bytes from the next words
// is now valid: adjust buffers of words.
w1->setBuffer(string(buffer.begin(), buffer.begin() + firstInvalidByte));
w2->setBuffer(string(buffer.begin() + firstInvalidByte, buffer.end()));
words[i + 1]->setBuffer(string(buffer.begin() + firstInvalidByte, buffer.end()));
// Remove unused words
for (size_t j = 0 ; j < mergeWords - 2 ; ++j)
{
words.erase(words.begin() + i + 2);
--n;
}
// If the next word is now empty, remove it
if (w2->getBuffer().empty())
if (words[i + 1]->getBuffer().empty())
{
words.erase(words.begin() + i + 1);
--n;

79
tests/parser/textTest.cpp
View File

@ -62,6 +62,7 @@ VMIME_TEST_SUITE_BEGIN(textTest)
VMIME_TEST(testWronglyPaddedB64Words)
VMIME_TEST(testFixBrokenWords)
VMIME_TEST(testUnknownCharset)
VMIME_TEST_LIST_END
@ -632,19 +633,21 @@ VMIME_TEST_SUITE_BEGIN(textTest)
("=?utf-8?Q?Gwena=C3?="
"=?utf-8?Q?=ABl?=", &outText);
VASSERT_EQ("1", "Gwena\xebl",
outText.getConvertedText(vmime::charset("iso-8859-1"), opts));
VASSERT_EQ("1.1", 1, outText.getWordCount());
VASSERT_EQ("1.2", "Gwena\xc3\xabl", outText.getWordAt(0)->getBuffer());
VASSERT_EQ("1.3", vmime::charset("utf-8"), outText.getWordAt(0)->getCharset());
// Test case 2
vmime::text::decodeAndUnfold
("=?utf-8?B?5Lit6Yu85qmf5qKw6JGj5LqL5pyDMTAz5bm056ysMDXlsYbn?="
"=?utf-8?B?rKwwN+asoeitsOeoiw==?=", &outText);
VASSERT_EQ("2", "\xe4\xb8\xad\xe9\x8b\xbc\xe6\xa9\x9f\xe6\xa2\xb0"
VASSERT_EQ("2.1", 1, outText.getWordCount());
VASSERT_EQ("2.2", "\xe4\xb8\xad\xe9\x8b\xbc\xe6\xa9\x9f\xe6\xa2\xb0"
"\xe8\x91\xa3\xe4\xba\x8b\xe6\x9c\x83\x31\x30\x33\xe5\xb9\xb4"
"\xe7\xac\xac\x30\x35\xe5\xb1\x86\xe7\xac\xac\x30\x37\xe6\xac"
"\xa1\xe8\xad\xb0\xe7\xa8\x8b",
outText.getConvertedText(vmime::charset("utf-8")));
"\xa1\xe8\xad\xb0\xe7\xa8\x8b", outText.getWordAt(0)->getBuffer());
VASSERT_EQ("2.3", vmime::charset("utf-8"), outText.getWordAt(0)->getCharset());
// Test case 3 (a character spanning over 3 words: 'を' = E3 82 92)
vmime::text::decodeAndUnfold
@ -652,15 +655,65 @@ VMIME_TEST_SUITE_BEGIN(textTest)
"=?utf-8?Q?=82?="
"=?utf-8?Q?=92xyz?=", &outText);
std::string out; // decode as UTF-16 then rencode to UTF-8 for easier comparison
vmime::charset::convert(
outText.getConvertedText(vmime::charset("utf-16"), opts),
out,
vmime::charset("utf-16"),
vmime::charset("utf-8")
);
VASSERT_EQ("3.1", 1, outText.getWordCount());
VASSERT_EQ("3.2", "abc\xe3\x82\x92xyz", outText.getWordAt(0)->getBuffer());
VASSERT_EQ("3.3", vmime::charset("utf-8"), outText.getWordAt(0)->getCharset());
VASSERT_EQ("3", "abc\xe3\x82\x92xyz", out);
// Test case 4 (remains invalid)
vmime::text::decodeAndUnfold
("=?utf-8?Q?abc=E3?="
"=?utf-8?Q?=82?="
"=?utf-8?Q?xy?="
"=?utf-8?Q?z?=", &outText);
VASSERT_EQ("4.1", 2, outText.getWordCount());
VASSERT_EQ("4.2", "abc", outText.getWordAt(0)->getBuffer());
VASSERT_EQ("4.3", vmime::charset("utf-8"), outText.getWordAt(0)->getCharset());
VASSERT_EQ("4.4", "\xe3\x82xyz", outText.getWordAt(1)->getBuffer());
VASSERT_EQ("4.5", vmime::charset("utf-8"), outText.getWordAt(1)->getCharset());
// Test case 5 (remains partially invalid)
vmime::text::decodeAndUnfold
("=?utf-8?Q?abc=E3?="
"=?utf-8?Q?=82?="
"=?utf-8?Q?\x92xy?="
"=?utf-8?Q?z\xc3?=", &outText);
VASSERT_EQ("5.1", 2, outText.getWordCount());
VASSERT_EQ("5.2", "abc\xe3\x82\x92xyz", outText.getWordAt(0)->getBuffer());
VASSERT_EQ("5.3", vmime::charset("utf-8"), outText.getWordAt(0)->getCharset());
VASSERT_EQ("5.4", "\xc3", outText.getWordAt(1)->getBuffer());
VASSERT_EQ("5.5", vmime::charset("utf-8"), outText.getWordAt(1)->getCharset());
}
void testUnknownCharset()
{
vmime::text t;
vmime::text::decodeAndUnfold("=?gb2312?B?wdaRY8PA?=", &t);
VASSERT_EQ("1.1", 1, t.getWordCount());
VASSERT_EQ("1.2", "\xc1\xd6\x91\x63\xc3\xc0", t.getWordAt(0)->getBuffer());
VASSERT_EQ("1.3", vmime::charset("gb2312"), t.getWordAt(0)->getCharset());
vmime::parsingContext ctx;
const vmime::string hfieldBuffer = "From: '=?gb2312?B?wdaRY8PA?=' <a.b@c.de>";
vmime::shared_ptr <vmime::headerField> hfield =
vmime::headerField::parseNext(ctx, hfieldBuffer, 0, hfieldBuffer.size());
vmime::shared_ptr <vmime::mailbox> hvalue =
hfield->getValue <vmime::mailbox>();
VASSERT_EQ("2.1", 3, hvalue->getName().getWordCount());
VASSERT_EQ("2.2", "'", hvalue->getName().getWordAt(0)->getBuffer());
VASSERT_EQ("2.3", vmime::charset("us-ascii"), hvalue->getName().getWordAt(0)->getCharset());
VASSERT_EQ("2.4", "\xc1\xd6\x91\x63\xc3\xc0", hvalue->getName().getWordAt(1)->getBuffer());
VASSERT_EQ("2.5", vmime::charset("gb2312"), hvalue->getName().getWordAt(1)->getCharset());
VASSERT_EQ("2.6", "'", hvalue->getName().getWordAt(2)->getBuffer());
VASSERT_EQ("2.7", vmime::charset("us-ascii"), hvalue->getName().getWordAt(2)->getCharset());
}
VMIME_TEST_SUITE_END