Added ability to parse directly from an input stream (eg. file). This allows very big messages to be parsed without loading the whole message data into memory.

8 files changed, 341 insertions, 0 deletions

diff --git a/src/utility/inputStreamAdapter.cpp b/src/utility/inputStreamAdapter.cpp
index b44b0841..441307b5 100644
--- a/src/utility/inputStreamAdapter.cpp
+++ b/src/utility/inputStreamAdapter.cpp
@@ -65,6 +65,18 @@ stream::size_type inputStreamAdapter::skip(const size_type count)
 }
 
 
+stream::size_type inputStreamAdapter::getPosition() const
+{
+	return m_stream.tellg();
+}
+
+
+void inputStreamAdapter::seek(const size_type pos)
+{
+	m_stream.seekg(pos, std::ios_base::beg);
+}
+
+
 } // utility
 } // vmime
 
diff --git a/src/utility/inputStreamByteBufferAdapter.cpp b/src/utility/inputStreamByteBufferAdapter.cpp
index 92e779fe..907f1eec 100644
--- a/src/utility/inputStreamByteBufferAdapter.cpp
+++ b/src/utility/inputStreamByteBufferAdapter.cpp
@@ -85,6 +85,19 @@ stream::size_type inputStreamByteBufferAdapter::skip(const size_type count)
 }
 
 
+stream::size_type inputStreamByteBufferAdapter::getPosition() const
+{
+	return m_pos;
+}
+
+
+void inputStreamByteBufferAdapter::seek(const size_type pos)
+{
+	if (pos <= m_length)
+		m_pos = pos;
+}
+
+
 } // utility
 } // vmime
 
diff --git a/src/utility/inputStreamStringAdapter.cpp b/src/utility/inputStreamStringAdapter.cpp
index 31c9fdaf..9b8fb0c7 100644
--- a/src/utility/inputStreamStringAdapter.cpp
+++ b/src/utility/inputStreamStringAdapter.cpp
@@ -89,6 +89,19 @@ stream::size_type inputStreamStringAdapter::skip(const size_type count)
 }
 
 
+stream::size_type inputStreamStringAdapter::getPosition() const
+{
+	return m_pos - m_begin;
+}
+
+
+void inputStreamStringAdapter::seek(const size_type pos)
+{
+	if (m_begin + pos <= m_end)
+		m_pos = m_begin + pos;
+}
+
+
 } // utility
 } // vmime
 
diff --git a/src/utility/inputStreamStringProxyAdapter.cpp b/src/utility/inputStreamStringProxyAdapter.cpp
index 5e4b60b9..feecddd5 100644
--- a/src/utility/inputStreamStringProxyAdapter.cpp
+++ b/src/utility/inputStreamStringProxyAdapter.cpp
@@ -84,6 +84,19 @@ stream::size_type inputStreamStringProxyAdapter::skip(const size_type count)
 }
 
 
+stream::size_type inputStreamStringProxyAdapter::getPosition() const
+{
+	return m_pos;
+}
+
+
+void inputStreamStringProxyAdapter::seek(const size_type pos)
+{
+	if (pos <= m_buffer.length())
+		m_pos = pos;
+}
+
+
 } // utility
 } // vmime
 
diff --git a/src/utility/parserInputStreamAdapter.cpp b/src/utility/parserInputStreamAdapter.cpp
new file mode 100644
index 00000000..7a38ef1b
--- /dev/null
+++ b/src/utility/parserInputStreamAdapter.cpp
@@ -0,0 +1,162 @@
+//
+// VMime library (http://www.vmime.org)
+// Copyright (C) 2002-2012 Vincent Richard <[email protected]>
+//
+// This program is free software; you can redistribute it and/or
+// modify it under the terms of the GNU General Public License as
+// published by the Free Software Foundation; either version 3 of
+// the License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+//
+// Linking this library statically or dynamically with other modules is making
+// a combined work based on this library.  Thus, the terms and conditions of
+// the GNU General Public License cover the whole combination.
+//
+
+#include "vmime/utility/parserInputStreamAdapter.hpp"
+
+
+namespace vmime {
+namespace utility {
+
+
+parserInputStreamAdapter::parserInputStreamAdapter(ref <seekableInputStream> stream)
+	: m_stream(stream)
+{
+}
+
+
+bool parserInputStreamAdapter::eof() const
+{
+	return m_stream->eof();
+}
+
+
+void parserInputStreamAdapter::reset()
+{
+	m_stream->reset();
+}
+
+
+stream::size_type parserInputStreamAdapter::read
+	(value_type* const data, const size_type count)
+{
+	return m_stream->read(data, count);
+}
+
+
+ref <seekableInputStream> parserInputStreamAdapter::getUnderlyingStream()
+{
+	return m_stream;
+}
+
+
+const string parserInputStreamAdapter::extract(const size_type begin, const size_type end) const
+{
+	const size_type initialPos = m_stream->getPosition();
+
+	try
+	{
+		value_type *buffer = new value_type[end - begin + 1];
+
+		m_stream->seek(begin);
+
+		const size_type readBytes = m_stream->read(buffer, end - begin);
+		buffer[readBytes] = '\0';
+
+		m_stream->seek(initialPos);
+
+		string str(buffer, buffer + readBytes);
+		delete [] buffer;
+
+		return str;
+	}
+	catch (...)
+	{
+		m_stream->seek(initialPos);
+		throw;
+	}
+}
+
+
+stream::size_type parserInputStreamAdapter::findNext
+	(const std::string& token, const size_type startPosition)
+{
+	static const unsigned int BUFFER_SIZE = 4096;
+
+	// Token must not be longer than BUFFER_SIZE/2
+	if (token.empty() || token.length() > BUFFER_SIZE / 2)
+		return npos;
+
+	const size_type initialPos = getPosition();
+
+	seek(startPosition);
+
+	try
+	{
+		value_type findBuffer[BUFFER_SIZE];
+		value_type* findBuffer1 = findBuffer;
+		value_type* findBuffer2 = findBuffer + (BUFFER_SIZE / 2) * sizeof(value_type);
+
+		size_type findBufferLen = 0;
+		size_type findBufferOffset = 0;
+
+		// Fill in initial buffer
+		findBufferLen = read(findBuffer, BUFFER_SIZE * sizeof(value_type));
+
+		for (;;)
+		{
+			// Find token
+			for (value_type *begin = findBuffer, *end = findBuffer + findBufferLen - token.length() ;
+			     begin <= end ; ++begin)
+			{
+				if (begin[0] == token[0] &&
+				    (token.length() == 1 ||
+				     memcmp(static_cast <const void *>(&begin[1]),
+				            static_cast <const void *>(token.data() + 1),
+				            token.length() - 1) == 0))
+				{
+					seek(initialPos);
+					return startPosition + findBufferOffset + (begin - findBuffer);
+				}
+			}
+
+			// Rotate buffer
+			memcpy(findBuffer1, findBuffer2, (BUFFER_SIZE / 2) * sizeof(value_type));
+
+			// Read more bytes
+			if (findBufferLen < BUFFER_SIZE && eof())
+			{
+				break;
+			}
+			else
+			{
+				const size_type bytesRead = read(findBuffer2, (BUFFER_SIZE / 2) * sizeof(value_type));
+				findBufferLen = (BUFFER_SIZE / 2) + bytesRead;
+				findBufferOffset += (BUFFER_SIZE / 2);
+			}
+		}
+
+		seek(initialPos);
+	}
+	catch (...)
+	{
+		seek(initialPos);
+		throw;
+	}
+
+	return npos;
+}
+
+
+} // utility
+} // vmime
+
diff --git a/src/utility/seekableInputStreamRegionAdapter.cpp b/src/utility/seekableInputStreamRegionAdapter.cpp
new file mode 100644
index 00000000..348618c0
--- /dev/null
+++ b/src/utility/seekableInputStreamRegionAdapter.cpp
@@ -0,0 +1,95 @@
+//
+// VMime library (http://www.vmime.org)
+// Copyright (C) 2002-2012 Vincent Richard <[email protected]>
+//
+// This program is free software; you can redistribute it and/or
+// modify it under the terms of the GNU General Public License as
+// published by the Free Software Foundation; either version 3 of
+// the License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+//
+// Linking this library statically or dynamically with other modules is making
+// a combined work based on this library.  Thus, the terms and conditions of
+// the GNU General Public License cover the whole combination.
+//
+
+#include "vmime/utility/seekableInputStreamRegionAdapter.hpp"
+
+
+namespace vmime {
+namespace utility {
+
+
+seekableInputStreamRegionAdapter::seekableInputStreamRegionAdapter
+	(ref <seekableInputStream> stream, const size_type begin, const size_type length)
+	: m_stream(stream), m_begin(begin), m_length(length)
+{
+}
+
+
+bool seekableInputStreamRegionAdapter::eof() const
+{
+	return getPosition() >= m_length;
+}
+
+
+void seekableInputStreamRegionAdapter::reset()
+{
+	m_stream->seek(m_begin);
+}
+
+
+stream::size_type seekableInputStreamRegionAdapter::read
+	(value_type* const data, const size_type count)
+{
+	if (getPosition() + count >= m_length)
+	{
+		const size_type remaining = m_length - getPosition();
+		return m_stream->read(data, remaining);
+	}
+	else
+	{
+		return m_stream->read(data, count);
+	}
+}
+
+
+stream::size_type seekableInputStreamRegionAdapter::skip(const size_type count)
+{
+	if (getPosition() + count >= m_length)
+	{
+		const size_type remaining = m_length - getPosition();
+		m_stream->skip(remaining);
+		return remaining;
+	}
+	else
+	{
+		m_stream->skip(count);
+		return count;
+	}
+}
+
+
+stream::size_type seekableInputStreamRegionAdapter::getPosition() const
+{
+	return m_stream->getPosition() - m_begin;
+}
+
+
+void seekableInputStreamRegionAdapter::seek(const size_type pos)
+{
+	m_stream->seek(m_begin + pos);
+}
+
+
+} // utility
+} // vmime
+
diff --git a/src/utility/stream.cpp b/src/utility/stream.cpp
index 1c940c2a..67c1f330 100644
--- a/src/utility/stream.cpp
+++ b/src/utility/stream.cpp
@@ -29,6 +29,9 @@ namespace vmime {
 namespace utility {
 
 
+const stream::size_type stream::npos = static_cast <size_type>(vmime::string::npos);
+
+
 stream::size_type stream::getBlockSize()
 {
 	return 32768;  // 32 KB
@@ -37,3 +40,4 @@ stream::size_type stream::getBlockSize()
 
 } // utility
 } // vmime
+
diff --git a/src/utility/streamUtils.cpp b/src/utility/streamUtils.cpp
index f1d3b9de..f7ea62f3 100644
--- a/src/utility/streamUtils.cpp
+++ b/src/utility/streamUtils.cpp
@@ -52,6 +52,35 @@ stream::size_type bufferedStreamCopy(inputStream& is, outputStream& os)
 }
 
 
+stream::size_type bufferedStreamCopyRange(inputStream& is, outputStream& os,
+	const stream::size_type start, const stream::size_type length)
+{
+	const stream::size_type blockSize =
+		std::min(is.getBlockSize(), os.getBlockSize());
+
+	is.skip(start);
+
+	std::vector <stream::value_type> vbuffer(blockSize);
+
+	stream::value_type* buffer = &vbuffer.front();
+	stream::size_type total = 0;
+
+	while (!is.eof() && total < length)
+	{
+		const stream::size_type remaining = std::min(length - total, blockSize);
+		const stream::size_type read = is.read(buffer, blockSize);
+
+		if (read != 0)
+		{
+			os.write(buffer, read);
+			total += read;
+		}
+	}
+
+	return total;
+}
+
+
 stream::size_type bufferedStreamCopy(inputStream& is, outputStream& os,
 	const stream::size_type length, progressListener* progress)
 {