Aren/haiku
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

NetServices: Refactor to make HttpParser stateful

Browse Source 
This change refactors various parts of the HTTP parsing. The HttpParser now
tracks what part of the message needs to be received next, and throws an error
if the object is used in the wrong way (when the caller requests to parse the
wrong part of he message).

The metadata about the transmission is now also saved in te parser. There is
partial work in there to start exposing the 'bytes written', which in
compressed streams is expected to differ from the bytes read. This is not
used yet.

This also simplifies the state tracking done by BHttpSession::Request.

Change-Id: I8532c6a5c8776456ea8bbccd6df7a44bac92b60d
This commit is contained in:
Niels Sascha Reedijk 2022-10-23 17:08:53 +01:00
parent a0b75afc0f
commit 3c9045fbbf
5 changed files with 444 additions and 212 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
src/kits/network/libnetservices2/HttpBuffer.cpp
View File

@ -74,38 +74,20 @@ HttpBuffer::ReadFrom(BDataIO* source, std::optional<size_t> maxSize)
}
/*!
\brief Use BDataIO::WriteExactly() on target to write the contents of the buffer.
*/
void
HttpBuffer::WriteExactlyTo(BDataIO* target)
{
if (RemainingBytes() == 0)
return;
auto status = target->WriteExactly(fBuffer.data() + fCurrentOffset, RemainingBytes());
if (status != B_OK) {
throw BNetworkRequestError("BDataIO::WriteExactly()", BNetworkRequestError::SystemError,
status);
}
// Entire buffer is written; reset internal buffer
Clear();
}
/*!
\brief Write the contents of the buffer through the helper \a func.
\param func Handle the actual writing. The function accepts a pointer and a size as inputs
and should return the number of actual written bytes, which may be fewer than the number
of available bytes.
\returns the actual number of bytes written to the \a func.
*/
void
size_t
HttpBuffer::WriteTo(HttpTransferFunction func , std::optional<size_t> maxSize)
{
if (RemainingBytes() == 0)
return;
return 0;
auto size = RemainingBytes();
if (maxSize.has_value() && *maxSize < size)
@ -116,6 +98,8 @@ HttpBuffer::WriteTo(HttpTransferFunction func , std::optional<size_t> maxSize)
throw BRuntimeError(__PRETTY_FUNCTION__, "More bytes written than were made available");
fCurrentOffset += bytesWritten;
return bytesWritten;
}

5
src/kits/network/libnetservices2/HttpBuffer.h
View File

@ -27,8 +27,9 @@ public:
HttpBuffer(size_t capacity = 8*1024);
ssize_t ReadFrom(BDataIO* source, std::optional<size_t> maxSize = std::nullopt);
void WriteExactlyTo(BDataIO* target);
void WriteTo(HttpTransferFunction func,
size_t WriteTo(HttpTransferFunction func,
std::optional<size_t> maxSize = std::nullopt);
void WriteExactlyTo(HttpTransferFunction func,
std::optional<size_t> maxSize = std::nullopt);
std::optional<BString> GetNextLine();

448
src/kits/network/libnetservices2/HttpParser.cpp
View File

@ -19,6 +19,24 @@ using namespace std::literals;
using namespace BPrivate::Network;
// #pragma mark -- HttpParser
/*!
\brief Explicitly mark the response as having no content.
This is done in cases where the request was a HEAD request. Setting it to no content, will
instruct the parser to move to completion after all the header fields have been parsed.
*/
void
HttpParser::SetNoContent() noexcept
{
if (fStreamState > HttpInputStreamState::Fields)
debugger("Cannot set the parser to no content after parsing of the body has started");
fBodyType = HttpBodyType::NoContent;
};
/*!
\brief Parse the status from the \a buffer and store it in \a status.
@ -30,6 +48,9 @@ using namespace BPrivate::Network;
bool
HttpParser::ParseStatus(HttpBuffer& buffer, BHttpStatus& status)
{
if (fStreamState != HttpInputStreamState::StatusLine)
debugger("The Status line has already been parsed");
auto statusLine = buffer.GetNextLine();
if (!statusLine)
return false;
@ -54,6 +75,7 @@ HttpParser::ParseStatus(HttpBuffer& buffer, BHttpStatus& status)
status.text = std::move(statusLine.value());
fStatus.code = status.code; // cache the status code
fStreamState = HttpInputStreamState::Fields;
return true;
}
@ -76,8 +98,11 @@ HttpParser::ParseStatus(HttpBuffer& buffer, BHttpStatus& status)
bool
HttpParser::ParseFields(HttpBuffer& buffer, BHttpFields& fields)
{
if (fStreamState != HttpInputStreamState::Fields)
debugger("The parser is not expecting header fields at this point");
auto fieldLine = buffer.GetNextLine();
while (fieldLine && !fieldLine.value().IsEmpty()){
// Parse next header line
fields.AddField(fieldLine.value());
@ -91,17 +116,20 @@ HttpParser::ParseFields(HttpBuffer& buffer, BHttpFields& fields)
// Determine the properties for the body
// RFC 7230 section 3.3.3 has a prioritized list of 7 rules around determining the body:
std::optional<off_t> bodyBytesTotal = std::nullopt;
if (fBodyType == HttpBodyType::NoContent
|| fStatus.StatusCode() == BHttpStatusCode::NoContent
|| fStatus.StatusCode() == BHttpStatusCode::NotModified) {
// [1] In case of HEAD (set previously), status codes 1xx (TODO!), status code 204 or 304, no content
// [2] NOT SUPPORTED: when doing a CONNECT request, no content
fBodyType = HttpBodyType::NoContent;
fStreamState = HttpInputStreamState::Done;
} else if (auto header = fields.FindField("Transfer-Encoding"sv);
header != fields.end() && header->Value() == "chunked"sv) {
// [3] If there is a Transfer-Encoding heading set to 'chunked'
// TODO: support the more advanced rules in the RFC around the meaning of this field
fBodyType = HttpBodyType::Chunked;
fStreamState = HttpInputStreamState::Body;
} else if (fields.CountFields("Content-Length"sv) > 0) {
// [4] When there is no Transfer-Encoding, then look for Content-Encoding:
// - If there are more than one, the values must match
@ -120,12 +148,13 @@ HttpParser::ParseFields(HttpBuffer& buffer, BHttpFields& fields)
}
}
}
auto bodyBytesTotal = std::stol(contentLength);
if (bodyBytesTotal == 0)
bodyBytesTotal = std::stol(contentLength);
if (*bodyBytesTotal == 0) {
fBodyType = HttpBodyType::NoContent;
else {
fStreamState = HttpInputStreamState::Done;
} else {
fBodyType = HttpBodyType::FixedSize;
fBodyBytesTotal = bodyBytesTotal;
fStreamState = HttpInputStreamState::Body;
}
} catch (const std::logic_error& e) {
throw BNetworkRequestError(__PRETTY_FUNCTION__,
@ -136,94 +165,241 @@ HttpParser::ParseFields(HttpBuffer& buffer, BHttpFields& fields)
// [6] Applies to request messages only (this is a response)
// [7] If nothing else then the received message is all data until connection close
// (this is the default)
fStreamState = HttpInputStreamState::Body;
}
// Content-Encoding
// Set up the body parser based on the logic above.
switch (fBodyType) {
case HttpBodyType::VariableSize:
fBodyParser = std::make_unique<HttpRawBodyParser>();
break;
case HttpBodyType::FixedSize:
fBodyParser = std::make_unique<HttpRawBodyParser>(*bodyBytesTotal);
break;
case HttpBodyType::Chunked:
fBodyParser = std::make_unique<HttpChunkedBodyParser>();
break;
case HttpBodyType::NoContent:
default:
return true;
}
// Check Content-Encoding for compression
auto header = fields.FindField("Content-Encoding"sv);
if (header != fields.end()
&& (header->Value() == "gzip" || header->Value() == "deflate"))
{
_SetGzipCompression();
fBodyParser = std::make_unique<HttpBodyDecompression>(std::move(fBodyParser));
}
return true;
}
/*!
\brief Parse the body from the \a buffer and use \a writeToBody function to save.
The \a readEnd parameter indicates to the parser that the buffer currently contains all the
expected data for this request.
*/
size_t
HttpParser::ParseBody(HttpBuffer& buffer, HttpTransferFunction writeToBody)
HttpParser::ParseBody(HttpBuffer& buffer, HttpTransferFunction writeToBody, bool readEnd)
{
if (fBodyType == HttpBodyType::NoContent) {
return 0;
} else if (fBodyType == HttpBodyType::Chunked) {
return _ParseBodyChunked(buffer, writeToBody);
} else {
return _ParseBodyRaw(buffer, writeToBody);
}
if (fStreamState < HttpInputStreamState::Body || fStreamState == HttpInputStreamState::Done)
debugger("The parser is not in the correct state to parse a body");
auto parseResult = fBodyParser->ParseBody(buffer, writeToBody, readEnd);
if (parseResult.complete)
fStreamState = HttpInputStreamState::Done;
return parseResult.bytesParsed;
}
/*!
\brief Parse the Gzip compression from the body.
\brief Return if the body is currently expecting to having content.
\exception std::bad_alloc in case there is an error allocating memory.
This may change if the header fields have not yet been parsed, as these may contain
instructions about the body having no content.
*/
void
HttpParser::_SetGzipCompression()
bool
HttpParser::HasContent() const noexcept
{
fDecompressorStorage = std::make_unique<BMallocIO>();
BDataIO* stream = nullptr;
auto result = BZlibCompressionAlgorithm()
.CreateDecompressingOutputStream(fDecompressorStorage.get(), nullptr, stream);
if (result != B_OK) {
throw BNetworkRequestError(
"BZlibCompressionAlgorithm().CreateCompressingOutputStream",
BNetworkRequestError::SystemError, result);
}
fDecompressingStream = std::unique_ptr<BDataIO>(stream);
return fBodyType != HttpBodyType::NoContent;
}
/*!
\brief Parse the body from the \a buffer and use \a writeToBody function to save.
\brief Return the total size of the body, if known.
*/
size_t
HttpParser::_ParseBodyRaw(HttpBuffer& buffer, HttpTransferFunction writeToBody)
std::optional<off_t>
HttpParser::BodyBytesTotal() const noexcept
{
if (fBodyBytesTotal && (fTransferredBodySize + static_cast<off_t>(buffer.RemainingBytes()))
> *fBodyBytesTotal)
throw BNetworkRequestError(__PRETTY_FUNCTION__, BNetworkRequestError::ProtocolError);
if (fBodyParser)
return fBodyParser->TotalBodySize();
return std::nullopt;
}
/*!
\brief Return the number of body bytes transferred from the response.
*/
off_t
HttpParser::BodyBytesTransferred() const noexcept
{
if (fBodyParser)
return fBodyParser->TransferredBodySize();
return 0;
}
/*!
\brief Check if the body is fully parsed.
*/
bool
HttpParser::Complete() const noexcept
{
return fStreamState == HttpInputStreamState::Done;
}
// #pragma mark -- HttpBodyParser
/*!
\brief Default implementation to return std::nullopt.
*/
std::optional<off_t>
HttpBodyParser::TotalBodySize() const noexcept
{
return std::nullopt;
}
/*!
\brief Return the number of body bytes read from the stream so far.
For chunked transfers, this excludes the chunk headers and other metadata.
*/
off_t
HttpBodyParser::TransferredBodySize() const noexcept
{
return fTransferredBodySize;
}
// #pragma mark -- HttpRawBodyParser
/*!
\brief Construct a HttpRawBodyParser with an unknown content size.
*/
HttpRawBodyParser::HttpRawBodyParser()
{
}
/*!
\brief Construct a HttpRawBodyParser with expected \a bodyBytesTotal size.
*/
HttpRawBodyParser::HttpRawBodyParser(off_t bodyBytesTotal)
:
fBodyBytesTotal(bodyBytesTotal)
{
}
/*!
\brief Parse a regular (non-chunked) body from a buffer.
The buffer is parsed into a target using the \a writeToBody function.
The \a readEnd argument indicates whether the current \a buffer contains all the expected data.
In case the total body size is known, and the remaining bytes in the buffer are smaller than
the expected remainder, a ProtocolError will be raised. The data in the buffer will *not* be
copied to the target.
Also, if the body size is known, and the data in the \a buffer is larger than the expected
expected length, then it will only read the bytes needed and leave the remainder in the buffer.
It is required that the \a writeToBody function writes all the bytes it is asked to; this
method does not support partial writes and throws an exception when it fails.
\exception BNetworkRequestError In case the buffer contains too little or invalid data.
\returns The number of bytes parsed from the \a buffer.
*/
BodyParseResult
HttpRawBodyParser::ParseBody(HttpBuffer& buffer, HttpTransferFunction writeToBody, bool readEnd)
{
auto bytesToRead = buffer.RemainingBytes();
auto readEnd = fBodyBytesTotal.value()
== (fTransferredBodySize + static_cast<off_t>(bytesToRead));
if (fBodyBytesTotal) {
auto expectedRemainingBytes = *fBodyBytesTotal - fTransferredBodySize;
if (expectedRemainingBytes < static_cast<off_t>(buffer.RemainingBytes()))
bytesToRead = expectedRemainingBytes;
else if (readEnd && expectedRemainingBytes > static_cast<off_t>(buffer.RemainingBytes())) {
throw BNetworkRequestError(__PRETTY_FUNCTION__, BNetworkRequestError::ProtocolError,
"Message body is incomplete; less data received than expected");
}
}
auto bytesRead = _ReadChunk(buffer, writeToBody, bytesToRead, readEnd);
// Copy the data
auto bytesRead = buffer.WriteTo(writeToBody, bytesToRead);
fTransferredBodySize += bytesRead;
return bytesRead;
if (bytesRead != bytesToRead) {
// Fail if not all expected bytes are written.
throw BNetworkRequestError(__PRETTY_FUNCTION__, BNetworkRequestError::SystemError,
"Could not write all available body bytes to the target.");
}
if (fBodyBytesTotal && *fBodyBytesTotal == fTransferredBodySize)
return {bytesRead, bytesRead, true};
else
return {bytesRead, bytesRead, false};
}
/*!
\brief Parse the body from the \a buffer and use \a writeToBody function to save.
\brief Override default implementation and return known body size (or std::nullopt)
*/
size_t
HttpParser::_ParseBodyChunked(HttpBuffer& buffer, HttpTransferFunction writeToBody)
std::optional<off_t>
HttpRawBodyParser::TotalBodySize() const noexcept
{
return fBodyBytesTotal;
}
// #pragma mark -- HttpChunkedBodyParser
/*!
\brief Parse a chunked body from a buffer.
The contents of the cunks are copied into a target using the \a writeToBody function.
The \a readEnd argument indicates whether the current \a buffer contains all the expected data.
In case the chunk argument indicates that more data was to come, an exception is thrown.
It is required that the \a writeToBody function writes all the bytes it is asked to; this
method does not support partial writes and throws an exception when it fails.
\exception BNetworkRequestError In case there is an error parsing the buffer, or there is too
little data.
\returns The number of bytes parsed from the \a buffer.
*/
BodyParseResult
HttpChunkedBodyParser::ParseBody(HttpBuffer& buffer, HttpTransferFunction writeToBody, bool readEnd)
{
size_t totalBytesRead = 0;
while (buffer.RemainingBytes() > 0) {
switch (fBodyState) {
case HttpBodyInputStreamState::ChunkSize:
switch (fChunkParserState) {
case ChunkSize:
{
// Read the next chunk size from the buffer; if unsuccesful wait for more data
auto chunkSizeString = buffer.GetNextLine();
if (!chunkSizeString)
return totalBytesRead;
return {totalBytesRead, totalBytesRead, false};
auto chunkSizeStr = std::string(chunkSizeString.value().String());
try {
size_t pos = 0;
@ -241,37 +417,40 @@ HttpParser::_ParseBodyChunked(HttpBuffer& buffer, HttpTransferFunction writeToBo
}
if (fRemainingChunkSize > 0)
fBodyState = HttpBodyInputStreamState::Chunk;
fChunkParserState = Chunk;
else
fBodyState = HttpBodyInputStreamState::Trailers;
fChunkParserState = Trailers;
break;
}
case HttpBodyInputStreamState::Chunk:
case Chunk:
{
size_t bytesToRead;
bool readEnd = false;
if (fRemainingChunkSize > static_cast<off_t>(buffer.RemainingBytes()))
bytesToRead = buffer.RemainingBytes();
else {
readEnd = true;
else
bytesToRead = fRemainingChunkSize;
auto bytesRead = buffer.WriteTo(writeToBody, bytesToRead);
if (bytesRead != bytesToRead) {
// Fail if not all expected bytes are written.
throw BNetworkRequestError(__PRETTY_FUNCTION__, BNetworkRequestError::SystemError,
"Could not write all available body bytes to the target.");
}
auto bytesRead = _ReadChunk(buffer, writeToBody, bytesToRead, readEnd);
fTransferredBodySize += bytesRead;
totalBytesRead += bytesRead;
fRemainingChunkSize -= bytesRead;
if (fRemainingChunkSize == 0)
fBodyState = HttpBodyInputStreamState::ChunkEnd;
fChunkParserState = ChunkEnd;
break;
}
case HttpBodyInputStreamState::ChunkEnd:
case ChunkEnd:
{
if (buffer.RemainingBytes() < 2) {
// not enough data in the buffer to finish the chunk
return totalBytesRead;
return {totalBytesRead, totalBytesRead, false};
}
auto chunkEndString = buffer.GetNextLine();
if (!chunkEndString || chunkEndString.value().Length() != 0) {
@ -280,106 +459,117 @@ HttpParser::_ParseBodyChunked(HttpBuffer& buffer, HttpTransferFunction writeToBo
BNetworkRequestError::ProtocolError);
}
fBodyState = HttpBodyInputStreamState::ChunkSize;
fChunkParserState = ChunkSize;
break;
}
case HttpBodyInputStreamState::Trailers:
case Trailers:
{
auto trailerString = buffer.GetNextLine();
if (!trailerString) {
// More data to come
return totalBytesRead;
return {totalBytesRead, totalBytesRead, false};
}
if (trailerString.value().Length() > 0) {
// Ignore empty trailers for now
// TODO: review if the API should support trailing headers
} else {
fBodyState = HttpBodyInputStreamState::Done;
return totalBytesRead;
fChunkParserState = Complete;
return {totalBytesRead, totalBytesRead, true};
}
break;
}
case HttpBodyInputStreamState::Done:
return totalBytesRead;
case Complete:
return {totalBytesRead, totalBytesRead, true};
}
}
return totalBytesRead;
return {totalBytesRead, totalBytesRead, false};
}
// #pragma mark -- HttpBodyDecompression
/*!
\brief Set up a decompression stream that decompresses the data read by \a bodyParser.
*/
HttpBodyDecompression::HttpBodyDecompression(std::unique_ptr<HttpBodyParser> bodyParser)
{
fDecompressorStorage = std::make_unique<BMallocIO>();
BDataIO* stream = nullptr;
auto result = BZlibCompressionAlgorithm()
.CreateDecompressingOutputStream(fDecompressorStorage.get(), nullptr, stream);
if (result != B_OK) {
throw BNetworkRequestError(
"BZlibCompressionAlgorithm().CreateCompressingOutputStream",
BNetworkRequestError::SystemError, result);
}
fDecompressingStream = std::unique_ptr<BDataIO>(stream);
fBodyParser = std::move(bodyParser);
}
/*!
\brief Check if the body is fully parsed.
\brief Read a compressed body into a target..
The stream captures chunked or raw data, and decompresses it. The decompressed data is then
copied into a target using the \a writeToBody function.
The \a readEnd argument indicates whether the current \a buffer contains all the expected data.
It is up for the underlying parser to determine if more data was expected, and therefore, if
there is an error.
It is required that the \a writeToBody function writes all the bytes it is asked to; this
method does not support partial writes and throws an exception when it fails.
\exception BNetworkRequestError In case there is an error parsing the buffer, or there is too
little data.
\returns The number of bytes parsed from the \a buffer.
*/
bool
HttpParser::Complete() const noexcept
BodyParseResult
HttpBodyDecompression::ParseBody(HttpBuffer& buffer, HttpTransferFunction writeToBody, bool readEnd)
{
if (fBodyType == HttpBodyType::Chunked)
return fBodyState == HttpBodyInputStreamState::Done;
else if (fBodyType == HttpBodyType::FixedSize)
return fBodyBytesTotal.value() == fTransferredBodySize;
else if (fBodyType == HttpBodyType::NoContent)
return true;
else
return false;
// Get the underlying raw or chunked parser to write data to our decompressionstream
auto parseResults = fBodyParser->ParseBody(buffer, [this](const std::byte* buffer, size_t bufferSize){
auto status = fDecompressingStream->WriteExactly(buffer, bufferSize);
if (status != B_OK) {
throw BNetworkRequestError("BDataIO::WriteExactly()",
BNetworkRequestError::SystemError, status);
}
return bufferSize;
}, readEnd);
fTransferredBodySize += parseResults.bytesParsed;
if (readEnd || parseResults.complete) {
// No more bytes expected so flush out the final bytes
if (auto status = fDecompressingStream->Flush(); status != B_OK) {
throw BNetworkRequestError("BZlibDecompressionStream::Flush()",
BNetworkRequestError::SystemError, status);
}
}
size_t bytesWritten = 0;
if (auto bodySize = fDecompressorStorage->Position(); bodySize > 0) {
bytesWritten = writeToBody(static_cast<const std::byte*>(fDecompressorStorage->Buffer()), bodySize);
if (static_cast<off_t>(bytesWritten) != bodySize) {
throw BNetworkRequestError(__PRETTY_FUNCTION__,
BNetworkRequestError::SystemError, B_PARTIAL_WRITE);
}
fDecompressorStorage->Seek(0, SEEK_SET);
}
return {parseResults.bytesParsed, bytesWritten, parseResults.complete};
}
/*!
\brief Read a chunk of data from the buffer and write it to the output.
If there is a compression algorithm applied, then it passes through the compression first.
\param buffer The buffer to read from
\param writeToBody The function that can write data from the buffer to the body.
\param size The maximum size to read from the buffer. When larger than the buffer size, the
remaining bytes from the buffer are read.
\param flush Setting this parameter will force the decompression to write out all data, if
applicable. Set when all the data has been received.
\exception BNetworkRequestError When there was any error with any of the system cals.
\brief Return the TotalBodySize() from the underlying chunked or raw parser.
*/
size_t
HttpParser::_ReadChunk(HttpBuffer& buffer, HttpTransferFunction writeToBody, size_t size, bool flush)
std::optional<off_t>
HttpBodyDecompression::TotalBodySize() const noexcept
{
if (size == 0)
return 0;
if (size > buffer.RemainingBytes())
size = buffer.RemainingBytes();
if (fDecompressingStream) {
buffer.WriteTo([this](const std::byte* buffer, size_t bufferSize){
auto status = fDecompressingStream->WriteExactly(buffer, bufferSize);
if (status != B_OK) {
throw BNetworkRequestError("BDataIO::WriteExactly()",
BNetworkRequestError::SystemError, status);
}
return bufferSize;
}, size);
if (flush) {
// No more bytes expected so flush out the final bytes
if (auto status = fDecompressingStream->Flush(); status != B_OK)
throw BNetworkRequestError("BZlibDecompressionStream::Flush()",
BNetworkRequestError::SystemError, status);
}
if (auto bodySize = fDecompressorStorage->Position(); bodySize > 0) {
auto bytesWritten
= writeToBody(static_cast<const std::byte*>(fDecompressorStorage->Buffer()),
bodySize);
if (static_cast<off_t>(bytesWritten) != bodySize) {
throw BNetworkRequestError(__PRETTY_FUNCTION__,
BNetworkRequestError::SystemError, B_PARTIAL_WRITE);
}
fDecompressorStorage->Seek(0, SEEK_SET);
}
} else {
// Write the body directly to the target
buffer.WriteTo(writeToBody, size);
}
return size;
return fBodyParser->TotalBodySize();
}

124
src/kits/network/libnetservices2/HttpParser.h
View File

@ -23,11 +23,10 @@ namespace Network {
using HttpTransferFunction = std::function<size_t (const std::byte*, size_t)>;
enum class HttpBodyInputStreamState {
ChunkSize,
ChunkEnd,
Chunk,
Trailers,
enum class HttpInputStreamState {
StatusLine,
Fields,
Body,
Done
};
@ -40,54 +39,105 @@ enum class HttpBodyType {
};
struct BodyParseResult {
size_t bytesParsed;
size_t bytesWritten;
bool complete;
};
class HttpBodyParser;
class HttpParser {
public:
HttpParser() {};
HttpParser() {};
// Explicitly mark request as having no content
void SetNoContent() { fBodyType = HttpBodyType::NoContent; };
void SetNoContent() noexcept;
// HTTP Header
bool ParseStatus(HttpBuffer& buffer, BHttpStatus& status);
bool ParseFields(HttpBuffer& buffer, BHttpFields& fields);
// HTTP Body
size_t ParseBody(HttpBuffer& buffer, HttpTransferFunction writeToBody);
void SetConnectionClosed();
// Parse data from response
bool ParseStatus(HttpBuffer& buffer, BHttpStatus& status);
bool ParseFields(HttpBuffer& buffer, BHttpFields& fields);
size_t ParseBody(HttpBuffer& buffer, HttpTransferFunction writeToBody,
bool readEnd);
HttpInputStreamState State() const noexcept { return fStreamState; }
// Details on the body status
bool HasContent() const noexcept { return fBodyType != HttpBodyType::NoContent; };
std::optional<off_t> BodyBytesTotal() const noexcept { return fBodyBytesTotal; };
off_t BodyBytesTransferred() const noexcept { return fTransferredBodySize; };
bool Complete() const noexcept;
bool HasContent() const noexcept;
std::optional<off_t> BodyBytesTotal() const noexcept;
off_t BodyBytesTransferred() const noexcept;
bool Complete() const noexcept;
private:
void _SetGzipCompression();
size_t _ParseBodyRaw(HttpBuffer& buffer, HttpTransferFunction writeToBody);
size_t _ParseBodyChunked(HttpBuffer& buffer, HttpTransferFunction writeToBody);
size_t _ReadChunk(HttpBuffer& buffer, HttpTransferFunction writeToBody,
size_t maxSize, bool flush);
off_t fHeaderBytes = 0;
BHttpStatus fStatus;
HttpInputStreamState fStreamState = HttpInputStreamState::StatusLine;
// Body
HttpBodyType fBodyType = HttpBodyType::VariableSize;
std::unique_ptr<HttpBodyParser> fBodyParser = nullptr;
};
class HttpBodyParser {
public:
virtual BodyParseResult ParseBody(HttpBuffer& buffer,
HttpTransferFunction writeToBody, bool readEnd) = 0;
virtual std::optional<off_t> TotalBodySize() const noexcept;
off_t TransferredBodySize() const noexcept;
protected:
off_t fTransferredBodySize = 0;
};
class HttpRawBodyParser : public HttpBodyParser {
public:
HttpRawBodyParser();
HttpRawBodyParser(off_t bodyBytesTotal);
virtual BodyParseResult ParseBody(HttpBuffer& buffer,
HttpTransferFunction writeToBody, bool readEnd) override;
virtual std::optional<off_t> TotalBodySize() const noexcept override;
private:
off_t fHeaderBytes = 0;
BHttpStatus fStatus;
std::optional<off_t> fBodyBytesTotal;
};
// Body type
HttpBodyType fBodyType = HttpBodyType::VariableSize;
// Support for chunked transfers
HttpBodyInputStreamState fBodyState = HttpBodyInputStreamState::ChunkSize;
off_t fRemainingChunkSize = 0;
bool fLastChunk = false;
class HttpChunkedBodyParser : public HttpBodyParser {
public:
virtual BodyParseResult ParseBody(HttpBuffer& buffer,
HttpTransferFunction writeToBody, bool readEnd) override;
// Receive stats
std::optional<off_t> fBodyBytesTotal = 0;
off_t fTransferredBodySize = 0;
private:
enum {
ChunkSize,
ChunkEnd,
Chunk,
Trailers,
Complete
} fChunkParserState = ChunkSize;
off_t fRemainingChunkSize = 0;
bool fLastChunk = false;
};
// Optional decompression
std::unique_ptr<BMallocIO> fDecompressorStorage = nullptr;
std::unique_ptr<BDataIO> fDecompressingStream = nullptr;
class HttpBodyDecompression : public HttpBodyParser {
public:
HttpBodyDecompression(
std::unique_ptr<HttpBodyParser> bodyParser);
virtual BodyParseResult ParseBody(HttpBuffer& buffer,
HttpTransferFunction writeToBody, bool readEnd) override;
virtual std::optional<off_t> TotalBodySize() const noexcept;
private:
std::unique_ptr<HttpBodyParser> fBodyParser;
std::unique_ptr<BMallocIO> fDecompressorStorage;
std::unique_ptr<BDataIO> fDecompressingStream;
};

51
src/kits/network/libnetservices2/HttpSession.cpp
View File

@ -43,7 +43,7 @@ using namespace BPrivate::Network;
/*!
\brief Maximum size of the HTTP Header lines of the message.
In the RFC there is no maximum, but we need to prevent the situation where we keep growing the
internal buffer waiting for the end of line ('\r\n\') characters to occur.
*/
@ -71,10 +71,7 @@ public:
InitialState,
Connected,
RequestSent,
StatusReceived,
HeadersReceived,
ContentReceived,
TrailingHeadersReceived
ContentReceived
};
RequestState State() const noexcept { return fRequestStatus; }
@ -398,7 +395,7 @@ BHttpSession::Impl::DataThreadFunc(void* arg)
auto request = std::move(data->fDataQueue.front());
data->fDataQueue.pop_front();
auto socket = request.Socket();
data->connectionMap.insert(std::make_pair(socket, std::move(request)));
// Add to objectList
@ -860,14 +857,11 @@ BHttpSession::Request::ReceiveResult()
if (bytesRead == B_WOULD_BLOCK || bytesRead == B_INTERRUPTED)
return false;
auto readEnd = bytesRead == 0;
// Parse the content in the buffer
switch (fRequestStatus) {
case InitialState:
[[fallthrough]];
case Connected:
throw BRuntimeError(__PRETTY_FUNCTION__,
"Read function called for object that is not yet connected or sent");
case RequestSent:
switch (fParser.State()) {
case HttpInputStreamState::StatusLine:
{
if (fBuffer.RemainingBytes() == static_cast<size_t>(bytesRead)) {
// In the initial run, the bytes in the buffer will match the bytes read to indicate
@ -919,18 +913,26 @@ BHttpSession::Request::ReceiveResult()
});
fResult->SetStatus(BHttpStatus{fStatus.code, std::move(fStatus.text)});
}
fRequestStatus = StatusReceived;
} else {
// We do not have enough data for the status line yet, continue receiving data.
// We do not have enough data for the status line yet
if (readEnd) {
throw BNetworkRequestError(__PRETTY_FUNCTION__,
BNetworkRequestError::ProtocolError,
"Response did not include a complete status line");
}
return false;
}
[[fallthrough]];
}
case StatusReceived:
case HttpInputStreamState::Fields:
{
if (!fParser.ParseFields(fBuffer, fFields)) {
// there may be more headers to receive.
// there may be more headers to receive, throw an error if there will be no more
if (readEnd) {
throw BNetworkRequestError(__PRETTY_FUNCTION__,
BNetworkRequestError::ProtocolError,
"Response did not include a complete header section");
}
break;
}
@ -986,7 +988,6 @@ BHttpSession::Request::ReceiveResult()
// Move headers to the result and inform listener
fResult->SetFields(std::move(fFields));
SendMessage(UrlEvent::HttpFields);
fRequestStatus = HeadersReceived;
if (!fParser.HasContent()) {
// Any requests with not content are finished
@ -999,7 +1000,7 @@ BHttpSession::Request::ReceiveResult()
}
[[fallthrough]];
}
case HeadersReceived:
case HttpInputStreamState::Body:
{
size_t bytesWrittenToBody;
// The bytesWrittenToBody may differ from the bytes parsed from the buffer when
@ -1007,7 +1008,7 @@ BHttpSession::Request::ReceiveResult()
bytesRead = fParser.ParseBody(fBuffer, [this, &bytesWrittenToBody](const std::byte* buffer, size_t size) {
bytesWrittenToBody = fResult->WriteToBody(buffer, size);
return bytesWrittenToBody;
});
}, readEnd);
SendMessage(UrlEvent::DownloadProgress, [this, bytesRead](BMessage& msg) {
msg.AddInt64(UrlEventData::NumBytes, bytesRead);
@ -1026,13 +1027,19 @@ BHttpSession::Request::ReceiveResult()
SendMessage(UrlEvent::RequestCompleted, [](BMessage& msg) {
msg.AddBool(UrlEventData::Success, true);
});
fRequestStatus = ContentReceived;
return true;
} else if (readEnd) {
// the parsing of the body is not complete but we are at the end of the data
throw BNetworkRequestError(__PRETTY_FUNCTION__,
BNetworkRequestError::ProtocolError,
"Unexpected end of data: more data was expected");
}
break;
}
default:
throw BRuntimeError(__PRETTY_FUNCTION__, "To do");
throw BRuntimeError(__PRETTY_FUNCTION__, "Not reachable");
}
// There is more to receive