Aren/haiku
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

tests/net: Working integration tests for HTTP client

Browse Source 
This patch is part 1 of 3 with the goal of having a working
integration test harness for BHttpRequest. In this patch the existing
test cases were expanded and fixed for HTTP. In followup patches the
test harness will be updated to support HTTPS and reverse proxies.

Before this patch the tests for BHttpRequest had hard dependencies on
the external services httpbin.org and portquiz.net. These tests
eventually stopped working because the owner of those services made
changes, causing the assertions in these tests to fail.

The goal of these patches is to make a test harness that allows for
the same kinds of end-to-end integration tests but without any
external dependencies.

The test suite now includes a Python script called testserver.py which
is a HTTP echo server of sorts. When it receives a request, it will
echo the request headers and request body back to the client as a
text/plain response body.

The TestServer class manages the lifecycle of this testserver.py
process. Each test case calls Start() on the server to start a new
instance, and then it is shut down when the destructor is called. On
each invocation a random port is assigned by the kernel in TestServer,
and that socket file descriptor is provided to the child testserver.py
script.

Authorization tests are supported, currently implementing Basic and
Digest auth. If the test server receives a request for a path
/auth/<auth-scheme>/<expected-username>/<expected-password>, then the
appropriate authorization scheme will be employed. For example, if
/auth/basic/foo/bar is used as the path, then the server will expect
the Authorization header to contain an appropriate Basic auth
payload.

The tests now perform a bit more validation than before, validating
the expected HTTP headers and response body is returned from the
server.

The following tests are not fixed yet or were removed:
* PortTest was removed entirely since I'm not sure of the point of this
  test, and that functionality seems to be covered by the existing tests
  anyway.
* HTTPS tests are not functional yet, but will be in a followup
  patch. THis requires updating testserver.py to generate a
  self-signed TLS cert if --use-tls is provided.
* ProxyTest was disabled before this patch, but can be enabled in a
  followup patch by providing a reverse proxy in the test harness.

Change-Id: Ia201ef4583b7636c61e77072a03db936cb0092be
Reviewed-on: https://review.haiku-os.org/c/haiku/+/2243
Reviewed-by: Adrien Destugues <pulkomandy@gmail.com>
This commit is contained in:
Kyle Ambroff-Kao 2020-02-08 01:21:16 -08:00 committed by Adrien Destugues
parent 3fded0b515
commit 0dde5052bb
6 changed files with 1084 additions and 138 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

433
src/tests/kits/net/service/HttpTest.cpp
View File

@ -1,29 +1,131 @@
/*
* Copyright 2010, Christophe Huriaux
* Copyright 2014, Haiku, inc.
* Copyright 2014-2020, Haiku, inc.
* Distributed under the terms of the MIT licence
*/
#include "HttpTest.h"
#include <algorithm>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <cstdio>
#include <fstream>
#include <map>
#include <string>
#include <NetworkKit.h>
#include <HttpRequest.h>
#include <NetworkKit.h>
#include <UrlProtocolListener.h>
#include <cppunit/TestCaller.h>
static const int kHeaderCountInTrivialRequest = 7;
// FIXME This is too strict and not very useful.
#include "TestServer.h"
HttpTest::HttpTest()
: fBaseUrl("http://httpbin.org/")
namespace {
typedef std::map<std::string, std::string> HttpHeaderMap;
class TestListener : public BUrlProtocolListener {
public:
TestListener(const std::string& expectedResponseBody,
const HttpHeaderMap& expectedResponseHeaders)
:
fExpectedResponseBody(expectedResponseBody),
fExpectedResponseHeaders(expectedResponseHeaders)
{
}
virtual void DataReceived(
BUrlRequest *caller,
const char *data,
off_t position,
ssize_t size)
{
std::copy_n(
data + position,
size,
std::back_inserter(fActualResponseBody));
}
virtual void HeadersReceived(
BUrlRequest* caller,
const BUrlResult& result)
{
const BHttpResult& http_result
= dynamic_cast<const BHttpResult&>(result);
const BHttpHeaders& headers = http_result.Headers();
for (int32 i = 0; i < headers.CountHeaders(); ++i) {
const BHttpHeader& header = headers.HeaderAt(i);
fActualResponseHeaders[std::string(header.Name())]
= std::string(header.Value());
}
}
void Verify()
{
CPPUNIT_ASSERT_EQUAL(fExpectedResponseBody, fActualResponseBody);
for (HttpHeaderMap::iterator iter = fActualResponseHeaders.begin();
iter != fActualResponseHeaders.end();
++iter)
{
CPPUNIT_ASSERT_EQUAL_MESSAGE(
"(header " + iter->first + ")",
fExpectedResponseHeaders[iter->first],
iter->second);
}
CPPUNIT_ASSERT_EQUAL(
fExpectedResponseHeaders.size(),
fActualResponseHeaders.size());
}
private:
std::string fExpectedResponseBody;
std::string fActualResponseBody;
HttpHeaderMap fExpectedResponseHeaders;
HttpHeaderMap fActualResponseHeaders;
};
void SendAuthenticatedRequest(
BUrlContext &context,
BUrl &testUrl,
const std::string& expectedResponseBody,
const HttpHeaderMap &expectedResponseHeaders)
{
TestListener listener(expectedResponseBody, expectedResponseHeaders);
BHttpRequest request(testUrl, false, "HTTP", &listener, &context);
request.SetUserName("walter");
request.SetPassword("secret");
CPPUNIT_ASSERT(request.Run());
while (request.IsRunning())
snooze(1000);
CPPUNIT_ASSERT_EQUAL(B_OK, request.Status());
const BHttpResult &result =
dynamic_cast<const BHttpResult &>(request.Result());
CPPUNIT_ASSERT_EQUAL(200, result.StatusCode());
CPPUNIT_ASSERT_EQUAL(BString("OK"), result.StatusText());
listener.Verify();
}
}
HttpTest::HttpTest(TestServerMode mode)
:
fTestServer(mode)
{
}
@ -33,41 +135,68 @@ HttpTest::~HttpTest()
}
void
HttpTest::setUp()
{
CPPUNIT_ASSERT_EQUAL_MESSAGE(
"Starting up test server",
B_OK,
fTestServer.StartIfNotRunning());
}
void
HttpTest::GetTest()
{
BUrl testUrl(fBaseUrl, "/user-agent");
BUrlContext* c = new BUrlContext();
c->AcquireReference();
BHttpRequest t(testUrl);
BUrl testUrl(fTestServer.BaseUrl(), "/");
BUrlContext* context = new BUrlContext();
context->AcquireReference();
t.SetContext(c);
std::string expectedResponseBody(
"Path: /\r\n"
"\r\n"
"Headers:\r\n"
"--------\r\n"
"Host: 127.0.0.1:PORT\r\n"
"Accept: */*\r\n"
"Accept-Encoding: gzip\r\n"
"Connection: close\r\n"
"User-Agent: Services Kit (Haiku)\r\n");
HttpHeaderMap expectedResponseHeaders;
expectedResponseHeaders["Content-Encoding"] = "gzip";
expectedResponseHeaders["Content-Length"] = "144";
expectedResponseHeaders["Content-Type"] = "text/plain";
expectedResponseHeaders["Date"] = "Sun, 09 Feb 2020 19:32:42 GMT";
expectedResponseHeaders["Server"] = "Test HTTP Server for Haiku";
TestListener listener(expectedResponseBody, expectedResponseHeaders);
CPPUNIT_ASSERT(t.Run());
while (t.IsRunning())
BHttpRequest request(testUrl, false, "HTTP", &listener, context);
CPPUNIT_ASSERT(request.Run());
while (request.IsRunning())
snooze(1000);
CPPUNIT_ASSERT_EQUAL(B_OK, t.Status());
CPPUNIT_ASSERT_EQUAL(B_OK, request.Status());
const BHttpResult& r = dynamic_cast<const BHttpResult&>(t.Result());
CPPUNIT_ASSERT_EQUAL(200, r.StatusCode());
CPPUNIT_ASSERT_EQUAL(BString("OK"), r.StatusText());
CPPUNIT_ASSERT_EQUAL(kHeaderCountInTrivialRequest,
r.Headers().CountHeaders());
CPPUNIT_ASSERT_EQUAL(42, r.Length());
// Fixed size as we know the response format.
CPPUNIT_ASSERT(!c->GetCookieJar().GetIterator().HasNext());
const BHttpResult& result
= dynamic_cast<const BHttpResult&>(request.Result());
CPPUNIT_ASSERT_EQUAL(200, result.StatusCode());
CPPUNIT_ASSERT_EQUAL(BString("OK"), result.StatusText());
CPPUNIT_ASSERT_EQUAL(144, result.Length());
listener.Verify();
CPPUNIT_ASSERT(!context->GetCookieJar().GetIterator().HasNext());
// This page should not set cookies
c->ReleaseReference();
context->ReleaseReference();
}
void
HttpTest::ProxyTest()
{
BUrl testUrl(fBaseUrl, "/user-agent");
BUrl testUrl(fTestServer.BaseUrl(), "/user-agent");
BUrlContext* c = new BUrlContext();
c->AcquireReference();
@ -87,9 +216,6 @@ HttpTest::ProxyTest()
CPPUNIT_ASSERT_EQUAL(B_OK, t.Status());
const BHttpResult& r = dynamic_cast<const BHttpResult&>(t.Result());
printf("%s\n", r.StatusText().String());
CPPUNIT_ASSERT_EQUAL(200, r.StatusCode());
CPPUNIT_ASSERT_EQUAL(BString("OK"), r.StatusText());
CPPUNIT_ASSERT_EQUAL(42, r.Length());
@ -101,119 +227,182 @@ printf("%s\n", r.StatusText().String());
}
class PortTestListener: public BUrlProtocolListener
{
public:
virtual ~PortTestListener() {};
void DataReceived(BUrlRequest*, const char* data, off_t,
ssize_t size)
{
fResult.Append(data, size);
}
BString fResult;
};
void
HttpTest::PortTest()
{
BUrl testUrl("http://portquiz.net:4242");
BHttpRequest t(testUrl);
// portquiz returns more easily parseable results when UA is Wget...
t.SetUserAgent("Wget/1.15 (haiku testsuite)");
PortTestListener listener;
t.SetListener(&listener);
CPPUNIT_ASSERT(t.Run());
while (t.IsRunning())
snooze(1000);
CPPUNIT_ASSERT_EQUAL(B_OK, t.Status());
const BHttpResult& r = dynamic_cast<const BHttpResult&>(t.Result());
CPPUNIT_ASSERT_EQUAL(200, r.StatusCode());
CPPUNIT_ASSERT(listener.fResult.StartsWith("Port 4242 test successful!"));
}
void
HttpTest::UploadTest()
{
BUrl testUrl(fBaseUrl, "/post");
BUrlContext c;
BHttpRequest t(testUrl);
// The test server will echo the POST body back to us in the HTTP response,
// so here we load it into memory so that we can compare to make sure that
// the server received it.
std::string fileContents;
{
std::ifstream inputStream("/system/data/licenses/MIT");
CPPUNIT_ASSERT(inputStream.is_open());
fileContents = std::string(
std::istreambuf_iterator<char>(inputStream),
std::istreambuf_iterator<char>());
CPPUNIT_ASSERT(!fileContents.empty());
}
t.SetContext(&c);
std::string expectedResponseBody(
"Path: /post\r\n"
"\r\n"
"Headers:\r\n"
"--------\r\n"
"Host: 127.0.0.1:PORT\r\n"
"Accept: */*\r\n"
"Accept-Encoding: gzip\r\n"
"Connection: close\r\n"
"User-Agent: Services Kit (Haiku)\r\n"
"Content-Type: multipart/form-data; boundary=<<BOUNDARY-ID>>\r\n"
"Content-Length: 1381\r\n"
"\r\n"
"Request body:\r\n"
"-------------\r\n"
"--<<BOUNDARY-ID>>\r\n"
"Content-Disposition: form-data; name=\"_uploadfile\";"
" filename=\"MIT\"\r\n"
"Content-Type: text/plain\r\n"
"\r\n"
+ fileContents
+ "\r\n"
"--<<BOUNDARY-ID>>\r\n"
"Content-Disposition: form-data; name=\"hello\"\r\n"
"\r\n"
"world\r\n"
"--<<BOUNDARY-ID>>--\r\n"
"\r\n");
HttpHeaderMap expectedResponseHeaders;
expectedResponseHeaders["Content-Encoding"] = "gzip";
expectedResponseHeaders["Content-Length"] = "900";
expectedResponseHeaders["Content-Type"] = "text/plain";
expectedResponseHeaders["Date"] = "Sun, 09 Feb 2020 19:32:42 GMT";
expectedResponseHeaders["Server"] = "Test HTTP Server for Haiku";
TestListener listener(expectedResponseBody, expectedResponseHeaders);
BHttpForm f;
f.AddString("hello", "world");
CPPUNIT_ASSERT(f.AddFile("_uploadfile", BPath("/system/data/licenses/MIT"))
== B_OK);
BUrl testUrl(fTestServer.BaseUrl(), "/post");
t.SetPostFields(f);
BUrlContext context;
BHttpRequest request(testUrl, false, "HTTP", &listener, &context);
CPPUNIT_ASSERT(t.Run());
BHttpForm form;
form.AddString("hello", "world");
CPPUNIT_ASSERT_EQUAL(
B_OK,
form.AddFile("_uploadfile", BPath("/system/data/licenses/MIT")));
while (t.IsRunning())
request.SetPostFields(form);
CPPUNIT_ASSERT(request.Run());
while (request.IsRunning())
snooze(1000);
CPPUNIT_ASSERT_EQUAL(B_OK, t.Status());
CPPUNIT_ASSERT_EQUAL(B_OK, request.Status());
const BHttpResult& r = dynamic_cast<const BHttpResult&>(t.Result());
CPPUNIT_ASSERT_EQUAL(200, r.StatusCode());
CPPUNIT_ASSERT_EQUAL(BString("OK"), r.StatusText());
CPPUNIT_ASSERT_EQUAL(466, r.Length());
// Fixed size as we know the response format.
const BHttpResult &result =
dynamic_cast<const BHttpResult &>(request.Result());
CPPUNIT_ASSERT_EQUAL(200, result.StatusCode());
CPPUNIT_ASSERT_EQUAL(BString("OK"), result.StatusText());
CPPUNIT_ASSERT_EQUAL(900, result.Length());
listener.Verify();
}
void
HttpTest::AuthBasicTest()
{
BUrl testUrl(fBaseUrl, "/basic-auth/walter/secret");
_AuthTest(testUrl);
BUrlContext context;
BUrl testUrl(fTestServer.BaseUrl(), "/auth/basic/walter/secret");
std::string expectedResponseBody(
"Path: /auth/basic/walter/secret\r\n"
"\r\n"
"Headers:\r\n"
"--------\r\n"
"Host: 127.0.0.1:PORT\r\n"
"Accept: */*\r\n"
"Accept-Encoding: gzip\r\n"
"Connection: close\r\n"
"User-Agent: Services Kit (Haiku)\r\n"
"Referer: http://127.0.0.1:PORT/auth/basic/walter/secret\r\n"
"Authorization: Basic d2FsdGVyOnNlY3JldA==\r\n");
HttpHeaderMap expectedResponseHeaders;
expectedResponseHeaders["Content-Encoding"] = "gzip";
expectedResponseHeaders["Content-Length"] = "210";
expectedResponseHeaders["Content-Type"] = "text/plain";
expectedResponseHeaders["Date"] = "Sun, 09 Feb 2020 19:32:42 GMT";
expectedResponseHeaders["Server"] = "Test HTTP Server for Haiku";
expectedResponseHeaders["Www-Authenticate"] = "Basic realm=\"Fake Realm\"";
SendAuthenticatedRequest(context, testUrl, expectedResponseBody,
expectedResponseHeaders);
CPPUNIT_ASSERT(!context.GetCookieJar().GetIterator().HasNext());
// This page should not set cookies
}
void
HttpTest::AuthDigestTest()
{
BUrl testUrl(fBaseUrl, "/digest-auth/auth/walter/secret");
_AuthTest(testUrl);
}
BUrlContext context;
BUrl testUrl(fTestServer.BaseUrl(), "/auth/digest/walter/secret");
void
HttpTest::_AuthTest(BUrl& testUrl)
{
BUrlContext c;
BHttpRequest t(testUrl);
std::string expectedResponseBody(
"Path: /auth/digest/walter/secret\r\n"
"\r\n"
"Headers:\r\n"
"--------\r\n"
"Host: 127.0.0.1:PORT\r\n"
"Accept: */*\r\n"
"Accept-Encoding: gzip\r\n"
"Connection: close\r\n"
"User-Agent: Services Kit (Haiku)\r\n"
"Referer: http://127.0.0.1:PORT/auth/digest/walter/secret\r\n"
"Authorization: Digest username=\"walter\","
" realm=\"user@shredder\","
" nonce=\"f3a95f20879dd891a5544bf96a3e5518\","
" algorithm=MD5,"
" opaque=\"f0bb55f1221a51b6d38117c331611799\","
" uri=\"/auth/digest/walter/secret\","
" qop=auth,"
" cnonce=\"60a3d95d286a732374f0f35fb6d21e79\","
" nc=00000001,"
" response=\"f4264de468aa1a91d81ac40fa73445f3\"\r\n"
"Cookie: stale_after=never; fake=fake_value\r\n");
t.SetContext(&c);
HttpHeaderMap expectedResponseHeaders;
expectedResponseHeaders["Content-Encoding"] = "gzip";
expectedResponseHeaders["Content-Length"] = "401";
expectedResponseHeaders["Content-Type"] = "text/plain";
expectedResponseHeaders["Date"] = "Sun, 09 Feb 2020 19:32:42 GMT";
expectedResponseHeaders["Server"] = "Test HTTP Server for Haiku";
expectedResponseHeaders["Set-Cookie"] = "fake=fake_value; Path=/";
expectedResponseHeaders["Www-Authenticate"]
= "Digest realm=\"user@shredder\", "
"nonce=\"f3a95f20879dd891a5544bf96a3e5518\", "
"qop=\"auth\", "
"opaque=f0bb55f1221a51b6d38117c331611799, "
"algorithm=MD5, "
"stale=FALSE";
t.SetUserName("walter");
t.SetPassword("secret");
SendAuthenticatedRequest(context, testUrl, expectedResponseBody,
expectedResponseHeaders);
CPPUNIT_ASSERT(t.Run());
while (t.IsRunning())
snooze(1000);
CPPUNIT_ASSERT_EQUAL(B_OK, t.Status());
const BHttpResult& r = dynamic_cast<const BHttpResult&>(t.Result());
CPPUNIT_ASSERT_EQUAL(200, r.StatusCode());
CPPUNIT_ASSERT_EQUAL(BString("OK"), r.StatusText());
CPPUNIT_ASSERT_EQUAL(kHeaderCountInTrivialRequest,
r.Headers().CountHeaders());
CPPUNIT_ASSERT_EQUAL(48, r.Length());
// Fixed size as we know the response format.
std::map<BString, BString> cookies;
BNetworkCookieJar::Iterator iter
= context.GetCookieJar().GetIterator();
while (iter.HasNext()) {
const BNetworkCookie* cookie = iter.Next();
cookies[cookie->Name()] = cookie->Value();
}
CPPUNIT_ASSERT_EQUAL(2, cookies.size());
CPPUNIT_ASSERT_EQUAL(BString("fake_value"), cookies["fake"]);
CPPUNIT_ASSERT_EQUAL(BString("never"), cookies["stale_after"]);
}
@ -249,10 +438,6 @@ HttpTest::AddTests(BTestSuite& parent)
// HTTP + HTTPs
_AddCommonTests<HttpTest>("HttpTest::", suite);
// HTTP-only
suite.addTest(new CppUnit::TestCaller<HttpTest>(
"HttpTest::PortTest", &HttpTest::PortTest));
// TODO: reaches out to some mysterious IP 120.203.214.182 which does
// not respond anymore?
//suite.addTest(new CppUnit::TestCaller<HttpTest>("HttpTest::ProxyTest",
@ -261,6 +446,9 @@ HttpTest::AddTests(BTestSuite& parent)
parent.addTest("HttpTest", &suite);
}
// The HTTPS tests are disabled for now until --use-tls is implemented
// in testserver.py.
#if 0
{
CppUnit::TestSuite& suite = *new CppUnit::TestSuite("HttpsTest");
@ -269,6 +457,7 @@ HttpTest::AddTests(BTestSuite& parent)
parent.addTest("HttpsTest", &suite);
}
#endif
}
@ -276,7 +465,7 @@ HttpTest::AddTests(BTestSuite& parent)
HttpsTest::HttpsTest()
: HttpTest()
:
HttpTest(TEST_SERVER_MODE_HTTPS)
{
fBaseUrl.SetProtocol("https");
}

33
src/tests/kits/net/service/HttpTest.h
View File

@ -1,5 +1,5 @@
/*
* Copyright 2014 Haiku, inc.
* Copyright 2014-2020 Haiku, inc.
* Distributed under the terms of the MIT License.
*/
#ifndef HTTP_TEST_H
@ -13,29 +13,30 @@
#include <cppunit/TestSuite.h>
#include "TestServer.h"
class HttpTest: public BTestCase {
public:
HttpTest();
virtual ~HttpTest();
HttpTest(TestServerMode mode
= TEST_SERVER_MODE_HTTP);
virtual ~HttpTest();
void GetTest();
void PortTest();
void UploadTest();
void AuthBasicTest();
void AuthDigestTest();
void ProxyTest();
virtual void setUp();
static void AddTests(BTestSuite& suite);
void GetTest();
void UploadTest();
void AuthBasicTest();
void AuthDigestTest();
void ProxyTest();
static void AddTests(BTestSuite& suite);
private:
void _AuthTest(BUrl& url);
template<class T> static void _AddCommonTests(BString prefix,
CppUnit::TestSuite& suite);
template<class T> static void _AddCommonTests(BString prefix,
CppUnit::TestSuite& suite);
protected:
BUrl fBaseUrl;
TestServer fTestServer;
};

1
src/tests/kits/net/service/Jamfile
View File

@ -9,6 +9,7 @@ UnitTestLib servicekittest.so :
DataTest.cpp
HttpTest.cpp
UrlTest.cpp
TestServer.cpp
: be $(TARGET_NETWORK_LIBS) $(HAIKU_NETAPI_LIB) [ TargetLibstdc++ ]
;

229
src/tests/kits/net/service/TestServer.cpp Normal file
View File

@ -0,0 +1,229 @@
/*
* Copyright 2020 Haiku, Inc. All rights reserved.
* Distributed under the terms of the MIT License.
*
* Authors:
* Kyle Ambroff-Kao, kyle@ambroffkao.com
*/
#include "TestServer.h"
#include <netinet/in.h>
#include <posix/libgen.h>
#include <sstream>
#include <string>
#include <sys/socket.h>
#include <sys/wait.h>
#include <unistd.h>
#include <vector>
#include <AutoDeleter.h>
#include <TestShell.h>
namespace {
template <typename T>
std::string to_string(T value)
{
std::ostringstream s;
s << value;
return s.str();
}
void exec(const std::vector<std::string>& args)
{
const char** argv = new const char*[args.size() + 1];
ArrayDeleter<const char*> _(argv);
for (size_t i = 0; i < args.size(); ++i) {
argv[i] = args[i].c_str();
}
argv[args.size()] = NULL;
execv(args[0].c_str(), const_cast<char* const*>(argv));
}
}
TestServer::TestServer(TestServerMode mode)
:
fMode(mode),
fRunning(false),
fChildPid(-1),
fSocketFd(-1),
fServerPort(0)
{
}
TestServer::~TestServer()
{
if (fChildPid != -1) {
::kill(fChildPid, SIGTERM);
pid_t result = -1;
while (result != fChildPid) {
result = ::waitpid(fChildPid, NULL, 0);
}
}
if (fSocketFd != -1) {
::close(fSocketFd);
fSocketFd = -1;
}
}
// The job of this method is to spawn a child process that will later be killed
// by the destructor. The steps are roughly:
//
// 1. If the child server process is already running, return early
// 2. Choose a random TCP port by binding to the loopback interface.
// 3. Spawn a child Python process to run testserver.py.
// 4. Return immediately allowing the tests to be performed by the caller of
// TestServer::StartIfNotRunning(). We don't have to wait for the child
// process to start up because the socket has already been created. The
// tests will block until accept() is called in the child.
status_t TestServer::StartIfNotRunning()
{
if (fRunning == true) {
return B_OK;
}
// Bind to a random unused TCP port.
{
// Create socket with port 0 to get an unused one selected by the
// kernel.
int socket_fd = ::socket(AF_INET, SOCK_STREAM, 0);
if (socket_fd == -1) {
fprintf(
stderr,
"ERROR: Unable to create socket: %s\n",
strerror(errno));
return B_ERROR;
}
fSocketFd = socket_fd;
// We may quickly reclaim the same socket between test runs, so allow
// for reuse.
{
int reuse = 1;
int result = ::setsockopt(
socket_fd,
SOL_SOCKET,
SO_REUSEPORT,
&reuse,
sizeof(reuse));
if (result == -1) {
fprintf(
stderr,
"ERROR: Unable to set socket options on fd %d: %s\n",
socket_fd,
strerror(errno));
return B_ERROR;
}
}
// Bind to loopback 127.0.0.1
struct sockaddr_in server_address;
server_address.sin_family = AF_INET;
server_address.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
int bind_result = ::bind(
socket_fd,
reinterpret_cast<struct sockaddr*>(&server_address),
sizeof(server_address));
if (bind_result == -1) {
fprintf(
stderr,
"ERROR: Unable to bind to loopback interface: %s\n",
strerror(errno));
return B_ERROR;
}
// Listen is apparently required before getsockname will work.
if (::listen(socket_fd, 32) == -1) {
fprintf(stderr, "ERROR: listen() failed: %s\n", strerror(errno));
return B_ERROR;
}
// Now get the port from the socket.
socklen_t server_address_length = sizeof(server_address);
::getsockname(
socket_fd,
reinterpret_cast<struct sockaddr*>(&server_address),
&server_address_length);
fServerPort = ntohs(server_address.sin_port);
}
fprintf(stderr, "Binding to port %d for test server\n", fServerPort);
pid_t child = ::fork();
if (child < 0)
return B_ERROR;
if (child > 0) {
// The child process has started. It may take a short amount of time
// before the child process is ready to call accept(), but that's OK.
//
// Since the socket has already been created above, the tests will not
// get ECONNREFUSED and will block until the child process calls
// accept(). So we don't have to busy loop here waiting for a
// connection to the child.
fRunning = true;
fChildPid = child;
return B_OK;
}
// This is the child process. We can exec the server process.
char* testFileSource = strdup(__FILE__);
MemoryDeleter _(testFileSource);
std::string testSrcDir(dirname(testFileSource));
std::string testServerScript = testSrcDir + "/" + "testserver.py";
std::string socket_fd_string = to_string(fSocketFd);
std::string server_port_string = to_string(fServerPort);
std::vector<std::string> child_process_args;
child_process_args.push_back("/bin/python3");
child_process_args.push_back(testServerScript);
child_process_args.push_back("--port");
child_process_args.push_back(server_port_string);
child_process_args.push_back("--fd");
child_process_args.push_back(socket_fd_string);
if (fMode == TEST_SERVER_MODE_HTTPS) {
child_process_args.push_back("--use-tls");
}
exec(child_process_args);
// If we reach this point we failed to load the Python image.
fprintf(
stderr,
"Unable to spawn %s: %s\n",
testServerScript.c_str(),
strerror(errno));
exit(1);
}
BUrl TestServer::BaseUrl() const
{
std::string scheme;
switch(fMode) {
case TEST_SERVER_MODE_HTTP:
scheme = "http://";
break;
case TEST_SERVER_MODE_HTTPS:
scheme = "https://";
break;
}
std::string baseUrl = scheme + "127.0.0.1:" + to_string(fServerPort) + "/";
return BUrl(baseUrl.c_str());
}

38
src/tests/kits/net/service/TestServer.h Normal file
View File

@ -0,0 +1,38 @@
/*
* Copyright 2020 Haiku, Inc. All rights reserved.
* Distributed under the terms of the MIT License.
*
* Authors:
* Kyle Ambroff-Kao, kyle@ambroffkao.com
*/
#ifndef TEST_SERVER_H
#define TEST_SERVER_H
#include <os/support/SupportDefs.h>
#include <os/support/Url.h>
enum TestServerMode {
TEST_SERVER_MODE_HTTP,
TEST_SERVER_MODE_HTTPS,
};
class TestServer {
public:
TestServer(TestServerMode mode);
~TestServer();
status_t StartIfNotRunning();
BUrl BaseUrl() const;
private:
TestServerMode fMode;
bool fRunning;
pid_t fChildPid;
int fSocketFd;
uint16_t fServerPort;
};
#endif // TEST_SERVER_H

488
src/tests/kits/net/service/testserver.py Normal file
View File

@ -0,0 +1,488 @@
#
# Copyright 2020 Haiku, Inc. All rights reserved.
# Distributed under the terms of the MIT License.
#
# Authors:
# Kyle Ambroff-Kao, kyle@ambroffkao.com
#
"""
HTTP(S) server used for integration testing of ServicesKit.
This service receives HTTP requests and just echos them back in the response.
This is intentionally not using any fancy frameworks or libraries so as to not
require any dependencies, and also to allow for adding endpoints to replicate
behavior of other servers in the future.
"""
import abc
import base64
import gzip
import hashlib
import http.server
import io
import optparse
import os
import re
import socket
import sys
import zlib
MULTIPART_FORM_BOUNDARY_RE = re.compile(
r'^multipart/form-data; boundary=(----------------------------\d+)$')
AUTH_PATH_RE = re.compile(
r'^/auth/(?P<strategy>(basic|digest))'
'/(?P<username>[a-z0-9]+)/(?P<password>[a-z0-9]+)',
re.IGNORECASE)
class RequestHandler(http.server.BaseHTTPRequestHandler):
"""
Any GET or POST request just gets echoed back to the sender. If the path
ends with a numeric component like "/404" or "/500", then that value will
be set as the status code in the response.
Note that this isn't meant to replicate expected functionality exactly.
Rather than implementing all of these status codes as expected per RFC,
such as having an empty response body for 201 response, only the
functionality that is required to handle requests from HttpTests is
implemented.
There can also be endpoints here that are intentionally non-compliant in
order to exercize the HTTP client's behavior when a server is badly
behaved.
"""
def do_GET(self, write_response=True):
authorized, extra_headers = self._authorize()
if not authorized:
return
encoding, response_body = self._build_response_body()
self.send_response(
extract_desired_status_code_from_path(self.path, 200))
self.send_header('Content-Type', 'text/plain')
self.send_header('Content-Length', str(len(response_body)))
if encoding:
self.send_header('Content-Encoding', encoding)
for header_name, header_value in extra_headers:
self.send_header(header_name, header_value)
self.end_headers()
if write_response:
self.wfile.write(response_body)
def do_HEAD(self):
self.do_GET(False)
def do_POST(self):
authorized, extra_headers = self._authorize()
if not authorized:
return
encoding, response_body = self._build_response_body()
self.send_response(
extract_desired_status_code_from_path(self.path, 200))
self.send_header('Content-Type', 'text/plain')
self.send_header('Content-Length', str(len(response_body)))
if encoding:
self.send_header('Content-Encoding', encoding)
for header_name, header_value in extra_headers:
self.send_header(header_name, header_value)
self.end_headers()
self.wfile.write(response_body)
def do_DELETE(self):
self._not_supported()
def do_PATCH(self):
self._not_supported()
def do_OPTIONS(self):
self._not_supported()
def send_response(self, code, message=None):
self.log_request(code)
self.send_response_only(code, message)
self.send_header('Server', 'Test HTTP Server for Haiku')
self.send_header('Date', 'Sun, 09 Feb 2020 19:32:42 GMT')
def _build_response_body(self):
# The post-body may be multi-part/form-data, in which case the client
# will have generated some random identifier to identify the boundary.
# If that's the case, we'll replace it here in order to allow the test
# client to validate the response data without needing to predict the
# boundary identifier. This makes the response body deterministic even
# though the boundary will change with every request, and lets the
# tests in HttpTests hard-code the entire expected response body for
# validation.
boundary_id_value = None
supported_encodings = [
e.strip()
for e in self.headers.get('Accept-Encoding', '').split(',')
if e.strip()]
if 'gzip' in supported_encodings:
encoding = 'gzip'
output_stream = GzipResponseBodyBuilder()
elif 'deflate' in supported_encodings:
encoding = 'deflate'
output_stream = DeflateResponseBodyBuilder()
else:
encoding = None
output_stream = RawResponseBodyBuilder()
output_stream.write(
'Path: {}\r\n\r\n'.format(self.path).encode('utf-8'))
output_stream.write(b'Headers:\r\n')
output_stream.write(b'--------\r\n')
for header in self.headers:
for header_value in self.headers.get_all(header):
if header == 'Host' or header == 'Referer':
# The server port can change between runs which will change
# the size and contents of the response body. To make tests
# that verify the contents of the response body easier the
# server port will be stripped from these headers when
# echoed to the response body.
header_value = re.sub(r':[0-9]+', ':PORT', header_value)
if header == 'Content-Type':
match = MULTIPART_FORM_BOUNDARY_RE.match(
self.headers.get('Content-Type', 'text/plain'))
if match is not None:
boundary_id_value = match.group(1)
header_value = header_value.replace(
boundary_id_value,
'<<BOUNDARY-ID>>')
output_stream.write(
'{}: {}\r\n'.format(header, header_value).encode('utf-8'))
content_length = int(self.headers.get('Content-Length', 0))
if content_length > 0:
output_stream.write(b'\r\n')
output_stream.write(b'Request body:\r\n')
output_stream.write(b'-------------\r\n')
body_bytes = self.rfile.read(content_length).decode('utf-8')
if boundary_id_value:
body_bytes = body_bytes.replace(
boundary_id_value, '<<BOUNDARY-ID>>')
output_stream.write(body_bytes.encode('utf-8'))
output_stream.write(b'\r\n')
return encoding, output_stream.get_bytes()
def _not_supported(self):
self.send_response(405, '{} not supported'.format(self.command))
self.end_headers()
self.wfile.write(
'{} not supported\r\n'.format(self.command).encode('utf-8'))
def _authorize(self):
"""
Authorizes the request. If True is returned that means that the
request was not authorized and the 4xx response has been send to the
client.
"""
# We only authorize paths like
# /auth/<strategy>/<expected-username>/<expected-password>
match = AUTH_PATH_RE.match(self.path)
if match is None:
return True, []
strategy = match.group('strategy')
expected_username = match.group('username')
expected_password = match.group('password')
if strategy == 'basic':
return self._handle_basic_auth(
expected_username,
expected_password)
elif strategy == 'digest':
return self._handle_digest_auth(
expected_username,
expected_password)
else:
raise NotImplementedError(
'Unimplemented authorization strategy ' + strategy)
def _handle_basic_auth(self, expected_username, expected_password):
authorization = self.headers.get('Authorization', None)
auth_type = None
encoded_credentials = None
username = None
password = None
if authorization:
auth_type, encoded_credentials = authorization.split()
if encoded_credentials is not None:
decoded = base64.decodebytes(encoded_credentials.encode('utf-8'))
username, password = decoded.decode('utf-8').split(':')
if authorization is None or auth_type != 'Basic' \
or encoded_credentials is None \
or username != expected_username \
or password != expected_password:
self.send_response(401, 'Not authorized')
self.send_header('Www-Authenticate', 'Basic realm="Fake Realm"')
self.end_headers()
return False, []
return True, [('Www-Authenticate', 'Basic realm="Fake Realm"')]
def _handle_digest_auth(self, expected_username, expected_password):
"""
Implement enough of the digest auth RFC to make tests pass.
"""
# Note: These values will always be the same because we want the
# response to be deterministic for testing purposes.
NONCE = 'f3a95f20879dd891a5544bf96a3e5518'
OPAQUE = 'f0bb55f1221a51b6d38117c331611799'
extra_headers = []
authorization = self.headers.get('Authorization', None)
credentials = None
auth_type = None
if authorization is not None:
auth_type, fields = authorization.split(maxsplit=1)
if auth_type == 'Digest':
credentials = parse_kv_pair_header(fields)
expected_response_hash = None
if credentials:
expected_response_hash = compute_digest_challenge_response_hash(
self.command,
self.path,
'',
credentials,
expected_password)
if authorization is None or credentials is None \
or auth_type != 'Digest' \
or expected_response_hash != credentials.get('response'):
self.send_response(401, 'Not authorized')
self.send_header(
'Www-Authenticate',
'Digest realm="user@shredder",'
' nonce="{}",'
' qop="auth",'
' opaque={},'
' algorithm=MD5,'
' stale=FALSE'.format(NONCE, OPAQUE))
self.send_header('Set-Cookie', 'stale_after=never; Path=/')
self.send_header('Set-Cookie', 'fake=fake_value; Path=/')
self.end_headers()
return False, extra_headers
return True, extra_headers
class ResponseBodyBuilder(object):
__meta__ = abc.ABCMeta
@abc.abstractmethod
def write(self, bytes):
raise NotImplementedError()
@abc.abstractmethod
def get_bytes(self):
raise NotImplementedError()
class RawResponseBodyBuilder(ResponseBodyBuilder):
def __init__(self):
self.buf = io.BytesIO()
def write(self, bytes):
self.buf.write(bytes)
def get_bytes(self):
return self.buf.getvalue()
class GzipResponseBodyBuilder(ResponseBodyBuilder):
def __init__(self):
self.buf = io.BytesIO()
self.compressor = gzip.GzipFile(
mode='wb',
compresslevel=4,
fileobj=self.buf)
def write(self, bytes):
self.compressor.write(bytes)
def get_bytes(self):
self.compressor.close()
return self.buf.getvalue()
class DeflateResponseBodyBuilder(ResponseBodyBuilder):
def __init__(self):
self.raw = RawResponseBodyBuilder()
def write(self, bytes):
self.raw.write(bytes)
def get_bytes(self):
return zlib.compress(self.raw.get_bytes())
def extract_desired_status_code_from_path(path, default=200):
status_code = default
path_parts = os.path.split(path)
try:
status_code = int(path_parts[-1])
except ValueError:
pass
return status_code
def compute_digest_challenge_response_hash(
request_method,
request_uri,
request_body,
credentials,
expected_password):
"""
Compute hash as defined by RFC2069, although this isn't an attempt to be
perfect, just enough for basic integration tests in HttpTests to work.
:param credentials: Map of values parsed from the Authorization header
from the client.
:param expected_password: The known correct password of the user
attempting to authenticate.
:return: None if a hash cannot be produced, otherwise the hash as defined
by RFC2069.
"""
algorithm = credentials.get('algorithm')
if algorithm == 'MD5':
hashfunc = hashlib.md5
elif algorithm == 'SHA-256':
hashfunc = hashlib.sha256
elif algorithm == 'SHA-512':
hashfunc = hashlib.sha512
else:
return None
realm = credentials.get('realm')
username = credentials.get('username')
ha1 = hashfunc(':'.join([
username,
realm,
expected_password]).encode('utf-8')).hexdigest()
qop = credentials.get('qop')
if qop is None or qop == 'auth':
ha2 = hashfunc(':'.join([
request_method,
request_uri]).encode('utf-8')).hexdigest()
elif qop == 'auth-int':
ha2 = hashfunc(':'.join([
request_method,
request_uri,
request_body]).encode('utf-8')).hexdigest()
else:
ha2 = None
if ha1 is None or ha2 is None:
return None
if qop is None:
return hashfunc(':'.join([
ha1,
credentials.get('nonce', ''),
ha2]).encode('utf-8')).hexdigest()
elif qop == 'auth' or qop == 'auth-int':
hash_components = [
ha1,
credentials.get('nonce', ''),
credentials.get('nc', ''),
credentials.get('cnonce', ''),
qop,
ha2]
return hashfunc(':'.join(hash_components).encode('utf-8')).hexdigest()
def parse_kv_pair_header(header_value, sep=','):
d = {}
for kvpair in header_value.split(sep):
key, value = kvpair.strip().split('=')
d[key.strip()] = value.strip().strip('"')
return d
def main():
options = parse_args(sys.argv)
if options.use_tls:
# TODO: Generate a self-signed TLS cert to test HTTPS.
raise NotImplementedError()
bind_addr = (options.bind_addr, options.port)
if options.server_socket_fd:
server = http.server.HTTPServer(
bind_addr,
RequestHandler,
bind_and_activate=False)
server.socket = socket.fromfd(
options.server_socket_fd,
socket.AF_INET,
socket.SOCK_STREAM)
server.server_port = server.socket.getsockname()[1]
else:
# A socket hasn't been open for us already, so we'll just use
# a random port here.
server = http.server.HTTPServer(bind_addr, RequestHandler)
try:
print(
'Test server listening on port',
server.server_port,
file=sys.stderr)
server.serve_forever(0.01)
except KeyboardInterrupt:
server.server_close()
def parse_args(argv):
parser = optparse.OptionParser(
usage='Usage: %prog [OPTIONS]',
description=__doc__)
parser.add_option(
'--bind-addr',
default='127.0.0.1',
dest='bind_addr',
help='By default only bind to loopback')
parser.add_option(
'--use-tls',
dest='use_tls',
default=False,
action='store_true',
help='If set, a self-signed TLS certificate, key and CA will be'
' generated for testing purposes.')
parser.add_option(
'--port',
dest='port',
default=0,
type='int',
help='If not specified a random port will be used.')
parser.add_option(
"--fd",
dest='server_socket_fd',
default=None,
type='int',
help='A socket FD to use for accept() instead of binding a new one.')
options, args = parser.parse_args(argv)
if len(args) > 1:
parser.error('Unexpected arguments: {}'.format(', '.join(args[1:])))
return options
if __name__ == '__main__':
main()