HaikuDepot : More Backend Communications Improvements

Added missing files from prior commit.
This commit is contained in:
Andrew Lindesay 2017-12-19 21:48:27 +01:00
parent 197c659920
commit 0df6decf1b
9 changed files with 909 additions and 0 deletions

View File

@ -0,0 +1,84 @@
/*
* Copyright 2017, Andrew Lindesay <apl@lindesay.co.nz>.
* All rights reserved. Distributed under the terms of the MIT License.
*/
#include "AbstractSingleFileServerProcess.h"
#include "Logger.h"
#include "ServerSettings.h"
#include "StorageUtils.h"
AbstractSingleFileServerProcess::AbstractSingleFileServerProcess(
AbstractServerProcessListener* listener, uint32 options)
:
AbstractServerProcess(listener, options)
{
}
AbstractSingleFileServerProcess::~AbstractSingleFileServerProcess()
{
}
status_t
AbstractSingleFileServerProcess::RunInternal()
{
if (Logger::IsInfoEnabled())
printf("[%s] will fetch data\n", Name());
BPath localPath = LocalPath();
BString urlPathComponent = UrlPathComponent();
status_t result = B_OK;
if (IsSuccess(result) && HasOption(SERVER_PROCESS_DROP_CACHE))
result = DeleteLocalFile(localPath);
bool hasData;
off_t size;
if (IsSuccess(result))
result = StorageUtils::ExistsObject(localPath, &hasData, NULL, &size);
hasData = hasData && size > 0;
if (IsSuccess(result) && ShouldAttemptNetworkDownload(hasData)) {
result = DownloadToLocalFileAtomically(
localPath,
ServerSettings::CreateFullUrl(urlPathComponent));
}
if (IsSuccess(result) || result == APP_ERR_NOT_MODIFIED) {
status_t hasDataResult = StorageUtils::ExistsObject(
localPath, &hasData, NULL, &size);
hasData = hasData && size > 0;
if (hasDataResult == B_OK && !hasData)
result = APP_ERR_NO_DATA;
}
if (IsSuccess(result) || result == APP_ERR_NOT_MODIFIED) {
if (Logger::IsInfoEnabled())
printf("[%s] did fetch data\n", Name());
// now load the data in and process it.
printf("[%s] will process data\n", Name());
result = ProcessLocalData();
switch (result) {
case B_OK:
printf("[%s] did process data\n", Name());
break;
default:
MoveDamagedFileAside(localPath);
break;
}
}
return result;
}

View File

@ -0,0 +1,30 @@
/*
* Copyright 2017, Andrew Lindesay <apl@lindesay.co.nz>.
* All rights reserved. Distributed under the terms of the MIT License.
*/
#ifndef ABSTRACT_SINGLE_FILE_SERVER_PROCESS_H
#define ABSTRACT_SINGLE_FILE_SERVER_PROCESS_H
#include "AbstractServerProcess.h"
class AbstractSingleFileServerProcess : public AbstractServerProcess {
public:
AbstractSingleFileServerProcess(
AbstractServerProcessListener* listener,
uint32 options);
virtual ~AbstractSingleFileServerProcess();
protected:
virtual status_t RunInternal();
virtual status_t ProcessLocalData() = 0;
virtual BString UrlPathComponent() = 0;
virtual BPath& LocalPath() = 0;
};
#endif // ABSTRACT_SINGLE_FILE_SERVER_PROCESS_H

View File

@ -0,0 +1,131 @@
/*
* Copyright 2017, Andrew Lindesay <apl@lindesay.co.nz>.
* All rights reserved. Distributed under the terms of the MIT License.
*/
#include "BulkLoadContext.h"
BulkLoadContext::BulkLoadContext()
:
fState(BULK_LOAD_INITIAL),
fIconProcess(NULL),
fRepositoryProcess(NULL),
fPkgProcesses(new List<AbstractServerProcess*, true>()),
fProcessOptions(0)
{
}
BulkLoadContext::~BulkLoadContext()
{
StopAllProcesses();
if (fIconProcess != NULL)
delete fIconProcess;
if (fRepositoryProcess != NULL)
delete fRepositoryProcess;
int32 count = fPkgProcesses->CountItems();
int32 i;
for (i = 0; i < count; i++)
delete fPkgProcesses->ItemAt(i);
delete fPkgProcesses;
}
bulk_load_state
BulkLoadContext::State()
{
return fState;
}
void
BulkLoadContext::SetState(bulk_load_state value)
{
fState = value;
}
void
BulkLoadContext::StopAllProcesses()
{
if (fIconProcess != NULL)
fIconProcess->Stop();
if (fRepositoryProcess != NULL)
fRepositoryProcess->Stop();
int32 count = fPkgProcesses->CountItems();
int32 i;
for (i = 0; i < count; i++)
fPkgProcesses->ItemAt(i)->Stop();
}
AbstractServerProcess*
BulkLoadContext::IconProcess()
{
return fIconProcess;
}
void
BulkLoadContext::SetIconProcess(AbstractServerProcess* value)
{
fIconProcess = value;
}
AbstractServerProcess*
BulkLoadContext::RepositoryProcess()
{
return fRepositoryProcess;
}
void
BulkLoadContext::SetRepositoryProcess(
AbstractServerProcess* value)
{
fRepositoryProcess = value;
}
int32
BulkLoadContext::CountPkgProcesses()
{
return fPkgProcesses->CountItems();
}
AbstractServerProcess*
BulkLoadContext::PkgProcessAt(int32 index)
{
return fPkgProcesses->ItemAt(index);
}
void
BulkLoadContext::AddPkgProcess(AbstractServerProcess *value)
{
fPkgProcesses->Add(value);
}
void
BulkLoadContext::AddProcessOption(uint32 flag)
{
fProcessOptions = fProcessOptions | flag;
}
uint32
BulkLoadContext::ProcessOptions()
{
return fProcessOptions;
}

View File

@ -0,0 +1,66 @@
/*
* Copyright 2017, Andrew Lindesay <apl@lindesay.co.nz>.
* All rights reserved. Distributed under the terms of the MIT License.
*/
#ifndef BULK_LOAD_CONTEXT_H
#define BULK_LOAD_CONTEXT_H
#include <String.h>
#include <File.h>
#include <Path.h>
#include "AbstractServerProcess.h"
#include "List.h"
typedef enum bulk_load_state {
BULK_LOAD_INITIAL = 1,
BULK_LOAD_REPOSITORY_AND_REFERENCE = 2,
BULK_LOAD_PKGS_AND_ICONS = 3,
BULK_LOAD_COMPLETE = 4
} bulk_load_state;
class BulkLoadContext {
public:
BulkLoadContext();
virtual ~BulkLoadContext();
void StopAllProcesses();
bulk_load_state State();
void SetState(bulk_load_state value);
AbstractServerProcess*
IconProcess();
void SetIconProcess(AbstractServerProcess* value);
AbstractServerProcess*
RepositoryProcess();
void SetRepositoryProcess(
AbstractServerProcess* value);
int32 CountPkgProcesses();
AbstractServerProcess*
PkgProcessAt(int32 index);
void AddPkgProcess(AbstractServerProcess *value);
void AddProcessOption(uint32 flag);
uint32 ProcessOptions();
private:
bulk_load_state
fState;
AbstractServerProcess*
fIconProcess;
AbstractServerProcess*
fRepositoryProcess;
List<AbstractServerProcess*, true>*
fPkgProcesses;
uint32 fProcessOptions;
};
#endif // BULK_LOAD_CONTEXT_H

View File

@ -0,0 +1,369 @@
/*
* Copyright 2017, Andrew Lindesay <apl@lindesay.co.nz>.
* All rights reserved. Distributed under the terms of the MIT License.
*/
#include "BulkLoadStateMachine.h"
#include <Autolock.h>
#include <NetworkInterface.h>
#include <NetworkRoster.h>
#include "Logger.h"
#include "PkgDataUpdateProcess.h"
#include "RepositoryDataUpdateProcess.h"
#include "ServerIconExportUpdateProcess.h"
#include "ServerSettings.h"
BulkLoadStateMachine::BulkLoadStateMachine(Model* model)
:
fBulkLoadContext(NULL),
fModel(model)
{
}
BulkLoadStateMachine::~BulkLoadStateMachine()
{
Stop();
}
bool
BulkLoadStateMachine::IsRunning()
{
BAutolock locker(&fLock);
return fBulkLoadContext != NULL;
}
/*! This gets invoked when one of the background processes has exited. */
void
BulkLoadStateMachine::ServerProcessExited()
{
ContextPoll();
}
static char* bulk_load_state_name(bulk_load_state state) {
switch(state) {
case BULK_LOAD_INITIAL:
return "BULK_LOAD_INITIAL";
case BULK_LOAD_REPOSITORY_AND_REFERENCE:
return "BULK_LOAD_REPOSITORY_AND_REFERENCE";
case BULK_LOAD_PKGS_AND_ICONS:
return "BULK_LOAD_PKGS_AND_ICONS";
case BULK_LOAD_COMPLETE:
return "BULK_LOAD_COMPLETE";
default:
return "???";
}
}
void
BulkLoadStateMachine::SetContextState(bulk_load_state state)
{
if (Logger::IsDebugEnabled()) {
printf("bulk load - transition to state [%s]\n",
bulk_load_state_name(state));
}
fBulkLoadContext->SetState(state);
}
/*! Bulk loading data into the model can be considered to be a state
machine. This method is invoked each time that an event state
change happens.
*/
void
BulkLoadStateMachine::ContextPoll()
{
BAutolock locker(&fLock);
if (Logger::IsDebugEnabled())
printf("bulk load - context poll\n");
if (CanTransitionTo(BULK_LOAD_REPOSITORY_AND_REFERENCE)) {
SetContextState(BULK_LOAD_REPOSITORY_AND_REFERENCE);
InitiateBulkPopulateIcons();
if (InitiateBulkRepositories() != B_OK)
ContextPoll();
return;
}
if (CanTransitionTo(BULK_LOAD_PKGS_AND_ICONS)) {
fModel->LogDepotsWithNoWebAppRepositoryCode();
SetContextState(BULK_LOAD_PKGS_AND_ICONS);
InitiateBulkPopulatePackagesForAllDepots();
return;
}
if (CanTransitionTo(BULK_LOAD_COMPLETE)) {
SetContextState(BULK_LOAD_COMPLETE);
delete fBulkLoadContext;
fBulkLoadContext = NULL;
return;
}
}
bool
BulkLoadStateMachine::CanTransitionTo(bulk_load_state targetState)
{
if (fBulkLoadContext != NULL) {
bulk_load_state existingState = fBulkLoadContext->State();
switch (targetState) {
case BULK_LOAD_INITIAL:
return false;
case BULK_LOAD_REPOSITORY_AND_REFERENCE:
return existingState == BULK_LOAD_INITIAL;
case BULK_LOAD_PKGS_AND_ICONS:
return (existingState == BULK_LOAD_REPOSITORY_AND_REFERENCE)
&& ((fBulkLoadContext->RepositoryProcess() == NULL)
|| !fBulkLoadContext->RepositoryProcess()->IsRunning());
case BULK_LOAD_COMPLETE:
if ((existingState == BULK_LOAD_PKGS_AND_ICONS)
&& ((fBulkLoadContext->IconProcess() == NULL)
|| !fBulkLoadContext->IconProcess()->IsRunning())) {
int32 i;
for (i = 0; i < fBulkLoadContext->CountPkgProcesses(); i++) {
AbstractServerProcess* process =
fBulkLoadContext->PkgProcessAt(i);
if (process->IsRunning())
return false;
}
return true;
}
break;
}
}
return false;
}
void
BulkLoadStateMachine::StopAllProcesses()
{
BAutolock locker(&fLock);
if (fBulkLoadContext != NULL) {
if (NULL != fBulkLoadContext->IconProcess())
fBulkLoadContext->IconProcess()->Stop();
if (NULL != fBulkLoadContext->RepositoryProcess())
fBulkLoadContext->RepositoryProcess()->Stop();
int32 i;
for(i = 0; i < fBulkLoadContext->CountPkgProcesses(); i++) {
AbstractServerProcess* serverProcess =
fBulkLoadContext->PkgProcessAt(i);
serverProcess->Stop();
}
}
}
void
BulkLoadStateMachine::Start()
{
if (Logger::IsInfoEnabled())
printf("bulk load - start\n");
Stop();
{
BAutolock locker(&fLock);
if (!IsRunning()) {
fBulkLoadContext = new BulkLoadContext();
if (ServerSettings::ForceNoNetwork() || !HasNetwork())
fBulkLoadContext->AddProcessOption(
SERVER_PROCESS_NO_NETWORKING);
if (ServerSettings::PreferCache())
fBulkLoadContext->AddProcessOption(SERVER_PROCESS_PREFER_CACHE);
if (ServerSettings::DropCache())
fBulkLoadContext->AddProcessOption(SERVER_PROCESS_DROP_CACHE);
ContextPoll();
}
}
}
void
BulkLoadStateMachine::Stop()
{
StopAllProcesses();
// spin lock to wait for the bulk-load processes to complete.
while (IsRunning())
snooze(500000);
}
/*! This method is the initial function that is invoked on starting a new
thread. It will start a server process that is part of the bulk-load.
*/
status_t
BulkLoadStateMachine::StartProcess(void* cookie)
{
AbstractServerProcess* process =
static_cast<AbstractServerProcess*>(cookie);
if (Logger::IsInfoEnabled()) {
printf("bulk load - starting process [%s]\n",
process->Name());
}
process->Run();
return B_OK;
}
status_t
BulkLoadStateMachine::InitiateServerProcess(AbstractServerProcess* process)
{
if (Logger::IsInfoEnabled())
printf("bulk load - initiating [%s]\n", process->Name());
thread_id tid = spawn_thread(&StartProcess,
process->Name(), B_NORMAL_PRIORITY, process);
if (tid >= 0) {
resume_thread(tid);
return B_OK;
}
return B_ERROR;
}
status_t
BulkLoadStateMachine::InitiateBulkRepositories()
{
status_t result = B_OK;
BPath dataPath;
fBulkLoadContext->SetRepositoryProcess(NULL);
result = fModel->DumpExportRepositoryDataPath(dataPath);
if (result != B_OK) {
BAutolock locker(&fLock);
printf("unable to obtain the path for storing the repository data\n");
ContextPoll();
return B_ERROR;
}
fBulkLoadContext->SetRepositoryProcess(
new RepositoryDataUpdateProcess(this, dataPath, fModel,
fBulkLoadContext->ProcessOptions()));
return InitiateServerProcess(fBulkLoadContext->RepositoryProcess());
}
status_t
BulkLoadStateMachine::InitiateBulkPopulateIcons()
{
BPath path;
if (fModel->IconStoragePath(path) != B_OK) {
BAutolock locker(&fLock);
printf("unable to obtain the path for storing icons\n");
ContextPoll();
return B_ERROR;
}
AbstractServerProcess *process = new ServerIconExportUpdateProcess(
this, path, fModel, fBulkLoadContext->ProcessOptions());
fBulkLoadContext->SetIconProcess(process);
return InitiateServerProcess(process);
}
status_t
BulkLoadStateMachine::InitiateBulkPopulatePackagesForDepot(
const DepotInfo& depotInfo)
{
BString repositorySourceCode = depotInfo.WebAppRepositorySourceCode();
if (repositorySourceCode.Length() == 0) {
printf("the depot [%s] has no repository source code\n",
depotInfo.Name().String());
return B_ERROR;
}
BPath repositorySourcePkgDataPath;
if (fModel->DumpExportPkgDataPath(repositorySourcePkgDataPath,
repositorySourceCode) != B_OK) {
BAutolock locker(&fLock);
printf("unable to obtain the path for storing data for [%s]\n",
repositorySourceCode.String());
ContextPoll();
return B_ERROR;
}
AbstractServerProcess *process = new PkgDataUpdateProcess(
this, repositorySourcePkgDataPath, fModel->PreferredLanguage(),
repositorySourceCode, depotInfo.Name(), fModel,
fBulkLoadContext->ProcessOptions());
fBulkLoadContext->AddPkgProcess(process);
return InitiateServerProcess(process);
}
// static
void
BulkLoadStateMachine::InitiatePopulatePackagesForDepotCallback(
const DepotInfo& depotInfo, void* context)
{
BulkLoadStateMachine* stateMachine =
static_cast<BulkLoadStateMachine*>(context);
stateMachine->InitiateBulkPopulatePackagesForDepot(depotInfo);
}
void
BulkLoadStateMachine::InitiateBulkPopulatePackagesForAllDepots()
{
fModel->ForAllDepots(&InitiatePopulatePackagesForDepotCallback, this);
printf("did initiate populate package data for %" B_PRId32 " depots\n",
fBulkLoadContext->CountPkgProcesses());
if (0 == fBulkLoadContext->CountPkgProcesses())
ContextPoll();
}
bool
BulkLoadStateMachine::HasNetwork()
{
BNetworkRoster& roster = BNetworkRoster::Default();
BNetworkInterface interface;
uint32 cookie = 0;
while (roster.GetNextInterface(&cookie, interface) == B_OK) {
uint32 flags = interface.Flags();
if ((flags & IFF_LOOPBACK) == 0
&& (flags & (IFF_UP | IFF_LINK)) == (IFF_UP | IFF_LINK)) {
return true;
}
}
return false;
}

View File

@ -0,0 +1,60 @@
/*
* Copyright 2017, Andrew Lindesay <apl@lindesay.co.nz>.
* All rights reserved. Distributed under the terms of the MIT License.
*/
#ifndef BULK_LOAD_STATE_MACHINE_H
#define BULK_LOAD_STATE_MACHINE_H
#include <String.h>
#include <File.h>
#include <Locker.h>
#include <Path.h>
#include "AbstractServerProcess.h"
#include "Model.h"
class BulkLoadStateMachine : public AbstractServerProcessListener {
public:
BulkLoadStateMachine(Model* model);
virtual ~BulkLoadStateMachine();
bool IsRunning();
void Start();
void Stop();
void ServerProcessExited();
private:
static status_t StartProcess(void* cookie);
void ContextPoll();
void SetContextState(bulk_load_state state);
void StopAllProcesses();
bool HasNetwork();
bool CanTransitionTo(
bulk_load_state targetState);
static void InitiatePopulatePackagesForDepotCallback(
const DepotInfo& depotInfo,
void* context);
status_t InitiateServerProcess(
AbstractServerProcess* process);
status_t InitiateBulkRepositories();
status_t InitiateBulkPopulateIcons();
status_t InitiateBulkPopulatePackagesForDepot(
const DepotInfo& depotInfo);
void InitiateBulkPopulatePackagesForAllDepots();
private:
BLocker fLock;
BulkLoadContext* fBulkLoadContext;
Model* fModel;
};
#endif // BULK_LOAD_STATE_MACHINE_H

View File

@ -0,0 +1,13 @@
/*
* Copyright 2017, Andrew Lindesay <apl@lindesay.co.nz>.
* All rights reserved. Distributed under the terms of the MIT License.
*/
#ifndef STOPPABLE_H
#define STOPPABLE_H
class Stoppable {
public:
virtual bool WasStopped() = 0;
};
#endif // STOPPABLE_H

View File

@ -0,0 +1,131 @@
/*
* Copyright 2017, Andrew Lindesay <apl@lindesay.co.nz>.
* All rights reserved. Distributed under the terms of the MIT License.
*/
#include "ListTest.h"
#include <stdio.h>
#include <AutoDeleter.h>
#include <Json.h>
#include <cppunit/TestCaller.h>
#include <cppunit/TestSuite.h>
#include "List.h"
ListTest::ListTest()
{
}
ListTest::~ListTest()
{
}
static int32 CompareStrings(const BString& a, const BString& b)
{
return a.Compare(b);
}
static int32 CompareWithContextString(const void* context, const BString& str)
{
const char* contextString = static_cast<char*>(context);
return -1 * str.Compare(contextString);
}
void
ListTest::TestBinarySearch()
{
List<BString, false> list;
BString tmp;
for(char c = 'a'; c <= 'z'; c++) {
tmp.SetToFormat("%c", c);
list.AddOrdered(tmp, &CompareStrings);
}
// ----------------------
int32 aIndex = list.BinarySearch("a", &CompareWithContextString);
int32 hIndex = list.BinarySearch("h", &CompareWithContextString);
int32 uIndex = list.BinarySearch("u", &CompareWithContextString);
int32 zIndex = list.BinarySearch("z", &CompareWithContextString);
int32 ampersandIndex = list.BinarySearch("&", &CompareWithContextString);
// ----------------------
CPPUNIT_ASSERT_EQUAL(0, aIndex);
CPPUNIT_ASSERT_EQUAL(7, hIndex);
CPPUNIT_ASSERT_EQUAL(20, uIndex);
CPPUNIT_ASSERT_EQUAL(25, zIndex);
CPPUNIT_ASSERT_EQUAL(-1, ampersandIndex);
}
void
ListTest::TestAddOrdered()
{
List<BString, false> list;
// ----------------------
list.AddOrdered(BString("p"), &CompareStrings); //1
list.AddOrdered(BString("o"), &CompareStrings);
list.AddOrdered(BString("n"), &CompareStrings);
list.AddOrdered(BString("s"), &CompareStrings);
list.AddOrdered(BString("b"), &CompareStrings); //5
list.AddOrdered(BString("y"), &CompareStrings);
list.AddOrdered(BString("r"), &CompareStrings);
list.AddOrdered(BString("d"), &CompareStrings);
list.AddOrdered(BString("i"), &CompareStrings);
list.AddOrdered(BString("k"), &CompareStrings); //10
list.AddOrdered(BString("t"), &CompareStrings);
list.AddOrdered(BString("e"), &CompareStrings);
list.AddOrdered(BString("a"), &CompareStrings);
list.AddOrdered(BString("u"), &CompareStrings);
list.AddOrdered(BString("z"), &CompareStrings); // 15
list.AddOrdered(BString("q"), &CompareStrings);
// ----------------------
CPPUNIT_ASSERT_EQUAL_MESSAGE("expected count of package infos",
16, list.CountItems());
CPPUNIT_ASSERT_EQUAL(BString("a"), list.ItemAt(0));
CPPUNIT_ASSERT_EQUAL(BString("b"), list.ItemAt(1));
CPPUNIT_ASSERT_EQUAL(BString("d"), list.ItemAt(2));
CPPUNIT_ASSERT_EQUAL(BString("e"), list.ItemAt(3));
CPPUNIT_ASSERT_EQUAL(BString("i"), list.ItemAt(4));
CPPUNIT_ASSERT_EQUAL(BString("k"), list.ItemAt(5));
CPPUNIT_ASSERT_EQUAL(BString("n"), list.ItemAt(6));
CPPUNIT_ASSERT_EQUAL(BString("o"), list.ItemAt(7));
CPPUNIT_ASSERT_EQUAL(BString("p"), list.ItemAt(8));
CPPUNIT_ASSERT_EQUAL(BString("q"), list.ItemAt(9));
CPPUNIT_ASSERT_EQUAL(BString("r"), list.ItemAt(10));
CPPUNIT_ASSERT_EQUAL(BString("s"), list.ItemAt(11));
CPPUNIT_ASSERT_EQUAL(BString("t"), list.ItemAt(12));
CPPUNIT_ASSERT_EQUAL(BString("u"), list.ItemAt(13));
CPPUNIT_ASSERT_EQUAL(BString("y"), list.ItemAt(14));
CPPUNIT_ASSERT_EQUAL(BString("z"), list.ItemAt(15));
}
/*static*/ void
ListTest::AddTests(BTestSuite& parent)
{
CppUnit::TestSuite& suite = *new CppUnit::TestSuite(
"ListTest");
suite.addTest(
new CppUnit::TestCaller<ListTest>(
"ListTest::TestAddOrdered",
&ListTest::TestAddOrdered));
suite.addTest(
new CppUnit::TestCaller<ListTest>(
"ListTest::TestBinarySearch",
&ListTest::TestBinarySearch));
parent.addTest("ListTest", &suite);
}

View File

@ -0,0 +1,25 @@
/*
* Copyright 2017, Andrew Lindesay <apl@lindesay.co.nz>
* Distributed under the terms of the MIT License.
*/
#ifndef LIST_TEST_H
#define LIST_TEST_H
#include <TestCase.h>
#include <TestSuite.h>
class ListTest : public CppUnit::TestCase {
public:
ListTest();
virtual ~ListTest();
void TestAddOrdered();
void TestBinarySearch();
static void AddTests(BTestSuite& suite);
};
#endif // LIST_TEST_H