Aren/haiku
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HaikuDepot : More Backend Communications Improvements

Browse Source 
* Further improves the logging and provides some
  basic performance numbers.
* Moves the bulk-load logic out of the data-model
  class.
* Introduces a state-machine for the bulk-load
  process so that it will be more easily able to be
  shifted to non-blocking IO when the HTTP libraries
  can do that.
* Implements concurrent loading of the bulk-data to
  hopefully improve lead time for icons and meta-data.
* Loads data to a temporary file and then moves to the
  final location in order to avoid partially written
  data in the cache.
* Handles situations where no network is available;
  prevents attempt to access the network.
* Allows bulk-load processes to be cancelled when the
  application quits.
* Introduces command-line arguments to help simulate
  scenarios to help with testing performance and
  network absence.
* Implements ordered insert and binary search in the
  'List' class + basic unit test.
This commit is contained in:
Andrew Lindesay 2017-12-19 20:56:53 +01:00
parent 11b65332b4
commit 3094fef308
47 changed files with 1549 additions and 953 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/apps/haikudepot/Jamfile
View File

@ -69,6 +69,8 @@ Application HaikuDepot :
MarkupTextView.cpp MarkupTextView.cpp
MessagePackageListener.cpp MessagePackageListener.cpp
Model.cpp Model.cpp
BulkLoadContext.cpp
BulkLoadStateMachine.cpp
PackageAction.cpp PackageAction.cpp
PackageActionHandler.cpp PackageActionHandler.cpp
PackageContentsView.cpp PackageContentsView.cpp
@ -98,6 +100,7 @@ Application HaikuDepot :
# network + server # network + server
AbstractServerProcess.cpp AbstractServerProcess.cpp
AbstractSingleFileServerProcess.cpp
ServerSettings.cpp ServerSettings.cpp
WebAppInterface.cpp WebAppInterface.cpp
PkgDataUpdateProcess.cpp PkgDataUpdateProcess.cpp

92
src/apps/haikudepot/List.h
View File

@ -13,6 +13,9 @@
#include <SupportDefs.h> #include <SupportDefs.h>
#define BINARY_SEARCH_LINEAR_THRESHOLD 4
template <typename ItemType, bool PlainOldData, uint32 BlockSize = 8> template <typename ItemType, bool PlainOldData, uint32 BlockSize = 8>
class List { class List {
typedef List<ItemType, PlainOldData, BlockSize> SelfType; typedef List<ItemType, PlainOldData, BlockSize> SelfType;
@ -110,6 +113,30 @@ public:
return fCount; return fCount;
} }
/*! Note that the use of this method will depend on the list being ordered.
*/
inline int32 BinarySearch(const void* context,
int32 (*compareFunc)(const void* context, const ItemType& item))
{
if (fCount == 0)
return -1;
return _BinarySearchBounded(context, compareFunc, 0, fCount - 1);
}
inline bool AddOrdered(const ItemType& copyFrom,
int32 (*compareFunc)(const ItemType& one, const ItemType& two))
{
// special case
if (fCount == 0
|| compareFunc(copyFrom, ItemAtFast(fCount - 1)) > 0) {
return Add(copyFrom);
}
return _AddOrderedBounded(copyFrom, compareFunc, 0, fCount - 1);
}
inline bool Add(const ItemType& copyFrom) inline bool Add(const ItemType& copyFrom)
{ {
if (_Resize(fCount + 1)) { if (_Resize(fCount + 1)) {
@ -229,6 +256,71 @@ public:
} }
private: private:
inline int32 _BinarySearchLinearBounded(
const void* context,
int32 (*compareFunc)(const void* context, const ItemType& item),
int32 start, int32 end)
{
for(int32 i = start; i <= end; i++) {
if (compareFunc(context, ItemAtFast(i)) == 0)
return i;
}
return -1;
}
inline int32 _BinarySearchBounded(
const void* context,
int32 (*compareFunc)(const void* context, const ItemType& item),
int32 start, int32 end)
{
if (end - start < BINARY_SEARCH_LINEAR_THRESHOLD)
return _BinarySearchLinearBounded(context, compareFunc, start, end);
int32 mid = start + (end - start);
if (compareFunc(context, ItemAtFast(mid)) >= 0)
return _BinarySearchBounded(context, compareFunc, mid, end);
return _BinarySearchBounded(context, compareFunc, start, mid - 1);
}
inline bool _AddOrderedLinearBounded(
const ItemType& copyFrom,
int32 (*compareFunc)(const ItemType& one, const ItemType& two),
int32 start, int32 end)
{
for(int32 i = start; i <= (end + 1); i++) {
bool greaterBefore = (i == start)
|| (compareFunc(copyFrom, ItemAtFast(i - 1)) > 0);
if (greaterBefore) {
bool lessAfter = (i == end + 1)
|| (compareFunc(copyFrom, ItemAtFast(i)) <= 0);
if (lessAfter)
return Add(copyFrom, i);
}
}
printf("illegal state; unable to insert item into list\n");
exit(EXIT_FAILURE);
}
inline bool _AddOrderedBounded(
const ItemType& copyFrom,
int32 (*compareFunc)(const ItemType& one, const ItemType& two),
int32 start, int32 end)
{
if(end - start < BINARY_SEARCH_LINEAR_THRESHOLD)
return _AddOrderedLinearBounded(copyFrom, compareFunc, start, end);
int32 mid = start + (end - start);
if (compareFunc(copyFrom, ItemAtFast(mid)) >= 0)
return _AddOrderedBounded(copyFrom, compareFunc, mid, end);
return _AddOrderedBounded(copyFrom, compareFunc, start, mid - 1);
}
inline bool _Resize(uint32 count) inline bool _Resize(uint32 count)
{ {
if (count > fAllocatedCount) { if (count > fAllocatedCount) {

47
src/apps/haikudepot/model/LocalIconStore.cpp
View File

@ -15,10 +15,9 @@
#include "StorageUtils.h" #include "StorageUtils.h"
LocalIconStore::LocalIconStore() LocalIconStore::LocalIconStore(const BPath& path)
{ {
if (_EnsureIconStoragePath(fIconStoragePath) != B_OK) fIconStoragePath = path;
fprintf(stdout, "unable to setup icon storage\n");
} }
@ -42,7 +41,6 @@ LocalIconStore::_HasIconStoragePath() const
status_t status_t
LocalIconStore::TryFindIconPath(const BString& pkgName, BPath& path) const LocalIconStore::TryFindIconPath(const BString& pkgName, BPath& path) const
{ {
if (_HasIconStoragePath()) {
BPath bestIconPath; BPath bestIconPath;
BPath iconPkgPath(fIconStoragePath); BPath iconPkgPath(fIconStoragePath);
bool exists; bool exists;
@ -50,7 +48,7 @@ LocalIconStore::TryFindIconPath(const BString& pkgName, BPath& path) const
if ( (iconPkgPath.Append("hicn") == B_OK) if ( (iconPkgPath.Append("hicn") == B_OK)
&& (iconPkgPath.Append(pkgName) == B_OK) && (iconPkgPath.Append(pkgName) == B_OK)
&& (StorageUtils::ExistsDirectory(iconPkgPath, &exists, &isDir) && (StorageUtils::ExistsObject(iconPkgPath, &exists, &isDir, NULL)
== B_OK) == B_OK)
&& exists && exists
&& isDir && isDir
@ -60,47 +58,12 @@ LocalIconStore::TryFindIconPath(const BString& pkgName, BPath& path) const
path = bestIconPath; path = bestIconPath;
return B_OK; return B_OK;
} }
}
path.Unset(); path.Unset();
return B_FILE_NOT_FOUND; return B_FILE_NOT_FOUND;
} }
void
LocalIconStore::UpdateFromServerIfNecessary() const
{
if (_HasIconStoragePath()) {
BPath iconStoragePath(fIconStoragePath);
ServerIconExportUpdateProcess service(iconStoragePath);
service.Run();
if (Logger::IsDebugEnabled()) {
printf("did update the icons from server\n");
}
}
}
status_t
LocalIconStore::_EnsureIconStoragePath(BPath& path) const
{
BPath iconStoragePath;
if (find_directory(B_USER_CACHE_DIRECTORY, &iconStoragePath) == B_OK
&& iconStoragePath.Append("HaikuDepot") == B_OK
&& iconStoragePath.Append("__allicons") == B_OK
&& create_directory(iconStoragePath.Path(), 0777) == B_OK) {
path.SetTo(iconStoragePath.Path());
return B_OK;
}
path.Unset();
fprintf(stdout, "unable to find the user cache directory for icons");
return B_ERROR;
}
status_t status_t
LocalIconStore::_IdentifyBestIconFileAtDirectory(const BPath& directory, LocalIconStore::_IdentifyBestIconFileAtDirectory(const BPath& directory,
BPath& bestIconPath) const BPath& bestIconPath) const
@ -121,8 +84,8 @@ LocalIconStore::_IdentifyBestIconFileAtDirectory(const BPath& directory,
bool isDir; bool isDir;
if ( (workingPath.Append(iconLeafname) == B_OK if ( (workingPath.Append(iconLeafname) == B_OK
&& StorageUtils::ExistsDirectory( && StorageUtils::ExistsObject(
workingPath, &exists, &isDir) == B_OK) workingPath, &exists, &isDir, NULL) == B_OK)
&& exists && exists
&& !isDir) { && !isDir) {
bestIconPath.SetTo(workingPath.Path()); bestIconPath.SetTo(workingPath.Path());

4
src/apps/haikudepot/model/LocalIconStore.h
View File

@ -16,15 +16,13 @@
class LocalIconStore { class LocalIconStore {
public: public:
LocalIconStore(); LocalIconStore(const BPath& path);
virtual ~LocalIconStore(); virtual ~LocalIconStore();
status_t TryFindIconPath(const BString& pkgName, status_t TryFindIconPath(const BString& pkgName,
BPath& path) const; BPath& path) const;
void UpdateFromServerIfNecessary() const;
private: private:
bool _HasIconStoragePath() const; bool _HasIconStoragePath() const;
status_t _EnsureIconStoragePath(BPath& path) const;
status_t _IdentifyBestIconFileAtDirectory( status_t _IdentifyBestIconFileAtDirectory(
const BPath& directory, const BPath& directory,
BPath& bestIconPath) const; BPath& bestIconPath) const;

2
src/apps/haikudepot/model/Logger.h
View File

@ -2,8 +2,6 @@
* Copyright 2017, Andrew Lindesay <apl@lindesay.co.nz>. * Copyright 2017, Andrew Lindesay <apl@lindesay.co.nz>.
* All rights reserved. Distributed under the terms of the MIT License. * All rights reserved. Distributed under the terms of the MIT License.
*/ */
#ifndef LOGGER_H #ifndef LOGGER_H
#define LOGGER_H #define LOGGER_H

329
src/apps/haikudepot/model/Model.cpp
View File

@ -23,8 +23,6 @@
#include <Path.h> #include <Path.h>
#include "Logger.h" #include "Logger.h"
#include "PkgDataUpdateProcess.h"
#include "RepositoryDataUpdateProcess.h"
#include "StorageUtils.h" #include "StorageUtils.h"
@ -359,10 +357,7 @@ Model::Model()
fShowAvailablePackages(true), fShowAvailablePackages(true),
fShowInstalledPackages(true), fShowInstalledPackages(true),
fShowSourcePackages(false), fShowSourcePackages(false),
fShowDevelopPackages(false), fShowDevelopPackages(false)
fPopulateAllPackagesThread(-1),
fStopPopulatingAllPackages(false)
{ {
_UpdateIsFeaturedFilter(); _UpdateIsFeaturedFilter();
@ -417,7 +412,6 @@ Model::Model()
Model::~Model() Model::~Model()
{ {
StopPopulatingAllPackages();
} }
@ -701,7 +695,6 @@ Model::PopulatePackage(const PackageInfoRef& package, uint32 flags)
BMessage item; BMessage item;
if (items.FindMessage(name, &item) != B_OK) if (items.FindMessage(name, &item) != B_OK)
break; break;
// item.PrintToStream();
BString user; BString user;
BMessage userInfo; BMessage userInfo;
@ -768,32 +761,6 @@ Model::PopulatePackage(const PackageInfoRef& package, uint32 flags)
} }
void
Model::PopulateAllPackages()
{
StopPopulatingAllPackages();
fStopPopulatingAllPackages = false;
fPopulateAllPackagesThread = spawn_thread(&_PopulateAllPackagesEntry,
"Package populator", B_NORMAL_PRIORITY, this);
if (fPopulateAllPackagesThread >= 0)
resume_thread(fPopulateAllPackagesThread);
}
void
Model::StopPopulatingAllPackages()
{
if (fPopulateAllPackagesThread < 0)
return;
fStopPopulatingAllPackages = true;
wait_for_thread(fPopulateAllPackagesThread, NULL);
fPopulateAllPackagesThread = -1;
}
void void
Model::SetUsername(BString username) Model::SetUsername(BString username)
{ {
@ -837,8 +804,6 @@ Model::SetAuthorization(const BString& username, const BString& password,
} }
// #pragma mark - private
/*! When bulk repository data comes down from the server, it will /*! When bulk repository data comes down from the server, it will
arrive as a json.gz payload. This is stored locally as a cache arrive as a json.gz payload. This is stored locally as a cache
and this method will provide the on-disk storage location for and this method will provide the on-disk storage location for
@ -846,7 +811,7 @@ Model::SetAuthorization(const BString& username, const BString& password,
*/ */
status_t status_t
Model::_DumpExportRepositoryDataPath(BPath& path) const Model::DumpExportRepositoryDataPath(BPath& path) const
{ {
BPath repoDataPath; BPath repoDataPath;
@ -866,7 +831,26 @@ Model::_DumpExportRepositoryDataPath(BPath& path) const
status_t status_t
Model::_DumpExportPkgDataPath(BPath& path, Model::IconStoragePath(BPath& path) const
{
BPath iconStoragePath;
if (find_directory(B_USER_CACHE_DIRECTORY, &iconStoragePath) == B_OK
&& iconStoragePath.Append("HaikuDepot") == B_OK
&& iconStoragePath.Append("__allicons") == B_OK
&& create_directory(iconStoragePath.Path(), 0777) == B_OK) {
path.SetTo(iconStoragePath.Path());
return B_OK;
}
path.Unset();
fprintf(stdout, "unable to find the user cache directory for icons");
return B_ERROR;
}
status_t
Model::DumpExportPkgDataPath(BPath& path,
const BString& repositorySourceCode) const const BString& repositorySourceCode) const
{ {
BPath repoDataPath; BPath repoDataPath;
@ -889,24 +873,6 @@ Model::_DumpExportPkgDataPath(BPath& path,
} }
status_t
Model::PopulateWebAppRepositoryCodes()
{
status_t result = B_OK;
BPath dataPath;
result = _DumpExportRepositoryDataPath(dataPath);
if (result != B_OK)
return result;
RepositoryDataUpdateProcess process(dataPath, &fDepots);
result = process.Run();
return result;
}
void void
Model::_UpdateIsFeaturedFilter() Model::_UpdateIsFeaturedFilter()
{ {
@ -917,141 +883,6 @@ Model::_UpdateIsFeaturedFilter()
} }
int32
Model::_PopulateAllPackagesEntry(void* cookie)
{
Model* model = static_cast<Model*>(cookie);
model->_PopulateAllPackagesThread();
model->_PopulateAllPackagesIcons();
return 0;
}
void
Model::_PopulateAllPackagesIcons()
{
fLocalIconStore.UpdateFromServerIfNecessary();
int32 depotIndex = 0;
int32 packageIndex = 0;
int32 countIconsSet = 0;
fprintf(stdout, "will populate all packages' icons\n");
while (true) {
PackageInfoRef package;
BAutolock locker(&fLock);
if (depotIndex > fDepots.CountItems()) {
fprintf(stdout, "did populate %" B_PRId32 " packages' icons\n",
countIconsSet);
return;
}
const DepotInfo& depot = fDepots.ItemAt(depotIndex);
const PackageList& packages = depot.Packages();
if (packageIndex >= packages.CountItems()) {
// Need the next depot
packageIndex = 0;
depotIndex++;
} else {
package = packages.ItemAt(packageIndex);
if (Logger::IsDebugEnabled()) {
fprintf(stdout, "will populate package icon for [%s]\n",
package->Name().String());
}
if (_PopulatePackageIcon(package) == B_OK)
countIconsSet++;
packageIndex++;
}
}
}
void
Model::_PopulatePackagesForDepot(const DepotInfo& depotInfo)
{
BString repositorySourceCode = depotInfo.WebAppRepositorySourceCode();
BPath repositorySourcePkgDataPath;
if (B_OK != _DumpExportPkgDataPath(repositorySourcePkgDataPath,
repositorySourceCode)) {
printf("unable to obtain the path for storing data for [%s]\n",
repositorySourceCode.String());
} else {
printf("will fetch and process data for repository source [%s]\n",
repositorySourceCode.String());
PkgDataUpdateProcess process(
repositorySourcePkgDataPath,
fLock, repositorySourceCode, fPreferredLanguage,
depotInfo.Packages(),
fCategories);
process.Run();
printf("did fetch and process data for repository source [%s]\n",
repositorySourceCode.String());
}
}
void
Model::_PopulateAllPackagesThread()
{
int32 depotIndex = 0;
int32 count = 0;
for (depotIndex = 0;
depotIndex < fDepots.CountItems() && !fStopPopulatingAllPackages;
depotIndex++) {
const DepotInfo& depot = fDepots.ItemAt(depotIndex);
if (depot.WebAppRepositorySourceCode().Length() > 0) {
_PopulatePackagesForDepot(depot);
count++;
}
}
printf("did populate package data for %" B_PRIi32 " depots\n", count);
}
status_t
Model::_PopulatePackageIcon(const PackageInfoRef& package)
{
BPath bestIconPath;
if ( fLocalIconStore.TryFindIconPath(
package->Name(), bestIconPath) == B_OK) {
BFile bestIconFile(bestIconPath.Path(), O_RDONLY);
BitmapRef bitmapRef(new(std::nothrow)SharedBitmap(bestIconFile), true);
BAutolock locker(&fLock);
package->SetIcon(bitmapRef);
if (Logger::IsDebugEnabled()) {
fprintf(stdout, "have set the package icon for [%s] from [%s]\n",
package->Name().String(), bestIconPath.Path());
}
return B_OK;
}
if (Logger::IsDebugEnabled()) {
fprintf(stdout, "did not set the package icon for [%s]; no data\n",
package->Name().String());
}
return B_FILE_NOT_FOUND;
}
void void
Model::_PopulatePackageScreenshot(const PackageInfoRef& package, Model::_PopulatePackageScreenshot(const PackageInfoRef& package,
const ScreenshotInfo& info, int32 scaledWidth, bool fromCacheOnly) const ScreenshotInfo& info, int32 scaledWidth, bool fromCacheOnly)
@ -1128,3 +959,119 @@ Model::_NotifyAuthorizationChanged()
listener->AuthorizationChanged(); listener->AuthorizationChanged();
} }
} }
// temporary - should not be required once the repo info url is used.
static void
normalize_repository_base_url(BUrl& url)
{
if (url.Protocol() == "https")
url.SetProtocol("http");
BString path(url.Path());
if (path.EndsWith("/"))
url.SetPath(path.Truncate(path.Length() - 1));
}
void
Model::ForAllDepots(void (*func)(const DepotInfo& depot, void* context),
void* context)
{
for (int32 i = 0; i < fDepots.CountItems(); i++) {
DepotInfo depotInfo = fDepots.ItemAtFast(i);
func(depotInfo, context);
}
}
// TODO; should use the repo.info url and not the base url.
void
Model::ReplaceDepotByUrl(const BString& url,
DepotMapper* depotMapper, void* context)
{
for (int32 i = 0; i < fDepots.CountItems(); i++) {
DepotInfo depotInfo = fDepots.ItemAtFast(i);
BUrl url(url);
BUrl depotUrlNormalized(depotInfo.BaseURL());
normalize_repository_base_url(url);
normalize_repository_base_url(depotUrlNormalized);
if (url == depotUrlNormalized) {
BAutolock locker(&fLock);
fDepots.Replace(i, depotMapper->MapDepot(depotInfo, context));
}
}
}
void
Model::ForAllPackages(PackageConsumer* packageConsumer, void* context)
{
for (int32 i = 0; i < fDepots.CountItems(); i++) {
DepotInfo depotInfo = fDepots.ItemAtFast(i);
PackageList packages = depotInfo.Packages();
for(int32 j = 0; j < packages.CountItems(); j++) {
const PackageInfoRef& packageInfoRef = packages.ItemAtFast(j);
if (packageInfoRef != NULL) {
BAutolock locker(&fLock);
if (!packageConsumer->ConsumePackage(packageInfoRef, context))
return;
}
}
}
}
void
Model::ForPackageByNameInDepot(const BString& depotName,
const BString& packageName, PackageConsumer* packageConsumer, void* context)
{
int32 depotCount = fDepots.CountItems();
for (int32 i = 0; i < depotCount; i++) {
DepotInfo depotInfo = fDepots.ItemAtFast(i);
if (depotInfo.Name() == depotName) {
int32 packageIndex = depotInfo.PackageIndexByName(packageName);
if (-1 != packageIndex) {
PackageList packages = depotInfo.Packages();
const PackageInfoRef& packageInfoRef =
packages.ItemAtFast(packageIndex);
BAutolock locker(&fLock);
packageConsumer->ConsumePackage(packageInfoRef,
context);
}
return;
}
}
}
void
Model::LogDepotsWithNoWebAppRepositoryCode() const
{
int32 i;
for (i = 0; i < fDepots.CountItems(); i++) {
const DepotInfo& depot = fDepots.ItemAt(i);
if (depot.WebAppRepositoryCode().Length() == 0) {
printf("depot [%s]", depot.Name().String());
if (depot.BaseURL().Length() > 0)
printf(" (%s)", depot.BaseURL().String());
printf(" correlates with no repository in the haiku"
"depot server system\n");
}
}
}

72
src/apps/haikudepot/model/Model.h
View File

@ -9,7 +9,9 @@
#include <FindDirectory.h> #include <FindDirectory.h>
#include <Locker.h> #include <Locker.h>
#include "AbstractServerProcess.h"
#include "LocalIconStore.h" #include "LocalIconStore.h"
#include "BulkLoadContext.h"
#include "PackageInfo.h" #include "PackageInfo.h"
#include "WebAppInterface.h" #include "WebAppInterface.h"
@ -37,6 +39,22 @@ public:
virtual void AuthorizationChanged() = 0; virtual void AuthorizationChanged() = 0;
}; };
class DepotMapper {
public:
virtual DepotInfo MapDepot(const DepotInfo& depot,
void* context) = 0;
};
class PackageConsumer {
public:
virtual bool ConsumePackage(
const PackageInfoRef& packageInfoRef,
void* context) = 0;
};
typedef BReference<ModelListener> ModelListenerRef; typedef BReference<ModelListener> ModelListenerRef;
typedef List<ModelListenerRef, false> ModelListenerList; typedef List<ModelListenerRef, false> ModelListenerList;
@ -118,8 +136,6 @@ public:
bool ShowDevelopPackages() const bool ShowDevelopPackages() const
{ return fShowDevelopPackages; } { return fShowDevelopPackages; }
status_t PopulateWebAppRepositoryCodes();
// Retrieve package information // Retrieve package information
static const uint32 POPULATE_CACHED_RATING = 1 << 0; static const uint32 POPULATE_CACHED_RATING = 1 << 0;
static const uint32 POPULATE_CACHED_ICON = 1 << 1; static const uint32 POPULATE_CACHED_ICON = 1 << 1;
@ -131,8 +147,6 @@ public:
void PopulatePackage(const PackageInfoRef& package, void PopulatePackage(const PackageInfoRef& package,
uint32 flags); uint32 flags);
void PopulateAllPackages();
void StopPopulatingAllPackages();
const StringList& SupportedLanguages() const const StringList& SupportedLanguages() const
{ return fSupportedLanguages; } { return fSupportedLanguages; }
@ -149,20 +163,41 @@ public:
const WebAppInterface& GetWebAppInterface() const const WebAppInterface& GetWebAppInterface() const
{ return fWebAppInterface; } { return fWebAppInterface; }
void ReplaceDepotByUrl(
const BString& url,
DepotMapper* depotMapper,
void* context);
void ForAllDepots(
void (*func)(const DepotInfo& depot,
void* context),
void* context);
void ForAllPackages(PackageConsumer* packageConsumer,
void* context);
void ForPackageByNameInDepot(
const BString& depotName,
const BString& packageName,
PackageConsumer* packageConsumer,
void* context);
status_t IconStoragePath(BPath& path) const;
status_t DumpExportRepositoryDataPath(BPath& path) const;
status_t DumpExportPkgDataPath(BPath& path,
const BString& repositorySourceCode) const;
void LogDepotsWithNoWebAppRepositoryCode() const;
private: private:
status_t _DumpExportRepositoryDataPath(
BPath& path) const;
status_t _DumpExportPkgDataPath(BPath& path,
const BString& repositorySourceCode) const;
void _UpdateIsFeaturedFilter(); void _UpdateIsFeaturedFilter();
static int32 _PopulateAllPackagesEntry(void* cookie); static int32 _PopulateAllPackagesEntry(void* cookie);
void _PopulateAllPackagesThread();
void _PopulatePackagesForDepot(
const DepotInfo& depotInfo);
void _PopulateAllPackagesIcons(); void _PopulatePackageScreenshot(
const PackageInfoRef& package,
const ScreenshotInfo& info,
int32 scaledWidth, bool fromCacheOnly);
bool _GetCacheFile(BPath& path, BFile& file, bool _GetCacheFile(BPath& path, BFile& file,
directory_which directory, directory_which directory,
@ -175,14 +210,6 @@ private:
const char* fileName, const char* fileName,
bool ignoreAge, time_t maxAge) const; bool ignoreAge, time_t maxAge) const;
status_t _PopulatePackageIcon(
const PackageInfoRef& package);
void _PopulatePackageScreenshot(
const PackageInfoRef& package,
const ScreenshotInfo& info,
int32 scaledWidth,
bool fromCacheOnly);
void _NotifyAuthorizationChanged(); void _NotifyAuthorizationChanged();
private: private:
@ -190,8 +217,6 @@ private:
DepotList fDepots; DepotList fDepots;
LocalIconStore fLocalIconStore;
CategoryRef fCategoryAudio; CategoryRef fCategoryAudio;
CategoryRef fCategoryBusiness; CategoryRef fCategoryBusiness;
CategoryRef fCategoryDevelopment; CategoryRef fCategoryDevelopment;
@ -225,9 +250,6 @@ private:
bool fShowSourcePackages; bool fShowSourcePackages;
bool fShowDevelopPackages; bool fShowDevelopPackages;
thread_id fPopulateAllPackagesThread;
volatile bool fStopPopulatingAllPackages;
StringList fSupportedLanguages; StringList fSupportedLanguages;
BString fPreferredLanguage; BString fPreferredLanguage;

29
src/apps/haikudepot/model/PackageInfo.cpp
View File

@ -1,7 +1,7 @@
/* /*
* Copyright 2013-2014, Stephan Aßmus <superstippi@gmx.de>. * Copyright 2013-2014, Stephan Aßmus <superstippi@gmx.de>.
* Copyright 2013, Rene Gollent <rene@gollent.com>. * Copyright 2013, Rene Gollent <rene@gollent.com>.
* Copyright 2016, Andrew Lindesay <apl@lindesay.co.nz>. * Copyright 2016-2017, Andrew Lindesay <apl@lindesay.co.nz>.
* All rights reserved. Distributed under the terms of the MIT License. * All rights reserved. Distributed under the terms of the MIT License.
*/ */
@ -1047,10 +1047,35 @@ DepotInfo::operator!=(const DepotInfo& other) const
} }
static int32 PackageCompare(const PackageInfoRef& p1, const PackageInfoRef& p2)
{
return p1->Name().Compare(p2->Name());
}
/*! This method will insert the package into the list of packages
in order so that the list of packages remains in order.
*/
bool bool
DepotInfo::AddPackage(const PackageInfoRef& package) DepotInfo::AddPackage(const PackageInfoRef& package)
{ {
return fPackages.Add(package); return fPackages.AddOrdered(package, &PackageCompare);
}
static int32 PackageFixedNameCompare(const void* context,
const PackageInfoRef& package)
{
const BString* packageName = static_cast<BString*>(context);
return packageName->Compare(package->Name());
}
int32
DepotInfo::PackageIndexByName(const BString& packageName)
{
return fPackages.BinarySearch(&packageName, &PackageFixedNameCompare);
} }

4
src/apps/haikudepot/model/PackageInfo.h
View File

@ -1,6 +1,6 @@
/* /*
* Copyright 2013-2014, Stephan Aßmus <superstippi@gmx.de>. * Copyright 2013-2014, Stephan Aßmus <superstippi@gmx.de>.
* Copyright 2016, Andrew Lindesay <apl@lindesay.co.nz>. * Copyright 2016-2017, Andrew Lindesay <apl@lindesay.co.nz>.
* All rights reserved. Distributed under the terms of the MIT License. * All rights reserved. Distributed under the terms of the MIT License.
*/ */
#ifndef PACKAGE_INFO_H #ifndef PACKAGE_INFO_H
@ -411,6 +411,8 @@ public:
bool AddPackage(const PackageInfoRef& package); bool AddPackage(const PackageInfoRef& package);
int32 PackageIndexByName(const BString& packageName);
void SyncPackages(const PackageList& packages); void SyncPackages(const PackageList& packages);
void SetBaseURL(const BString& baseURL); void SetBaseURL(const BString& baseURL);

272
src/apps/haikudepot/server/AbstractServerProcess.cpp
View File

@ -4,13 +4,15 @@
*/ */
#include "AbstractServerProcess.h" #include "AbstractServerProcess.h"
#include <unistd.h>
#include <errno.h> #include <errno.h>
#include <string.h> #include <string.h>
#include <AutoDeleter.h> #include <AutoDeleter.h>
#include <Autolock.h>
#include <FileIO.h> #include <FileIO.h>
#include <HttpRequest.h>
#include <HttpTime.h> #include <HttpTime.h>
#include <Locker.h>
#include <UrlProtocolRoster.h> #include <UrlProtocolRoster.h>
#include <support/ZlibCompressionAlgorithm.h> #include <support/ZlibCompressionAlgorithm.h>
@ -18,6 +20,7 @@
#include "Logger.h" #include "Logger.h"
#include "ServerSettings.h" #include "ServerSettings.h"
#include "StandardMetaDataJsonEventListener.h" #include "StandardMetaDataJsonEventListener.h"
#include "StorageUtils.h"
#include "ToFileUrlProtocolListener.h" #include "ToFileUrlProtocolListener.h"
@ -31,6 +34,140 @@
#define TIMEOUT_MICROSECONDS 3e+7 #define TIMEOUT_MICROSECONDS 3e+7
AbstractServerProcess::AbstractServerProcess(
AbstractServerProcessListener* listener, uint32 options)
:
fLock(),
fListener(listener),
fWasStopped(false),
fProcessState(SERVER_PROCESS_INITIAL),
fErrorStatus(B_OK),
fOptions(options),
fRequest(NULL)
{
}
AbstractServerProcess::~AbstractServerProcess()
{
}
bool
AbstractServerProcess::HasOption(uint32 flag)
{
return (fOptions & flag) == flag;
}
bool
AbstractServerProcess::ShouldAttemptNetworkDownload(bool hasDataAlready)
{
return
!HasOption(SERVER_PROCESS_NO_NETWORKING)
&& !(HasOption(SERVER_PROCESS_PREFER_CACHE) && hasDataAlready);
}
status_t
AbstractServerProcess::Run()
{
{
BAutolock locker(&fLock);
if (ProcessState() != SERVER_PROCESS_INITIAL) {
printf("cannot start server process as it is not idle");
return B_NOT_ALLOWED;
}
fProcessState = SERVER_PROCESS_RUNNING;
}
SetErrorStatus(RunInternal());
SetProcessState(SERVER_PROCESS_COMPLETE);
// this process may be part of a larger bulk-load process and
// if so, the process orchestration needs to know when this
// process has completed.
if (fListener != NULL)
fListener->ServerProcessExited();
return ErrorStatus();
}
bool
AbstractServerProcess::WasStopped()
{
BAutolock locker(&fLock);
return fWasStopped;
}
status_t
AbstractServerProcess::ErrorStatus()
{
BAutolock locker(&fLock);
return fErrorStatus;
}
status_t
AbstractServerProcess::Stop()
{
BAutolock locker(&fLock);
fWasStopped = true;
return StopInternal();
}
status_t
AbstractServerProcess::StopInternal()
{
if (fRequest != NULL) {
return fRequest->Stop();
}
return B_NOT_ALLOWED;
}
bool
AbstractServerProcess::IsRunning()
{
return ProcessState() == SERVER_PROCESS_RUNNING;
}
void
AbstractServerProcess::SetErrorStatus(status_t value)
{
BAutolock locker(&fLock);
if (fErrorStatus == B_OK) {
fErrorStatus = value;
}
}
void
AbstractServerProcess::SetProcessState(process_state value)
{
BAutolock locker(&fLock);
fProcessState = value;
}
process_state
AbstractServerProcess::ProcessState()
{
BAutolock locker(&fLock);
return fProcessState;
}
status_t status_t
AbstractServerProcess::IfModifiedSinceHeaderValue(BString& headerValue) const AbstractServerProcess::IfModifiedSinceHeaderValue(BString& headerValue) const
{ {
@ -74,8 +211,9 @@ AbstractServerProcess::IfModifiedSinceHeaderValue(BString& headerValue,
headerValue.SetTo(modifiedHttpTime headerValue.SetTo(modifiedHttpTime
.ToString(BPrivate::B_HTTP_TIME_FORMAT_COOKIE)); .ToString(BPrivate::B_HTTP_TIME_FORMAT_COOKIE));
} else { } else {
fprintf(stderr, "unable to parse the meta-data date and time -" fprintf(stderr, "unable to parse the meta-data date and time from [%s]"
" cannot set the 'If-Modified-Since' header\n"); " - cannot set the 'If-Modified-Since' header\n",
metaDataPath.Path());
} }
return result; return result;
@ -163,10 +301,52 @@ AbstractServerProcess::ParseJsonFromFileWithListener(
} }
/*! In order to reduce the chance of failure half way through downloading a
large file, this method will download the file to a temporary file and
then it can rename the file to the final target file.
*/
status_t
AbstractServerProcess::DownloadToLocalFileAtomically(
const BPath& targetFilePath,
const BUrl& url)
{
BPath temporaryFilePath(tmpnam(NULL), NULL, true);
status_t result = DownloadToLocalFile(
temporaryFilePath, url, 0, 0);
// not copying if the data has not changed because the data will be
// zero length. This is if the result is APP_ERR_NOT_MODIFIED.
if (result == B_OK) {
// if the file is zero length then assume that something has
// gone wrong.
off_t size;
bool hasFile;
result = StorageUtils::ExistsObject(temporaryFilePath, &hasFile, NULL,
&size);
if (result == B_OK && hasFile && size > 0) {
if (rename(temporaryFilePath.Path(), targetFilePath.Path()) != 0) {
printf("[%s] did rename [%s] --> [%s]\n",
Name(), temporaryFilePath.Path(), targetFilePath.Path());
result = B_IO_ERROR;
}
}
}
return result;
}
status_t status_t
AbstractServerProcess::DownloadToLocalFile(const BPath& targetFilePath, AbstractServerProcess::DownloadToLocalFile(const BPath& targetFilePath,
const BUrl& url, uint32 redirects, uint32 failures) const BUrl& url, uint32 redirects, uint32 failures)
{ {
if (WasStopped())
return B_CANCELED;
if (redirects > MAX_REDIRECTS) { if (redirects > MAX_REDIRECTS) {
fprintf(stdout, "exceeded %d redirects --> failure\n", MAX_REDIRECTS); fprintf(stdout, "exceeded %d redirects --> failure\n", MAX_REDIRECTS);
return B_IO_ERROR; return B_IO_ERROR;
@ -177,10 +357,10 @@ AbstractServerProcess::DownloadToLocalFile(const BPath& targetFilePath,
return B_IO_ERROR; return B_IO_ERROR;
} }
fprintf(stdout, "will stream '%s' to [%s]\n", url.UrlString().String(), fprintf(stdout, "[%s] will stream '%s' to [%s]\n",
targetFilePath.Path()); Name(), url.UrlString().String(), targetFilePath.Path());
ToFileUrlProtocolListener listener(targetFilePath, LoggingName(), ToFileUrlProtocolListener listener(targetFilePath, Name(),
Logger::IsTraceEnabled()); Logger::IsTraceEnabled());
BHttpHeaders headers; BHttpHeaders headers;
@ -195,57 +375,77 @@ AbstractServerProcess::DownloadToLocalFile(const BPath& targetFilePath,
headers.AddHeader("If-Modified-Since", ifModifiedSinceHeader); headers.AddHeader("If-Modified-Since", ifModifiedSinceHeader);
} }
BHttpRequest *request = dynamic_cast<BHttpRequest *>( thread_id thread;
BUrlProtocolRoster::MakeRequest(url, &listener));
ObjectDeleter<BHttpRequest> requestDeleter(request); {
request->SetHeaders(headers); fRequest = dynamic_cast<BHttpRequest *>(
request->SetMaxRedirections(0); BUrlProtocolRoster::MakeRequest(url, &listener));
request->SetTimeout(TIMEOUT_MICROSECONDS); fRequest->SetHeaders(headers);
fRequest->SetMaxRedirections(0);
fRequest->SetTimeout(TIMEOUT_MICROSECONDS);
thread = fRequest->Run();
}
thread_id thread = request->Run();
wait_for_thread(thread, NULL); wait_for_thread(thread, NULL);
const BHttpResult& result = dynamic_cast<const BHttpResult&>( const BHttpResult& result = dynamic_cast<const BHttpResult&>(
request->Result()); fRequest->Result());
int32 statusCode = result.StatusCode(); int32 statusCode = result.StatusCode();
const BHttpHeaders responseHeaders = result.Headers();
const char *locationC = responseHeaders["Location"];
BString location;
if (locationC != NULL)
location.SetTo(locationC);
delete fRequest;
fRequest = NULL;
if (BHttpRequest::IsSuccessStatusCode(statusCode)) { if (BHttpRequest::IsSuccessStatusCode(statusCode)) {
fprintf(stdout, "did complete streaming data\n"); fprintf(stdout, "[%s] did complete streaming data [%"
B_PRIdSSIZE " bytes]\n", Name(), listener.ContentLength());
return B_OK; return B_OK;
} else if (statusCode == HTTP_STATUS_NOT_MODIFIED) { } else if (statusCode == HTTP_STATUS_NOT_MODIFIED) {
fprintf(stdout, "remote data has not changed since [%s]\n", fprintf(stdout, "[%s] remote data has not changed since [%s]\n",
ifModifiedSinceHeader.String()); Name(), ifModifiedSinceHeader.String());
return APP_ERR_NOT_MODIFIED; return APP_ERR_NOT_MODIFIED;
} else if (BHttpRequest::IsRedirectionStatusCode(statusCode)) { } else if (BHttpRequest::IsRedirectionStatusCode(statusCode)) {
const BHttpHeaders responseHeaders = result.Headers(); if (location.Length() != 0) {
const char *locationValue = responseHeaders["Location"]; BUrl location(result.Url(), location);
fprintf(stdout, "[%s] will redirect to; %s\n",
if (locationValue != NULL && strlen(locationValue) != 0) { Name(), location.UrlString().String());
BUrl location(result.Url(), locationValue);
fprintf(stdout, "will redirect to; %s\n",
location.UrlString().String());
return DownloadToLocalFile(targetFilePath, location, return DownloadToLocalFile(targetFilePath, location,
redirects + 1, 0); redirects + 1, 0);
} }
fprintf(stdout, "unable to find 'Location' header for redirect\n"); fprintf(stdout, "[%s] unable to find 'Location' header for redirect\n",
Name());
return B_IO_ERROR; return B_IO_ERROR;
} else { } else {
if (statusCode == 0 || (statusCode / 100) == 5) { if (statusCode == 0 || (statusCode / 100) == 5) {
fprintf(stdout, "error response from server; %" B_PRId32 " --> " fprintf(stdout, "error response from server [%" B_PRId32 "] --> "
"retry...\n", statusCode); "retry...\n", statusCode);
return DownloadToLocalFile(targetFilePath, url, redirects, return DownloadToLocalFile(targetFilePath, url, redirects,
failures + 1); failures + 1);
} }
fprintf(stdout, "unexpected response from server; %" B_PRId32 "\n", fprintf(stdout, "[%s] unexpected response from server [%" B_PRId32 "]\n",
statusCode); Name(), statusCode);
return B_IO_ERROR; return B_IO_ERROR;
} }
} }
status_t
AbstractServerProcess::DeleteLocalFile(const BPath& currentFilePath)
{
if (0 == remove(currentFilePath.Path()))
return B_OK;
return B_IO_ERROR;
}
/*! When a file is damaged or corrupted in some way, the file should be 'moved /*! When a file is damaged or corrupted in some way, the file should be 'moved
aside' so that it is not involved in the next update. This method will aside' so that it is not involved in the next update. This method will
create such an alternative 'damaged' file location and move this file to create such an alternative 'damaged' file location and move this file to
@ -263,13 +463,19 @@ AbstractServerProcess::MoveDamagedFileAside(const BPath& currentFilePath)
damagedFilePath.Append(damagedLeaf.String()); damagedFilePath.Append(damagedLeaf.String());
if (0 != rename(currentFilePath.Path(), damagedFilePath.Path())) { if (0 != rename(currentFilePath.Path(), damagedFilePath.Path())) {
printf("unable to move damaged file [%s] aside to [%s]\n", printf("[%s] unable to move damaged file [%s] aside to [%s]\n",
currentFilePath.Path(), damagedFilePath.Path()); Name(), currentFilePath.Path(), damagedFilePath.Path());
return B_IO_ERROR; return B_IO_ERROR;
} }
printf("did move damaged file [%s] aside to [%s]\n", printf("[%s] did move damaged file [%s] aside to [%s]\n",
currentFilePath.Path(), damagedFilePath.Path()); Name(), currentFilePath.Path(), damagedFilePath.Path());
return B_OK; return B_OK;
} }
bool
AbstractServerProcess::IsSuccess(status_t e) {
return e == B_OK || e == APP_ERR_NOT_MODIFIED;
}

80
src/apps/haikudepot/server/AbstractServerProcess.h
View File

@ -6,26 +6,66 @@
#ifndef ABSTRACT_SERVER_PROCESS_H #ifndef ABSTRACT_SERVER_PROCESS_H
#define ABSTRACT_SERVER_PROCESS_H #define ABSTRACT_SERVER_PROCESS_H
#include <HttpRequest.h>
#include <Json.h> #include <Json.h>
#include <String.h> #include <String.h>
#include <Url.h> #include <Url.h>
#include "StandardMetaData.h" #include "StandardMetaData.h"
#include "Stoppable.h"
#define APP_ERR_NOT_MODIFIED (B_APP_ERROR_BASE + 452) #define APP_ERR_NOT_MODIFIED (B_APP_ERROR_BASE + 452)
#define APP_ERR_NO_DATA (B_APP_ERROR_BASE + 453)
class AbstractServerProcess { typedef enum process_options {
SERVER_PROCESS_NO_NETWORKING = 1 << 0,
SERVER_PROCESS_PREFER_CACHE = 1 << 1,
SERVER_PROCESS_DROP_CACHE = 1 << 2
} process_options;
typedef enum process_state {
SERVER_PROCESS_INITIAL = 1,
SERVER_PROCESS_RUNNING = 2,
SERVER_PROCESS_COMPLETE = 3
} process_state;
/*! Clients are able to subclass from this 'interface' in order to accept
call-backs when a process has exited; either through success or through
failure.
*/
class AbstractServerProcessListener {
public: public:
virtual status_t Run() = 0; virtual void ServerProcessExited() = 0;
};
class AbstractServerProcess : public Stoppable {
public:
AbstractServerProcess(
AbstractServerProcessListener* listener,
uint32 options);
virtual ~AbstractServerProcess();
virtual const char* Name() = 0;
status_t Run();
status_t Stop();
status_t ErrorStatus();
bool IsRunning();
bool WasStopped();
protected: protected:
virtual status_t RunInternal() = 0;
virtual status_t StopInternal();
virtual void GetStandardMetaDataPath( virtual void GetStandardMetaDataPath(
BPath& path) const = 0; BPath& path) const = 0;
virtual void GetStandardMetaDataJsonPath( virtual void GetStandardMetaDataJsonPath(
BString& jsonPath) const = 0; BString& jsonPath) const = 0;
virtual const char* LoggingName() const = 0;
status_t IfModifiedSinceHeaderValue( status_t IfModifiedSinceHeaderValue(
BString& headerValue) const; BString& headerValue) const;
@ -43,15 +83,41 @@ protected:
BJsonEventListener *listener, BJsonEventListener *listener,
const BPath& path) const; const BPath& path) const;
status_t DownloadToLocalFileAtomically(
const BPath& targetFilePath,
const BUrl& url);
status_t DeleteLocalFile(const BPath& currentFilePath);
status_t MoveDamagedFileAside(
const BPath& currentFilePath);
bool HasOption(uint32 flag);
bool ShouldAttemptNetworkDownload(
bool hasDataAlready);
static bool IsSuccess(status_t e);
private:
BLocker fLock;
AbstractServerProcessListener*
fListener;
bool fWasStopped;
process_state fProcessState;
status_t fErrorStatus;
uint32 fOptions;
BHttpRequest* fRequest;
process_state ProcessState();
void SetErrorStatus(status_t value);
void SetProcessState(process_state value);
status_t DownloadToLocalFile( status_t DownloadToLocalFile(
const BPath& targetFilePath, const BPath& targetFilePath,
const BUrl& url, const BUrl& url,
uint32 redirects, uint32 failures); uint32 redirects, uint32 failures);
status_t MoveDamagedFileAside(
const BPath& currentFilePath);
private:
bool LooksLikeGzip(const char *pathStr) const; bool LooksLikeGzip(const char *pathStr) const;
}; };

212
src/apps/haikudepot/server/PkgDataUpdateProcess.cpp
View File

@ -11,6 +11,7 @@
#include <Autolock.h> #include <Autolock.h>
#include <FileIO.h> #include <FileIO.h>
#include <support/StopWatch.h>
#include <Url.h> #include <Url.h>
#include "Logger.h" #include "Logger.h"
@ -27,39 +28,46 @@
packages as they are parsed and processing them. packages as they are parsed and processing them.
*/ */
class PackageFillingPkgListener : public DumpExportPkgListener { class PackageFillingPkgListener :
public DumpExportPkgListener, public PackageConsumer {
public: public:
PackageFillingPkgListener( PackageFillingPkgListener(Model *model,
const PackageList& packages, BString& depotName, Stoppable* stoppable);
const CategoryList& categories,
BLocker& lock);
virtual ~PackageFillingPkgListener(); virtual ~PackageFillingPkgListener();
virtual void Handle(DumpExportPkg* item); virtual bool ConsumePackage(const PackageInfoRef& package,
void *context);
virtual bool Handle(DumpExportPkg* item);
virtual void Complete(); virtual void Complete();
uint32 Count();
private: private:
int32 IndexOfPackageByName(const BString& name) const; int32 IndexOfPackageByName(const BString& name) const;
int32 IndexOfCategoryByName( int32 IndexOfCategoryByName(
const BString& name) const; const BString& name) const;
int32 IndexOfCategoryByCode( int32 IndexOfCategoryByCode(
const BString& code) const; const BString& code) const;
const PackageList& fPackages;
const CategoryList& fCategories;
BLocker& fLock;
BString fDepotName;
Model* fModel;
CategoryList fCategories;
Stoppable* fStoppable;
uint32 fCount;
bool fDebugEnabled;
}; };
PackageFillingPkgListener::PackageFillingPkgListener( PackageFillingPkgListener::PackageFillingPkgListener(Model* model,
const PackageList& packages, const CategoryList& categories, BString& depotName, Stoppable* stoppable)
BLocker& lock)
: :
fPackages(packages), fDepotName(depotName),
fCategories(categories), fModel(model),
fLock(lock) fStoppable(stoppable),
fCount(0),
fDebugEnabled(Logger::IsDebugEnabled())
{ {
fCategories = model->Categories();
} }
@ -70,32 +78,14 @@ PackageFillingPkgListener::~PackageFillingPkgListener()
// TODO; performance could be improved by not needing the linear search // TODO; performance could be improved by not needing the linear search
int32 inline int32
PackageFillingPkgListener::IndexOfPackageByName(
const BString& name) const
{
int32 i;
for (i = 0; i < fPackages.CountItems(); i++) {
const PackageInfoRef& packageInfo = fPackages.ItemAt(i);
if (packageInfo->Name() == name)
return i;
}
return -1;
}
// TODO; performance could be improved by not needing the linear search
int32
PackageFillingPkgListener::IndexOfCategoryByName( PackageFillingPkgListener::IndexOfCategoryByName(
const BString& name) const const BString& name) const
{ {
int32 i; int32 i;
int32 categoryCount = fCategories.CountItems();
for (i = 0; i < fCategories.CountItems(); i++) { for (i = 0; i < categoryCount; i++) {
const CategoryRef categoryRef = fCategories.ItemAtFast(i); const CategoryRef categoryRef = fCategories.ItemAtFast(i);
if (categoryRef->Name() == name) if (categoryRef->Name() == name)
@ -106,18 +96,12 @@ PackageFillingPkgListener::IndexOfCategoryByName(
} }
void bool
PackageFillingPkgListener::Handle(DumpExportPkg* pkg) PackageFillingPkgListener::ConsumePackage(const PackageInfoRef& package,
void *context)
{ {
BAutolock locker(fLock); // lock from the model. DumpExportPkg* pkg = static_cast<DumpExportPkg*>(context);
int32 packageIndex = IndexOfPackageByName(*(pkg->Name()));
if (packageIndex == -1) {
printf("unable to find package data for pkg name [%s]\n",
pkg->Name()->String());
} else {
int32 i; int32 i;
const PackageInfoRef& packageInfo = fPackages.ItemAt(packageIndex);
if (0 != pkg->CountPkgVersions()) { if (0 != pkg->CountPkgVersions()) {
@ -127,20 +111,21 @@ PackageFillingPkgListener::Handle(DumpExportPkg* pkg)
DumpExportPkgVersion* pkgVersion = pkg->PkgVersionsItemAt(0); DumpExportPkgVersion* pkgVersion = pkg->PkgVersionsItemAt(0);
if (!pkgVersion->TitleIsNull()) if (!pkgVersion->TitleIsNull())
packageInfo->SetTitle(*(pkgVersion->Title())); package->SetTitle(*(pkgVersion->Title()));
if (!pkgVersion->SummaryIsNull()) if (!pkgVersion->SummaryIsNull())
packageInfo->SetShortDescription(*(pkgVersion->Summary())); package->SetShortDescription(*(pkgVersion->Summary()));
if (!pkgVersion->DescriptionIsNull()) { if (!pkgVersion->DescriptionIsNull())
packageInfo->SetFullDescription(*(pkgVersion->Description())); package->SetFullDescription(*(pkgVersion->Description()));
}
if (!pkgVersion->PayloadLengthIsNull()) if (!pkgVersion->PayloadLengthIsNull())
packageInfo->SetSize(pkgVersion->PayloadLength()); package->SetSize(pkgVersion->PayloadLength());
} }
for (i = 0; i < pkg->CountPkgCategories(); i++) { int32 countPkgCategories = pkg->CountPkgCategories();
for (i = 0; i < countPkgCategories; i++) {
BString* categoryCode = pkg->PkgCategoriesItemAt(i)->Code(); BString* categoryCode = pkg->PkgCategoriesItemAt(i)->Code();
int categoryIndex = IndexOfCategoryByName(*(categoryCode)); int categoryIndex = IndexOfCategoryByName(*(categoryCode));
@ -148,7 +133,7 @@ PackageFillingPkgListener::Handle(DumpExportPkg* pkg)
printf("unable to find the category for [%s]\n", printf("unable to find the category for [%s]\n",
categoryCode->String()); categoryCode->String());
} else { } else {
packageInfo->AddCategory( package->AddCategory(
fCategories.ItemAtFast(categoryIndex)); fCategories.ItemAtFast(categoryIndex));
} }
} }
@ -156,20 +141,20 @@ PackageFillingPkgListener::Handle(DumpExportPkg* pkg)
if (!pkg->DerivedRatingIsNull()) { if (!pkg->DerivedRatingIsNull()) {
RatingSummary summary; RatingSummary summary;
summary.averageRating = pkg->DerivedRating(); summary.averageRating = pkg->DerivedRating();
packageInfo->SetRatingSummary(summary); package->SetRatingSummary(summary);
} }
if (!pkg->ProminenceOrderingIsNull()) { if (!pkg->ProminenceOrderingIsNull())
packageInfo->SetProminence(pkg->ProminenceOrdering()); package->SetProminence(pkg->ProminenceOrdering());
}
if (!pkg->PkgChangelogContentIsNull()) { if (!pkg->PkgChangelogContentIsNull())
packageInfo->SetChangelog(*(pkg->PkgChangelogContent())); package->SetChangelog(*(pkg->PkgChangelogContent()));
}
for (i = 0; i < pkg->CountPkgScreenshots(); i++) { int32 countPkgScreenshots = pkg->CountPkgScreenshots();
for (i = 0; i < countPkgScreenshots; i++) {
DumpExportPkgScreenshot* screenshot = pkg->PkgScreenshotsItemAt(i); DumpExportPkgScreenshot* screenshot = pkg->PkgScreenshotsItemAt(i);
packageInfo->AddScreenshotInfo(ScreenshotInfo( package->AddScreenshotInfo(ScreenshotInfo(
*(screenshot->Code()), *(screenshot->Code()),
static_cast<int32>(screenshot->Width()), static_cast<int32>(screenshot->Width()),
static_cast<int32>(screenshot->Height()), static_cast<int32>(screenshot->Height()),
@ -177,10 +162,29 @@ PackageFillingPkgListener::Handle(DumpExportPkg* pkg)
)); ));
} }
if (Logger::IsDebugEnabled()) { if (fDebugEnabled) {
printf("did populate data for [%s]\n", pkg->Name()->String()); printf("did populate data for [%s] (%s)\n", pkg->Name()->String(),
} fDepotName.String());
} }
fCount++;
return !fStoppable->WasStopped();
}
uint32
PackageFillingPkgListener::Count()
{
return fCount;
}
bool
PackageFillingPkgListener::Handle(DumpExportPkg* pkg)
{
fModel->ForPackageByNameInDepot(fDepotName, *(pkg->Name()), this, pkg);
return !fStoppable->WasStopped();
} }
@ -191,20 +195,22 @@ PackageFillingPkgListener::Complete()
PkgDataUpdateProcess::PkgDataUpdateProcess( PkgDataUpdateProcess::PkgDataUpdateProcess(
AbstractServerProcessListener* listener,
const BPath& localFilePath, const BPath& localFilePath,
BLocker& lock,
BString repositorySourceCode,
BString naturalLanguageCode, BString naturalLanguageCode,
const PackageList& packages, BString repositorySourceCode,
const CategoryList& categories) BString depotName,
Model *model,
uint32 options)
: :
AbstractSingleFileServerProcess(listener, options),
fLocalFilePath(localFilePath), fLocalFilePath(localFilePath),
fNaturalLanguageCode(naturalLanguageCode), fNaturalLanguageCode(naturalLanguageCode),
fRepositorySourceCode(repositorySourceCode), fRepositorySourceCode(repositorySourceCode),
fPackages(packages), fModel(model),
fCategories(categories), fDepotName(depotName)
fLock(lock)
{ {
fName.SetToFormat("PkgDataUpdateProcess<%s>", depotName.String());
} }
@ -213,53 +219,50 @@ PkgDataUpdateProcess::~PkgDataUpdateProcess()
} }
status_t const char*
PkgDataUpdateProcess::Run() PkgDataUpdateProcess::Name()
{ {
printf("will fetch packages' data\n"); return fName.String();
}
BString
PkgDataUpdateProcess::UrlPathComponent()
{
BString urlPath; BString urlPath;
urlPath.SetToFormat("/__pkg/all-%s-%s.json.gz", urlPath.SetToFormat("/__pkg/all-%s-%s.json.gz",
fRepositorySourceCode.String(), fRepositorySourceCode.String(),
fNaturalLanguageCode.String()); fNaturalLanguageCode.String());
return urlPath;
}
status_t result = DownloadToLocalFile(fLocalFilePath,
ServerSettings::CreateFullUrl(urlPath),
0, 0);
if (result == B_OK || result == APP_ERR_NOT_MODIFIED) { BPath&
printf("did fetch packages' data\n"); PkgDataUpdateProcess::LocalPath()
{
// now load the data in and process it. return fLocalFilePath;
printf("will process packages' data\n");
result = PopulateDataToDepots();
switch (result) {
case B_OK:
printf("did process packages' data\n");
break;
default:
MoveDamagedFileAside(fLocalFilePath);
break;
}
}
return result;
} }
status_t status_t
PkgDataUpdateProcess::PopulateDataToDepots() PkgDataUpdateProcess::ProcessLocalData()
{ {
BStopWatch watch("PkgDataUpdateProcess::ProcessLocalData", true);
PackageFillingPkgListener* itemListener = PackageFillingPkgListener* itemListener =
new PackageFillingPkgListener(fPackages, fCategories, fLock); new PackageFillingPkgListener(fModel, fDepotName, this);
BulkContainerDumpExportPkgJsonListener* listener = BulkContainerDumpExportPkgJsonListener* listener =
new BulkContainerDumpExportPkgJsonListener(itemListener); new BulkContainerDumpExportPkgJsonListener(itemListener);
status_t result = ParseJsonFromFileWithListener(listener, fLocalFilePath); status_t result = ParseJsonFromFileWithListener(listener, fLocalFilePath);
if (Logger::IsInfoEnabled()) {
double secs = watch.ElapsedTime() / 1000000.0;
fprintf(stdout, "[%s] did process %" B_PRIi32 " packages' data "
"in (%6.3g secs)\n", Name(), itemListener->Count(), secs);
}
if (B_OK != result) if (B_OK != result)
return result; return result;
@ -280,10 +283,3 @@ PkgDataUpdateProcess::GetStandardMetaDataJsonPath(
{ {
jsonPath.SetTo("$.info"); jsonPath.SetTo("$.info");
} }
const char*
PkgDataUpdateProcess::LoggingName() const
{
return "pkg-data-update";
}

26
src/apps/haikudepot/server/PkgDataUpdateProcess.h
View File

@ -7,8 +7,9 @@
#define PACKAGE_DATA_UPDATE_PROCESS_H #define PACKAGE_DATA_UPDATE_PROCESS_H
#include "AbstractServerProcess.h" #include "AbstractSingleFileServerProcess.h"
#include "Model.h"
#include "PackageInfo.h" #include "PackageInfo.h"
#include <File.h> #include <File.h>
@ -17,34 +18,37 @@
#include <Url.h> #include <Url.h>
class PkgDataUpdateProcess : public AbstractServerProcess { class PkgDataUpdateProcess : public AbstractSingleFileServerProcess {
public: public:
PkgDataUpdateProcess( PkgDataUpdateProcess(
AbstractServerProcessListener* listener,
const BPath& localFilePath, const BPath& localFilePath,
BLocker& lock,
BString naturalLanguageCode, BString naturalLanguageCode,
BString repositorySourceCode, BString repositorySourceCode,
const PackageList& packages, BString depotName,
const CategoryList& categories); Model *model,
uint32 options);
virtual ~PkgDataUpdateProcess(); virtual ~PkgDataUpdateProcess();
status_t Run(); const char* Name();
protected: protected:
void GetStandardMetaDataPath(BPath& path) const; void GetStandardMetaDataPath(BPath& path) const;
void GetStandardMetaDataJsonPath( void GetStandardMetaDataJsonPath(
BString& jsonPath) const; BString& jsonPath) const;
const char* LoggingName() const;
BString UrlPathComponent();
status_t ProcessLocalData();
BPath& LocalPath();
private: private:
status_t PopulateDataToDepots();
BPath fLocalFilePath; BPath fLocalFilePath;
BString fNaturalLanguageCode; BString fNaturalLanguageCode;
BString fRepositorySourceCode; BString fRepositorySourceCode;
const PackageList& fPackages; Model* fModel;
const CategoryList& fCategories; BString fDepotName;
BLocker& fLock; BString fName;
}; };

198
src/apps/haikudepot/server/RepositoryDataUpdateProcess.cpp
View File

@ -14,6 +14,7 @@
#include "ServerSettings.h" #include "ServerSettings.h"
#include "StorageUtils.h" #include "StorageUtils.h"
#include "Logger.h"
#include "DumpExportRepository.h" #include "DumpExportRepository.h"
#include "DumpExportRepositorySource.h" #include "DumpExportRepositorySource.h"
#include "DumpExportRepositoryJsonListener.h" #include "DumpExportRepositoryJsonListener.h"
@ -25,13 +26,15 @@
from the server with the data about the depot. from the server with the data about the depot.
*/ */
class DepotMatchingRepositoryListener : public DumpExportRepositoryListener { class DepotMatchingRepositoryListener :
public DumpExportRepositoryListener, public DepotMapper {
public: public:
DepotMatchingRepositoryListener( DepotMatchingRepositoryListener(Model* model,
DepotList* depotList); Stoppable* stoppable);
virtual ~DepotMatchingRepositoryListener(); virtual ~DepotMatchingRepositoryListener();
virtual void Handle(DumpExportRepository* item); virtual DepotInfo MapDepot(const DepotInfo& depot, void *context);
virtual bool Handle(DumpExportRepository* item);
virtual void Complete(); virtual void Complete();
private: private:
@ -43,15 +46,17 @@ private:
int32 IndexOfUnassociatedDepotByUrl( int32 IndexOfUnassociatedDepotByUrl(
const BString& url) const; const BString& url) const;
DepotList* fDepotList; Model* fModel;
Stoppable* fStoppable;
}; };
DepotMatchingRepositoryListener::DepotMatchingRepositoryListener( DepotMatchingRepositoryListener::DepotMatchingRepositoryListener(
DepotList* depotList) Model* model, Stoppable* stoppable)
:
fModel(model),
fStoppable(stoppable)
{ {
fDepotList = depotList;
} }
@ -60,110 +65,86 @@ DepotMatchingRepositoryListener::~DepotMatchingRepositoryListener()
} }
void struct repository_and_repository_source {
DepotMatchingRepositoryListener::NormalizeUrl(BUrl& url) const DumpExportRepository* repository;
DumpExportRepositorySource* repositorySource;
};
/*! This is invoked as a result of logic in 'Handle(..)' that requests that the
model call this method with the requested DepotInfo instance.
*/
DepotInfo
DepotMatchingRepositoryListener::MapDepot(const DepotInfo& depot, void *context)
{ {
if (url.Protocol() == "https") repository_and_repository_source* repositoryAndRepositorySource =
url.SetProtocol("http"); (repository_and_repository_source*) context;
BString* repositoryCode =
repositoryAndRepositorySource->repository->Code();
BString* repositorySourceCode =
repositoryAndRepositorySource->repositorySource->Code();
BString path(url.Path()); DepotInfo modifiedDepotInfo(depot);
modifiedDepotInfo.SetWebAppRepositoryCode(BString(*repositoryCode));
modifiedDepotInfo.SetWebAppRepositorySourceCode(
BString(*repositorySourceCode));
if (path.EndsWith("/")) if (Logger::IsDebugEnabled()) {
url.SetPath(path.Truncate(path.Length() - 1)); printf("associated dept [%s] (%s) with server repository "
"source [%s] (%s)\n", modifiedDepotInfo.Name().String(),
modifiedDepotInfo.BaseURL().String(),
repositorySourceCode->String(),
repositoryAndRepositorySource->repositorySource->Url()->String());
} else {
printf("associated depot [%s] with server repository source [%s]\n",
modifiedDepotInfo.Name().String(), repositorySourceCode->String());
}
return modifiedDepotInfo;
} }
bool bool
DepotMatchingRepositoryListener::IsUnassociatedDepotByUrl(
const DepotInfo& depotInfo, const BString& urlStr) const
{
if (depotInfo.BaseURL().Length() > 0
&& depotInfo.WebAppRepositorySourceCode().Length() == 0) {
BUrl depotInfoBaseUrl(depotInfo.BaseURL());
BUrl url(urlStr);
NormalizeUrl(depotInfoBaseUrl);
NormalizeUrl(url);
if (depotInfoBaseUrl == url)
return true;
}
return false;
}
int32
DepotMatchingRepositoryListener::IndexOfUnassociatedDepotByUrl(
const BString& url) const
{
int32 i;
for (i = 0; i < fDepotList->CountItems(); i++) {
const DepotInfo& depot = fDepotList->ItemAt(i);
if (IsUnassociatedDepotByUrl(depot, url))
return i;
}
return -1;
}
void
DepotMatchingRepositoryListener::Handle(DumpExportRepository* repository) DepotMatchingRepositoryListener::Handle(DumpExportRepository* repository)
{ {
int32 i; int32 i;
for (i = 0; i < repository->CountRepositorySources(); i++) { for (i = 0; i < repository->CountRepositorySources(); i++) {
DumpExportRepositorySource *repositorySource = repository_and_repository_source repositoryAndRepositorySource;
repositoryAndRepositorySource.repository = repository;
repositoryAndRepositorySource.repositorySource =
repository->RepositorySourcesItemAt(i); repository->RepositorySourcesItemAt(i);
int32 depotIndex = IndexOfUnassociatedDepotByUrl(
*(repositorySource->Url()));
if (depotIndex >= 0) { // TODO; replace with the repo info url
DepotInfo modifiedDepotInfo(fDepotList->ItemAt(depotIndex)); BString* url = repositoryAndRepositorySource.repositorySource->Url();
modifiedDepotInfo.SetWebAppRepositoryCode(
BString(*(repository->Code())));
modifiedDepotInfo.SetWebAppRepositorySourceCode(
BString(*(repositorySource->Code())));
fDepotList->Replace(depotIndex, modifiedDepotInfo);
printf("associated depot [%s] with server" if (url->Length() > 0) {
" repository source [%s]\n", modifiedDepotInfo.Name().String(), fModel->ReplaceDepotByUrl(*url, this,
repositorySource->Code()->String()); &repositoryAndRepositorySource);
} }
} }
return !fStoppable->WasStopped();
} }
void void
DepotMatchingRepositoryListener::Complete() DepotMatchingRepositoryListener::Complete()
{ {
int32 i;
for (i = 0; i < fDepotList->CountItems(); i++) {
const DepotInfo& depot = fDepotList->ItemAt(i);
if (depot.WebAppRepositoryCode().Length() == 0) {
printf("depot [%s]", depot.Name().String());
if (depot.BaseURL().Length() > 0)
printf(" (%s)", depot.BaseURL().String());
printf(" correlates with no repository in the haiku"
"depot server system\n");
}
}
} }
RepositoryDataUpdateProcess::RepositoryDataUpdateProcess( RepositoryDataUpdateProcess::RepositoryDataUpdateProcess(
AbstractServerProcessListener* listener,
const BPath& localFilePath, const BPath& localFilePath,
DepotList* depotList) Model* model,
uint32 options)
:
AbstractSingleFileServerProcess(listener, options),
fLocalFilePath(localFilePath),
fModel(model)
{ {
fLocalFilePath = localFilePath;
fDepotList = depotList;
} }
@ -172,46 +153,32 @@ RepositoryDataUpdateProcess::~RepositoryDataUpdateProcess()
} }
status_t const char*
RepositoryDataUpdateProcess::Run() RepositoryDataUpdateProcess::Name()
{ {
printf("will fetch repositories data\n"); return "RepositoryDataUpdateProcess";
}
// TODO: add language ISO code to the path; just 'en' for now.
status_t result = DownloadToLocalFile(fLocalFilePath,
ServerSettings::CreateFullUrl("/__repository/all-en.json.gz"),
0, 0);
if (result == B_OK || result == APP_ERR_NOT_MODIFIED) { BString
printf("did fetch repositories data\n"); RepositoryDataUpdateProcess::UrlPathComponent()
{
return BString("/__repository/all-en.json.gz");
}
// now load the data in and process it.
printf("will process repository data and match to depots\n"); BPath&
result = PopulateDataToDepots(); RepositoryDataUpdateProcess::LocalPath()
{
switch (result) { return fLocalFilePath;
case B_OK:
printf("did process repository data and match to depots\n");
break;
default:
MoveDamagedFileAside(fLocalFilePath);
break;
}
} else {
printf("an error has arisen downloading the repositories' data\n");
}
return result;
} }
status_t status_t
RepositoryDataUpdateProcess::PopulateDataToDepots() RepositoryDataUpdateProcess::ProcessLocalData()
{ {
DepotMatchingRepositoryListener* itemListener = DepotMatchingRepositoryListener* itemListener =
new DepotMatchingRepositoryListener(fDepotList); new DepotMatchingRepositoryListener(fModel, this);
BulkContainerDumpExportRepositoryJsonListener* listener = BulkContainerDumpExportRepositoryJsonListener* listener =
new BulkContainerDumpExportRepositoryJsonListener(itemListener); new BulkContainerDumpExportRepositoryJsonListener(itemListener);
@ -238,10 +205,3 @@ RepositoryDataUpdateProcess::GetStandardMetaDataJsonPath(
{ {
jsonPath.SetTo("$.info"); jsonPath.SetTo("$.info");
} }
const char*
RepositoryDataUpdateProcess::LoggingName() const
{
return "repo-data-update";
}

19
src/apps/haikudepot/server/RepositoryDataUpdateProcess.h
View File

@ -7,8 +7,9 @@
#define REPOSITORY_DATA_UPDATE_PROCESS_H #define REPOSITORY_DATA_UPDATE_PROCESS_H
#include "AbstractServerProcess.h" #include "AbstractSingleFileServerProcess.h"
#include "Model.h"
#include "PackageInfo.h" #include "PackageInfo.h"
#include <File.h> #include <File.h>
@ -17,27 +18,29 @@
#include <Url.h> #include <Url.h>
class RepositoryDataUpdateProcess : public AbstractServerProcess { class RepositoryDataUpdateProcess : public AbstractSingleFileServerProcess {
public: public:
RepositoryDataUpdateProcess( RepositoryDataUpdateProcess(
AbstractServerProcessListener* listener,
const BPath& localFilePath, const BPath& localFilePath,
DepotList* depotList); Model* model, uint32 options);
virtual ~RepositoryDataUpdateProcess(); virtual ~RepositoryDataUpdateProcess();
status_t Run(); const char* Name();
protected: protected:
void GetStandardMetaDataPath(BPath& path) const; void GetStandardMetaDataPath(BPath& path) const;
void GetStandardMetaDataJsonPath( void GetStandardMetaDataJsonPath(
BString& jsonPath) const; BString& jsonPath) const;
const char* LoggingName() const;
BString UrlPathComponent();
status_t ProcessLocalData();
BPath& LocalPath();
private: private:
status_t PopulateDataToDepots();
BPath fLocalFilePath; BPath fLocalFilePath;
DepotList* fDepotList; Model* fModel;
}; };

144
src/apps/haikudepot/server/ServerIconExportUpdateProcess.cpp
View File

@ -9,9 +9,12 @@
#include <sys/stat.h> #include <sys/stat.h>
#include <time.h> #include <time.h>
#include <Autolock.h>
#include <FileIO.h> #include <FileIO.h>
#include <support/StopWatch.h>
#include <support/ZlibCompressionAlgorithm.h> #include <support/ZlibCompressionAlgorithm.h>
#include "Logger.h"
#include "ServerSettings.h" #include "ServerSettings.h"
#include "StorageUtils.h" #include "StorageUtils.h"
#include "TarArchiveService.h" #include "TarArchiveService.h"
@ -20,9 +23,17 @@
/*! This constructor will locate the cached data in a standardized location */ /*! This constructor will locate the cached data in a standardized location */
ServerIconExportUpdateProcess::ServerIconExportUpdateProcess( ServerIconExportUpdateProcess::ServerIconExportUpdateProcess(
const BPath& localStorageDirectoryPath) AbstractServerProcessListener* listener,
const BPath& localStorageDirectoryPath,
Model* model,
uint32 options)
:
AbstractServerProcess(listener, options),
fLocalStorageDirectoryPath(localStorageDirectoryPath),
fModel(model),
fLocalIconStore(LocalIconStore(localStorageDirectoryPath)),
fCountIconsSet(0)
{ {
fLocalStorageDirectoryPath = localStorageDirectoryPath;
} }
@ -31,8 +42,106 @@ ServerIconExportUpdateProcess::~ServerIconExportUpdateProcess()
} }
const char*
ServerIconExportUpdateProcess::Name()
{
return "ServerIconExportUpdateProcess";
}
status_t status_t
ServerIconExportUpdateProcess::Run() ServerIconExportUpdateProcess::RunInternal()
{
status_t result = B_OK;
if (IsSuccess(result) && HasOption(SERVER_PROCESS_DROP_CACHE)) {
result = StorageUtils::RemoveDirectoryContents(
fLocalStorageDirectoryPath);
}
if (IsSuccess(result)) {
bool hasData;
result = HasLocalData(&hasData);
if (IsSuccess(result) && ShouldAttemptNetworkDownload(hasData))
result = DownloadAndUnpack();
if (IsSuccess(result)) {
status_t hasDataResult = HasLocalData(&hasData);
if (IsSuccess(hasDataResult) && !hasData)
result = APP_ERR_NO_DATA;
}
}
if (IsSuccess(result) && !WasStopped())
result = Populate();
return result;
}
status_t
ServerIconExportUpdateProcess::PopulateForPkg(const PackageInfoRef& package)
{
BPath bestIconPath;
if ( fLocalIconStore.TryFindIconPath(
package->Name(), bestIconPath) == B_OK) {
BFile bestIconFile(bestIconPath.Path(), O_RDONLY);
BitmapRef bitmapRef(new(std::nothrow)SharedBitmap(bestIconFile), true);
// TODO; somehow handle the locking!
//BAutolock locker(&fLock);
package->SetIcon(bitmapRef);
if (Logger::IsDebugEnabled()) {
fprintf(stdout, "have set the package icon for [%s] from [%s]\n",
package->Name().String(), bestIconPath.Path());
}
fCountIconsSet++;
return B_OK;
}
if (Logger::IsDebugEnabled()) {
fprintf(stdout, "did not set the package icon for [%s]; no data\n",
package->Name().String());
}
return B_FILE_NOT_FOUND;
}
bool
ServerIconExportUpdateProcess::ConsumePackage(
const PackageInfoRef& packageInfoRef, void* context)
{
PopulateForPkg(packageInfoRef);
return !WasStopped();
}
status_t
ServerIconExportUpdateProcess::Populate()
{
BStopWatch watch("ServerIconExportUpdateProcess::Populate", true);
fModel->ForAllPackages(this, NULL);
if (Logger::IsInfoEnabled()) {
double secs = watch.ElapsedTime() / 1000000.0;
fprintf(stdout, "did populate %" B_PRId32 " packages' icons"
" (%6.3g secs)\n", fCountIconsSet, secs);
}
return B_OK;
}
status_t
ServerIconExportUpdateProcess::DownloadAndUnpack()
{ {
BPath tarGzFilePath(tmpnam(NULL)); BPath tarGzFilePath(tmpnam(NULL));
status_t result = B_OK; status_t result = B_OK;
@ -59,10 +168,15 @@ ServerIconExportUpdateProcess::Run()
zlibDecompressionParameters, tarIn); zlibDecompressionParameters, tarIn);
if (result == B_OK) { if (result == B_OK) {
BStopWatch watch("ServerIconExportUpdateProcess::DownloadAndUnpack_Unpack", true);
result = TarArchiveService::Unpack(*tarIn, result = TarArchiveService::Unpack(*tarIn,
fLocalStorageDirectoryPath); fLocalStorageDirectoryPath, NULL);
if (result == B_OK) { if (result == B_OK) {
double secs = watch.ElapsedTime() / 1000000.0;
fprintf(stdout, "did unpack icon tgz in (%6.3g secs)\n", secs);
if (0 != remove(tarGzFilePath.Path())) { if (0 != remove(tarGzFilePath.Path())) {
fprintf(stdout, "unable to delete the temporary tgz path; " fprintf(stdout, "unable to delete the temporary tgz path; "
"%s\n", tarGzFilePath.Path()); "%s\n", tarGzFilePath.Path());
@ -79,6 +193,17 @@ ServerIconExportUpdateProcess::Run()
} }
status_t
ServerIconExportUpdateProcess::HasLocalData(bool* result) const
{
BPath path;
off_t size;
GetStandardMetaDataPath(path);
return StorageUtils::ExistsObject(path, result, NULL, &size)
&& size > 0;
}
void void
ServerIconExportUpdateProcess::GetStandardMetaDataPath(BPath& path) const ServerIconExportUpdateProcess::GetStandardMetaDataPath(BPath& path) const
{ {
@ -96,16 +221,9 @@ ServerIconExportUpdateProcess::GetStandardMetaDataJsonPath(
} }
const char*
ServerIconExportUpdateProcess::LoggingName() const
{
return "icon-export-update";
}
status_t status_t
ServerIconExportUpdateProcess::Download(BPath& tarGzFilePath) ServerIconExportUpdateProcess::Download(BPath& tarGzFilePath)
{ {
return DownloadToLocalFile(tarGzFilePath, return DownloadToLocalFileAtomically(tarGzFilePath,
ServerSettings::CreateFullUrl("/__pkgicon/all.tar.gz"), 0, 0); ServerSettings::CreateFullUrl("/__pkgicon/all.tar.gz"));
} }

25
src/apps/haikudepot/server/ServerIconExportUpdateProcess.h
View File

@ -8,6 +8,8 @@
#include "AbstractServerProcess.h" #include "AbstractServerProcess.h"
#include "LocalIconStore.h"
#include "Model.h"
#include <File.h> #include <File.h>
#include <Path.h> #include <Path.h>
@ -15,26 +17,37 @@
#include <Url.h> #include <Url.h>
class ServerIconExportUpdateProcess : public AbstractServerProcess { class ServerIconExportUpdateProcess :
public AbstractServerProcess, public PackageConsumer {
public: public:
ServerIconExportUpdateProcess( ServerIconExportUpdateProcess(
const BPath& localStorageDirectoryPath); AbstractServerProcessListener* listener,
const BPath& localStorageDirectoryPath,
Model* model, uint32 options);
virtual ~ServerIconExportUpdateProcess(); virtual ~ServerIconExportUpdateProcess();
status_t Run(); const char* Name();
status_t RunInternal();
virtual bool ConsumePackage(
const PackageInfoRef& packageInfoRef,
void *context);
protected: protected:
status_t PopulateForPkg(const PackageInfoRef& package);
status_t Populate();
status_t DownloadAndUnpack();
status_t HasLocalData(bool* result) const;
void GetStandardMetaDataPath(BPath& path) const; void GetStandardMetaDataPath(BPath& path) const;
void GetStandardMetaDataJsonPath( void GetStandardMetaDataJsonPath(
BString& jsonPath) const; BString& jsonPath) const;
const char* LoggingName() const;
private: private:
status_t Download(BPath& tarGzFilePath); status_t Download(BPath& tarGzFilePath);
BPath fLocalStorageDirectoryPath; BPath fLocalStorageDirectoryPath;
Model* fModel;
LocalIconStore fLocalIconStore;
int32 fCountIconsSet;
}; };

44
src/apps/haikudepot/server/ServerSettings.cpp
View File

@ -21,6 +21,9 @@
BUrl ServerSettings::sBaseUrl = BUrl(BASEURL_DEFAULT); BUrl ServerSettings::sBaseUrl = BUrl(BASEURL_DEFAULT);
BString ServerSettings::sUserAgent = BString(); BString ServerSettings::sUserAgent = BString();
pthread_once_t ServerSettings::sUserAgentInitOnce = PTHREAD_ONCE_INIT; pthread_once_t ServerSettings::sUserAgentInitOnce = PTHREAD_ONCE_INIT;
bool ServerSettings::sPreferCache = false;
bool ServerSettings::sDropCache = false;
bool ServerSettings::sForceNoNetwork = false;
status_t status_t
@ -102,6 +105,7 @@ ServerSettings::_GetUserAgentVersionString()
return result; return result;
} }
void void
ServerSettings::AugmentHeaders(BHttpHeaders& headers) ServerSettings::AugmentHeaders(BHttpHeaders& headers)
{ {
@ -109,3 +113,43 @@ ServerSettings::AugmentHeaders(BHttpHeaders& headers)
} }
bool
ServerSettings::PreferCache()
{
return sPreferCache;
}
void
ServerSettings::SetPreferCache(bool value)
{
sPreferCache = value;
}
bool
ServerSettings::DropCache()
{
return sDropCache;
}
void
ServerSettings::SetDropCache(bool value)
{
sDropCache = value;
}
bool
ServerSettings::ForceNoNetwork()
{
return sForceNoNetwork;
}
void
ServerSettings::SetForceNoNetwork(bool value)
{
sForceNoNetwork = value;
}

10
src/apps/haikudepot/server/ServerSettings.h
View File

@ -20,6 +20,13 @@ public:
static BUrl CreateFullUrl( static BUrl CreateFullUrl(
const BString urlPathComponents); const BString urlPathComponents);
static bool PreferCache();
static void SetPreferCache(bool value);
static bool DropCache();
static void SetDropCache(bool value);
static bool ForceNoNetwork();
static void SetForceNoNetwork(bool value);
private: private:
static void _InitUserAgent(); static void _InitUserAgent();
static const BString _GetUserAgentVersionString(); static const BString _GetUserAgentVersionString();
@ -27,6 +34,9 @@ private:
static BUrl sBaseUrl; static BUrl sBaseUrl;
static BString sUserAgent; static BString sUserAgent;
static pthread_once_t sUserAgentInitOnce; static pthread_once_t sUserAgentInitOnce;
static bool sPreferCache;
static bool sDropCache;
static bool sForceNoNetwork;
}; };
#endif // SERVER_SETTINGS_H #endif // SERVER_SETTINGS_H

2
src/apps/haikudepot/server/dumpexportpkg/DumpExportPkg.cpp
View File

@ -1,7 +1,7 @@
/* /*
* Generated Model Object * Generated Model Object
* source json-schema : dumpexport.json * source json-schema : dumpexport.json
* generated at : 2017-11-05T22:30:10.254264 * generated at : 2017-12-07T23:22:17.116794
*/ */
#include "DumpExportPkg.h" #include "DumpExportPkg.h"

2
src/apps/haikudepot/server/dumpexportpkg/DumpExportPkg.h
View File

@ -1,7 +1,7 @@
/* /*
* Generated Model Object * Generated Model Object
* source json-schema : dumpexport.json * source json-schema : dumpexport.json
* generated at : 2017-11-05T22:30:10.253178 * generated at : 2017-12-07T23:22:17.115160
*/ */
#ifndef GEN_JSON_SCHEMA_MODEL__DUMPEXPORTPKG_H #ifndef GEN_JSON_SCHEMA_MODEL__DUMPEXPORTPKG_H

2
src/apps/haikudepot/server/dumpexportpkg/DumpExportPkgCategory.cpp
View File

@ -1,7 +1,7 @@
/* /*
* Generated Model Object * Generated Model Object
* source json-schema : dumpexport.json * source json-schema : dumpexport.json
* generated at : 2017-11-05T22:30:10.260503 * generated at : 2017-12-07T23:22:17.118616
*/ */
#include "DumpExportPkgCategory.h" #include "DumpExportPkgCategory.h"

2
src/apps/haikudepot/server/dumpexportpkg/DumpExportPkgCategory.h
View File

@ -1,7 +1,7 @@
/* /*
* Generated Model Object * Generated Model Object
* source json-schema : dumpexport.json * source json-schema : dumpexport.json
* generated at : 2017-11-05T22:30:10.259170 * generated at : 2017-12-07T23:22:17.118521
*/ */
#ifndef GEN_JSON_SCHEMA_MODEL__DUMPEXPORTPKGCATEGORY_H #ifndef GEN_JSON_SCHEMA_MODEL__DUMPEXPORTPKGCATEGORY_H

63
src/apps/haikudepot/server/dumpexportpkg/DumpExportPkgJsonListener.cpp
View File

@ -1,7 +1,7 @@
/* /*
* Generated Listener Object * Generated Listener Object
* source json-schema : dumpexport.json * source json-schema : dumpexport.json
* generated at : 2017-11-11T14:09:03.498648 * generated at : 2017-12-17T20:45:25.516772
*/ */
#include "DumpExportPkgJsonListener.h" #include "DumpExportPkgJsonListener.h"
#include "List.h" #include "List.h"
@ -29,12 +29,12 @@ public:
AbstractStackedDumpExportPkgJsonListener* Parent(); AbstractStackedDumpExportPkgJsonListener* Parent();
virtual void WillPop(); virtual bool WillPop();
protected: protected:
AbstractMainDumpExportPkgJsonListener* fMainListener; AbstractMainDumpExportPkgJsonListener* fMainListener;
void Pop(); bool Pop();
void Push(AbstractStackedDumpExportPkgJsonListener* stackedListener); void Push(AbstractStackedDumpExportPkgJsonListener* stackedListener);
private: private:
@ -193,7 +193,7 @@ public:
DumpExportPkgListener* itemListener); DumpExportPkgListener* itemListener);
~ItemEmittingStackedDumpExportPkgJsonListener(); ~ItemEmittingStackedDumpExportPkgJsonListener();
void WillPop(); bool WillPop();
private: private:
DumpExportPkgListener* fItemListener; DumpExportPkgListener* fItemListener;
@ -225,7 +225,7 @@ public:
~BulkContainerItemsStackedDumpExportPkgJsonListener(); ~BulkContainerItemsStackedDumpExportPkgJsonListener();
bool Handle(const BJsonEvent& event); bool Handle(const BJsonEvent& event);
void WillPop(); bool WillPop();
private: private:
DumpExportPkgListener* fItemListener; DumpExportPkgListener* fItemListener;
@ -275,16 +275,18 @@ AbstractStackedDumpExportPkgJsonListener::Push(AbstractStackedDumpExportPkgJsonL
fMainListener->SetStackedListener(stackedListener); fMainListener->SetStackedListener(stackedListener);
} }
void bool
AbstractStackedDumpExportPkgJsonListener::WillPop() AbstractStackedDumpExportPkgJsonListener::WillPop()
{ {
return true;
} }
void bool
AbstractStackedDumpExportPkgJsonListener::Pop() AbstractStackedDumpExportPkgJsonListener::Pop()
{ {
WillPop(); bool result = WillPop();
fMainListener->SetStackedListener(fParent); fMainListener->SetStackedListener(fParent);
return result;
} }
GeneralObjectStackedDumpExportPkgJsonListener::GeneralObjectStackedDumpExportPkgJsonListener( GeneralObjectStackedDumpExportPkgJsonListener::GeneralObjectStackedDumpExportPkgJsonListener(
@ -327,8 +329,7 @@ GeneralObjectStackedDumpExportPkgJsonListener::Handle(const BJsonEvent& event)
case B_JSON_OBJECT_END: case B_JSON_OBJECT_END:
{ {
Pop(); bool status = Pop() && (ErrorStatus() == B_OK);
bool status = (ErrorStatus() == B_OK);
delete this; delete this;
return status; return status;
} }
@ -378,8 +379,7 @@ GeneralArrayStackedDumpExportPkgJsonListener::Handle(const BJsonEvent& event)
case B_JSON_ARRAY_END: case B_JSON_ARRAY_END:
{ {
Pop(); bool status = Pop() && (ErrorStatus() == B_OK);
bool status = (ErrorStatus() == B_OK);
delete this; delete this;
return status; return status;
} }
@ -426,8 +426,7 @@ DumpExportPkg_StackedDumpExportPkgJsonListener::Handle(const BJsonEvent& event)
case B_JSON_OBJECT_END: case B_JSON_OBJECT_END:
{ {
Pop(); bool status = Pop() && (ErrorStatus() == B_OK);
bool status = (ErrorStatus() == B_OK);
delete this; delete this;
return status; return status;
} }
@ -549,8 +548,7 @@ DumpExportPkg_List_StackedDumpExportPkgJsonListener::Handle(const BJsonEvent& ev
case B_JSON_ARRAY_END: case B_JSON_ARRAY_END:
{ {
Pop(); bool status = Pop() && (ErrorStatus() == B_OK);
bool status = (ErrorStatus() == B_OK);
delete this; delete this;
return status; return status;
} }
@ -610,8 +608,7 @@ DumpExportPkgVersion_StackedDumpExportPkgJsonListener::Handle(const BJsonEvent&
case B_JSON_OBJECT_END: case B_JSON_OBJECT_END:
{ {
Pop(); bool status = Pop() && (ErrorStatus() == B_OK);
bool status = (ErrorStatus() == B_OK);
delete this; delete this;
return status; return status;
} }
@ -748,8 +745,7 @@ DumpExportPkgVersion_List_StackedDumpExportPkgJsonListener::Handle(const BJsonEv
case B_JSON_ARRAY_END: case B_JSON_ARRAY_END:
{ {
Pop(); bool status = Pop() && (ErrorStatus() == B_OK);
bool status = (ErrorStatus() == B_OK);
delete this; delete this;
return status; return status;
} }
@ -809,8 +805,7 @@ DumpExportPkgScreenshot_StackedDumpExportPkgJsonListener::Handle(const BJsonEven
case B_JSON_OBJECT_END: case B_JSON_OBJECT_END:
{ {
Pop(); bool status = Pop() && (ErrorStatus() == B_OK);
bool status = (ErrorStatus() == B_OK);
delete this; delete this;
return status; return status;
} }
@ -917,8 +912,7 @@ DumpExportPkgScreenshot_List_StackedDumpExportPkgJsonListener::Handle(const BJso
case B_JSON_ARRAY_END: case B_JSON_ARRAY_END:
{ {
Pop(); bool status = Pop() && (ErrorStatus() == B_OK);
bool status = (ErrorStatus() == B_OK);
delete this; delete this;
return status; return status;
} }
@ -978,8 +972,7 @@ DumpExportPkgCategory_StackedDumpExportPkgJsonListener::Handle(const BJsonEvent&
case B_JSON_OBJECT_END: case B_JSON_OBJECT_END:
{ {
Pop(); bool status = Pop() && (ErrorStatus() == B_OK);
bool status = (ErrorStatus() == B_OK);
delete this; delete this;
return status; return status;
} }
@ -1062,8 +1055,7 @@ DumpExportPkgCategory_List_StackedDumpExportPkgJsonListener::Handle(const BJsonE
case B_JSON_ARRAY_END: case B_JSON_ARRAY_END:
{ {
Pop(); bool status = Pop() && (ErrorStatus() == B_OK);
bool status = (ErrorStatus() == B_OK);
delete this; delete this;
return status; return status;
} }
@ -1101,11 +1093,13 @@ ItemEmittingStackedDumpExportPkgJsonListener::~ItemEmittingStackedDumpExportPkgJ
} }
void bool
ItemEmittingStackedDumpExportPkgJsonListener::WillPop() ItemEmittingStackedDumpExportPkgJsonListener::WillPop()
{ {
fItemListener->Handle(fTarget); bool result = fItemListener->Handle(fTarget);
delete fTarget; delete fTarget;
fTarget = NULL;
return result;
} }
@ -1150,8 +1144,7 @@ BulkContainerStackedDumpExportPkgJsonListener::Handle(const BJsonEvent& event)
case B_JSON_OBJECT_END: case B_JSON_OBJECT_END:
{ {
Pop(); bool status = Pop() && (ErrorStatus() == B_OK);
bool status = (ErrorStatus() == B_OK);
delete this; delete this;
return status; return status;
} }
@ -1191,8 +1184,7 @@ BulkContainerItemsStackedDumpExportPkgJsonListener::Handle(const BJsonEvent& eve
case B_JSON_ARRAY_END: case B_JSON_ARRAY_END:
{ {
Pop(); bool status = Pop() && (ErrorStatus() == B_OK);
bool status = (ErrorStatus() == B_OK);
delete this; delete this;
return status; return status;
} }
@ -1206,10 +1198,11 @@ BulkContainerItemsStackedDumpExportPkgJsonListener::Handle(const BJsonEvent& eve
} }
void bool
BulkContainerItemsStackedDumpExportPkgJsonListener::WillPop() BulkContainerItemsStackedDumpExportPkgJsonListener::WillPop()
{ {
fItemListener->Complete(); fItemListener->Complete();
return true;
} }

4
src/apps/haikudepot/server/dumpexportpkg/DumpExportPkgJsonListener.h
View File

@ -1,7 +1,7 @@
/* /*
* Generated Listener Object * Generated Listener Object
* source json-schema : dumpexport.json * source json-schema : dumpexport.json
* generated at : 2017-11-11T14:09:03.498089 * generated at : 2017-12-17T20:45:25.514143
*/ */
#ifndef GEN_JSON_SCHEMA_PARSER__SINGLEDUMPEXPORTPKGJSONLISTENER_H #ifndef GEN_JSON_SCHEMA_PARSER__SINGLEDUMPEXPORTPKGJSONLISTENER_H
@ -60,7 +60,7 @@ private:
*/ */
class DumpExportPkgListener { class DumpExportPkgListener {
public: public:
virtual void Handle(DumpExportPkg* item) = 0; virtual bool Handle(DumpExportPkg* item) = 0;
virtual void Complete() = 0; virtual void Complete() = 0;
}; };

2
src/apps/haikudepot/server/dumpexportpkg/DumpExportPkgScreenshot.cpp
View File

@ -1,7 +1,7 @@
/* /*
* Generated Model Object * Generated Model Object
* source json-schema : dumpexport.json * source json-schema : dumpexport.json
* generated at : 2017-11-05T22:30:10.257444 * generated at : 2017-12-07T23:22:17.118221
*/ */
#include "DumpExportPkgScreenshot.h" #include "DumpExportPkgScreenshot.h"

2
src/apps/haikudepot/server/dumpexportpkg/DumpExportPkgScreenshot.h
View File

@ -1,7 +1,7 @@
/* /*
* Generated Model Object * Generated Model Object
* source json-schema : dumpexport.json * source json-schema : dumpexport.json
* generated at : 2017-11-05T22:30:10.256842 * generated at : 2017-12-07T23:22:17.118049
*/ */
#ifndef GEN_JSON_SCHEMA_MODEL__DUMPEXPORTPKGSCREENSHOT_H #ifndef GEN_JSON_SCHEMA_MODEL__DUMPEXPORTPKGSCREENSHOT_H

2
src/apps/haikudepot/server/dumpexportpkg/DumpExportPkgVersion.cpp
View File

@ -1,7 +1,7 @@
/* /*
* Generated Model Object * Generated Model Object
* source json-schema : dumpexport.json * source json-schema : dumpexport.json
* generated at : 2017-11-05T22:30:10.255929 * generated at : 2017-12-07T23:22:17.117543
*/ */
#include "DumpExportPkgVersion.h" #include "DumpExportPkgVersion.h"

2
src/apps/haikudepot/server/dumpexportpkg/DumpExportPkgVersion.h
View File

@ -1,7 +1,7 @@
/* /*
* Generated Model Object * Generated Model Object
* source json-schema : dumpexport.json * source json-schema : dumpexport.json
* generated at : 2017-11-05T22:30:10.255268 * generated at : 2017-12-07T23:22:17.117333
*/ */
#ifndef GEN_JSON_SCHEMA_MODEL__DUMPEXPORTPKGVERSION_H #ifndef GEN_JSON_SCHEMA_MODEL__DUMPEXPORTPKGVERSION_H

2
src/apps/haikudepot/server/dumpexportrepository/DumpExportRepository.cpp
View File

@ -1,7 +1,7 @@
/* /*
* Generated Model Object * Generated Model Object
* source json-schema : dumpexport.json * source json-schema : dumpexport.json
* generated at : 2017-11-11T13:59:22.553529 * generated at : 2017-12-07T23:22:33.021497
*/ */
#include "DumpExportRepository.h" #include "DumpExportRepository.h"

2
src/apps/haikudepot/server/dumpexportrepository/DumpExportRepository.h
View File

@ -1,7 +1,7 @@
/* /*
* Generated Model Object * Generated Model Object
* source json-schema : dumpexport.json * source json-schema : dumpexport.json
* generated at : 2017-11-11T13:59:22.550365 * generated at : 2017-12-07T23:22:33.020747
*/ */
#ifndef GEN_JSON_SCHEMA_MODEL__DUMPEXPORTREPOSITORY_H #ifndef GEN_JSON_SCHEMA_MODEL__DUMPEXPORTREPOSITORY_H

61
src/apps/haikudepot/server/dumpexportrepository/DumpExportRepositoryJsonListener.cpp
View File

@ -1,10 +1,11 @@
/* /*
* Generated Listener Object * Generated Listener Object
* source json-schema : dumpexport.json * source json-schema : dumpexport.json
* generated at : 2017-11-11T14:08:39.276778 * generated at : 2017-12-18T23:07:02.401765
*/ */
#include "DumpExportRepositoryJsonListener.h" #include "DumpExportRepositoryJsonListener.h"
#include "List.h" #include "List.h"
#include <stdio.h> #include <stdio.h>
// #pragma mark - private interfaces for the stacked listeners // #pragma mark - private interfaces for the stacked listeners
@ -28,12 +29,12 @@ public:
AbstractStackedDumpExportRepositoryJsonListener* Parent(); AbstractStackedDumpExportRepositoryJsonListener* Parent();
virtual void WillPop(); virtual bool WillPop();
protected: protected:
AbstractMainDumpExportRepositoryJsonListener* fMainListener; AbstractMainDumpExportRepositoryJsonListener* fMainListener;
void Pop(); bool Pop();
void Push(AbstractStackedDumpExportRepositoryJsonListener* stackedListener); void Push(AbstractStackedDumpExportRepositoryJsonListener* stackedListener);
private: private:
@ -130,7 +131,7 @@ public:
DumpExportRepositoryListener* itemListener); DumpExportRepositoryListener* itemListener);
~ItemEmittingStackedDumpExportRepositoryJsonListener(); ~ItemEmittingStackedDumpExportRepositoryJsonListener();
void WillPop(); bool WillPop();
private: private:
DumpExportRepositoryListener* fItemListener; DumpExportRepositoryListener* fItemListener;
@ -162,7 +163,7 @@ public:
~BulkContainerItemsStackedDumpExportRepositoryJsonListener(); ~BulkContainerItemsStackedDumpExportRepositoryJsonListener();
bool Handle(const BJsonEvent& event); bool Handle(const BJsonEvent& event);
void WillPop(); bool WillPop();
private: private:
DumpExportRepositoryListener* fItemListener; DumpExportRepositoryListener* fItemListener;
@ -212,16 +213,18 @@ AbstractStackedDumpExportRepositoryJsonListener::Push(AbstractStackedDumpExportR
fMainListener->SetStackedListener(stackedListener); fMainListener->SetStackedListener(stackedListener);
} }
void bool
AbstractStackedDumpExportRepositoryJsonListener::WillPop() AbstractStackedDumpExportRepositoryJsonListener::WillPop()
{ {
return true;
} }
void bool
AbstractStackedDumpExportRepositoryJsonListener::Pop() AbstractStackedDumpExportRepositoryJsonListener::Pop()
{ {
WillPop(); bool result = WillPop();
fMainListener->SetStackedListener(fParent); fMainListener->SetStackedListener(fParent);
return result;
} }
GeneralObjectStackedDumpExportRepositoryJsonListener::GeneralObjectStackedDumpExportRepositoryJsonListener( GeneralObjectStackedDumpExportRepositoryJsonListener::GeneralObjectStackedDumpExportRepositoryJsonListener(
@ -264,11 +267,11 @@ GeneralObjectStackedDumpExportRepositoryJsonListener::Handle(const BJsonEvent& e
case B_JSON_OBJECT_END: case B_JSON_OBJECT_END:
{ {
Pop(); bool status = Pop() && (ErrorStatus() == B_OK);
bool status = (ErrorStatus() == B_OK);
delete this; delete this;
return status; return status;
} }
} }
return ErrorStatus() == B_OK; return ErrorStatus() == B_OK;
@ -314,11 +317,11 @@ GeneralArrayStackedDumpExportRepositoryJsonListener::Handle(const BJsonEvent& ev
case B_JSON_ARRAY_END: case B_JSON_ARRAY_END:
{ {
Pop(); bool status = Pop() && (ErrorStatus() == B_OK);
bool status = (ErrorStatus() == B_OK);
delete this; delete this;
return status; return status;
} }
} }
return ErrorStatus() == B_OK; return ErrorStatus() == B_OK;
@ -361,8 +364,7 @@ DumpExportRepository_StackedDumpExportRepositoryJsonListener::Handle(const BJson
case B_JSON_OBJECT_END: case B_JSON_OBJECT_END:
{ {
Pop(); bool status = Pop() && (ErrorStatus() == B_OK);
bool status = (ErrorStatus() == B_OK);
delete this; delete this;
return status; return status;
} }
@ -468,8 +470,7 @@ DumpExportRepository_List_StackedDumpExportRepositoryJsonListener::Handle(const
case B_JSON_ARRAY_END: case B_JSON_ARRAY_END:
{ {
Pop(); bool status = Pop() && (ErrorStatus() == B_OK);
bool status = (ErrorStatus() == B_OK);
delete this; delete this;
return status; return status;
} }
@ -529,8 +530,7 @@ DumpExportRepositorySource_StackedDumpExportRepositoryJsonListener::Handle(const
case B_JSON_OBJECT_END: case B_JSON_OBJECT_END:
{ {
Pop(); bool status = Pop() && (ErrorStatus() == B_OK);
bool status = (ErrorStatus() == B_OK);
delete this; delete this;
return status; return status;
} }
@ -540,6 +540,9 @@ DumpExportRepositorySource_StackedDumpExportRepositoryJsonListener::Handle(const
if (fNextItemName == "url") if (fNextItemName == "url")
fTarget->SetUrl(new BString(event.Content())); fTarget->SetUrl(new BString(event.Content()));
if (fNextItemName == "repoInfoUrl")
fTarget->SetRepoInfoUrl(new BString(event.Content()));
if (fNextItemName == "code") if (fNextItemName == "code")
fTarget->SetCode(new BString(event.Content())); fTarget->SetCode(new BString(event.Content()));
fNextItemName.SetTo(""); fNextItemName.SetTo("");
@ -556,6 +559,9 @@ DumpExportRepositorySource_StackedDumpExportRepositoryJsonListener::Handle(const
if (fNextItemName == "url") if (fNextItemName == "url")
fTarget->SetUrlNull(); fTarget->SetUrlNull();
if (fNextItemName == "repoInfoUrl")
fTarget->SetRepoInfoUrlNull();
if (fNextItemName == "code") if (fNextItemName == "code")
fTarget->SetCodeNull(); fTarget->SetCodeNull();
fNextItemName.SetTo(""); fNextItemName.SetTo("");
@ -619,8 +625,7 @@ DumpExportRepositorySource_List_StackedDumpExportRepositoryJsonListener::Handle(
case B_JSON_ARRAY_END: case B_JSON_ARRAY_END:
{ {
Pop(); bool status = Pop() && (ErrorStatus() == B_OK);
bool status = (ErrorStatus() == B_OK);
delete this; delete this;
return status; return status;
} }
@ -658,11 +663,13 @@ ItemEmittingStackedDumpExportRepositoryJsonListener::~ItemEmittingStackedDumpExp
} }
void bool
ItemEmittingStackedDumpExportRepositoryJsonListener::WillPop() ItemEmittingStackedDumpExportRepositoryJsonListener::WillPop()
{ {
fItemListener->Handle(fTarget); bool result = fItemListener->Handle(fTarget);
delete fTarget; delete fTarget;
fTarget = NULL;
return result;
} }
@ -707,8 +714,7 @@ BulkContainerStackedDumpExportRepositoryJsonListener::Handle(const BJsonEvent& e
case B_JSON_OBJECT_END: case B_JSON_OBJECT_END:
{ {
Pop(); bool status = Pop() && (ErrorStatus() == B_OK);
bool status = (ErrorStatus() == B_OK);
delete this; delete this;
return status; return status;
} }
@ -748,8 +754,7 @@ BulkContainerItemsStackedDumpExportRepositoryJsonListener::Handle(const BJsonEve
case B_JSON_ARRAY_END: case B_JSON_ARRAY_END:
{ {
Pop(); bool status = Pop() && (ErrorStatus() == B_OK);
bool status = (ErrorStatus() == B_OK);
delete this; delete this;
return status; return status;
} }
@ -763,10 +768,11 @@ BulkContainerItemsStackedDumpExportRepositoryJsonListener::Handle(const BJsonEve
} }
void bool
BulkContainerItemsStackedDumpExportRepositoryJsonListener::WillPop() BulkContainerItemsStackedDumpExportRepositoryJsonListener::WillPop()
{ {
fItemListener->Complete(); fItemListener->Complete();
return true;
} }
@ -895,7 +901,6 @@ BulkContainerDumpExportRepositoryJsonListener::Handle(const BJsonEvent& event)
BulkContainerStackedDumpExportRepositoryJsonListener* nextListener = BulkContainerStackedDumpExportRepositoryJsonListener* nextListener =
new BulkContainerStackedDumpExportRepositoryJsonListener( new BulkContainerStackedDumpExportRepositoryJsonListener(
this, NULL, fItemListener); this, NULL, fItemListener);
SetStackedListener(nextListener); SetStackedListener(nextListener);
return true; return true;
break; break;

4
src/apps/haikudepot/server/dumpexportrepository/DumpExportRepositoryJsonListener.h
View File

@ -1,7 +1,7 @@
/* /*
* Generated Listener Object * Generated Listener Object
* source json-schema : dumpexport.json * source json-schema : dumpexport.json
* generated at : 2017-11-11T14:08:39.274913 * generated at : 2017-12-18T23:07:02.399681
*/ */
#ifndef GEN_JSON_SCHEMA_PARSER__SINGLEDUMPEXPORTREPOSITORYJSONLISTENER_H #ifndef GEN_JSON_SCHEMA_PARSER__SINGLEDUMPEXPORTREPOSITORYJSONLISTENER_H
@ -60,7 +60,7 @@ private:
*/ */
class DumpExportRepositoryListener { class DumpExportRepositoryListener {
public: public:
virtual void Handle(DumpExportRepository* item) = 0; virtual bool Handle(DumpExportRepository* item) = 0;
virtual void Complete() = 0; virtual void Complete() = 0;
}; };

38
src/apps/haikudepot/server/dumpexportrepository/DumpExportRepositorySource.cpp
View File

@ -1,7 +1,7 @@
/* /*
* Generated Model Object * Generated Model Object
* source json-schema : dumpexport.json * source json-schema : dumpexport.json
* generated at : 2017-11-11T13:59:22.559632 * generated at : 2017-12-07T23:22:33.022114
*/ */
#include "DumpExportRepositorySource.h" #include "DumpExportRepositorySource.h"
@ -9,6 +9,7 @@
DumpExportRepositorySource::DumpExportRepositorySource() DumpExportRepositorySource::DumpExportRepositorySource()
{ {
fUrl = NULL; fUrl = NULL;
fRepoInfoUrl = NULL;
fCode = NULL; fCode = NULL;
} }
@ -19,6 +20,10 @@ DumpExportRepositorySource::~DumpExportRepositorySource()
delete fUrl; delete fUrl;
} }
if (fRepoInfoUrl != NULL) {
delete fRepoInfoUrl;
}
if (fCode != NULL) { if (fCode != NULL) {
delete fCode; delete fCode;
} }
@ -56,6 +61,37 @@ DumpExportRepositorySource::UrlIsNull()
} }
BString*
DumpExportRepositorySource::RepoInfoUrl()
{
return fRepoInfoUrl;
}
void
DumpExportRepositorySource::SetRepoInfoUrl(BString* value)
{
fRepoInfoUrl = value;
}
void
DumpExportRepositorySource::SetRepoInfoUrlNull()
{
if (!RepoInfoUrlIsNull()) {
delete fRepoInfoUrl;
fRepoInfoUrl = NULL;
}
}
bool
DumpExportRepositorySource::RepoInfoUrlIsNull()
{
return fRepoInfoUrl == NULL;
}
BString* BString*
DumpExportRepositorySource::Code() DumpExportRepositorySource::Code()
{ {

8
src/apps/haikudepot/server/dumpexportrepository/DumpExportRepositorySource.h
View File

@ -1,7 +1,7 @@
/* /*
* Generated Model Object * Generated Model Object
* source json-schema : dumpexport.json * source json-schema : dumpexport.json
* generated at : 2017-11-11T13:59:22.559237 * generated at : 2017-12-07T23:22:33.021952
*/ */
#ifndef GEN_JSON_SCHEMA_MODEL__DUMPEXPORTREPOSITORYSOURCE_H #ifndef GEN_JSON_SCHEMA_MODEL__DUMPEXPORTREPOSITORYSOURCE_H
@ -21,6 +21,11 @@ public:
void SetUrlNull(); void SetUrlNull();
bool UrlIsNull(); bool UrlIsNull();
BString* RepoInfoUrl();
void SetRepoInfoUrl(BString* value);
void SetRepoInfoUrlNull();
bool RepoInfoUrlIsNull();
BString* Code(); BString* Code();
void SetCode(BString* value); void SetCode(BString* value);
void SetCodeNull(); void SetCodeNull();
@ -28,6 +33,7 @@ public:
private: private:
BString* fUrl; BString* fUrl;
BString* fRepoInfoUrl;
BString* fCode; BString* fCode;
}; };

8
src/apps/haikudepot/tar/TarArchiveService.cpp
View File

@ -19,7 +19,8 @@
status_t status_t
TarArchiveService::Unpack(BDataIO& tarDataIo, BPath& targetDirectory) TarArchiveService::Unpack(BDataIO& tarDataIo, BPath& targetDirectory,
Stoppable* stoppable)
{ {
uint8 buffer[LENGTH_BLOCK]; uint8 buffer[LENGTH_BLOCK];
uint8 zero_buffer[LENGTH_BLOCK]; uint8 zero_buffer[LENGTH_BLOCK];
@ -30,8 +31,9 @@ TarArchiveService::Unpack(BDataIO& tarDataIo, BPath& targetDirectory)
memset(zero_buffer, 0, sizeof zero_buffer); memset(zero_buffer, 0, sizeof zero_buffer);
while (B_OK == result && B_OK == (result = tarDataIo.ReadExactly(buffer, while (B_OK == result
LENGTH_BLOCK))) { && (NULL == stoppable || !stoppable->WasStopped())
&& B_OK == (result = tarDataIo.ReadExactly(buffer, LENGTH_BLOCK))) {
count_items_read++; count_items_read++;

4
src/apps/haikudepot/tar/TarArchiveService.h
View File

@ -7,6 +7,7 @@
#define TAR_ARCHIVE_SERVICE_H #define TAR_ARCHIVE_SERVICE_H
#include "AbstractServerProcess.h" #include "AbstractServerProcess.h"
#include "Stoppable.h"
#include "TarArchiveHeader.h" #include "TarArchiveHeader.h"
#include <String.h> #include <String.h>
@ -16,7 +17,8 @@
class TarArchiveService { class TarArchiveService {
public: public:
static status_t Unpack(BDataIO& tarDataIo, static status_t Unpack(BDataIO& tarDataIo,
BPath& targetDirectoryPath); BPath& targetDirectoryPath,
Stoppable* stoppable);
private: private:
static status_t _EnsurePathToTarItemFile( static status_t _EnsurePathToTarItemFile(

46
src/apps/haikudepot/ui/App.cpp
View File

@ -1,5 +1,6 @@
/* /*
* Copyright 2013, Stephan Aßmus <superstippi@gmx.de>. * Copyright 2013, Stephan Aßmus <superstippi@gmx.de>.
* Copyright 2017, Andrew Lindesay <apl@lindesay.co.nz>.
* All rights reserved. Distributed under the terms of the MIT License. * All rights reserved. Distributed under the terms of the MIT License.
*/ */
@ -127,6 +128,9 @@ enum arg_switch {
HELP_SWITCH, HELP_SWITCH,
WEB_APP_BASE_URL_SWITCH, WEB_APP_BASE_URL_SWITCH,
VERBOSITY_SWITCH, VERBOSITY_SWITCH,
FORCE_NO_NETWORKING_SWITCH,
PREFER_CACHE_SWITCH,
DROP_CACHE_SWITCH
}; };
@ -134,13 +138,24 @@ static void
app_print_help() app_print_help()
{ {
fprintf(stdout, "HaikuDepot "); fprintf(stdout, "HaikuDepot ");
fprintf(stdout, "[-u|--webappbaseurl <web-app-base-url>] "); fprintf(stdout, "[-u|--webappbaseurl <web-app-base-url>]\n");
fprintf(stdout, "[-v|--verbosity [off|info|debug|trace] "); fprintf(stdout, "[-v|--verbosity [off|info|debug|trace]\n");
fprintf(stdout, "[-h|--help]\n\n"); fprintf(stdout, "[--nonetworking]\n");
fprintf(stdout, "[--prefercache]\n");
fprintf(stdout, "[--dropcache]\n");
fprintf(stdout, "[-h|--help]\n");
fprintf(stdout, "\n");
fprintf(stdout, "'-h' : causes this help text to be printed out.\n"); fprintf(stdout, "'-h' : causes this help text to be printed out.\n");
fprintf(stdout, "'-v' : allows for the verbosity level to be set.\n"); fprintf(stdout, "'-v' : allows for the verbosity level to be set.\n");
fprintf(stdout, "'-u' : allows for the haiku depot server to be\n"); fprintf(stdout, "'-u' : allows for the haiku depot server url to be\n");
fprintf(stdout, " configured."); fprintf(stdout, " configured.\n");
fprintf(stdout, "'--nonetworking' : prevents network access.**\n");
fprintf(stdout, "'--prefercache' : prefer to get data from cache rather\n");
fprintf(stdout, " then obtain data from the network.**\n");
fprintf(stdout, "'--dropcache' : drop cached data before performing\n");
fprintf(stdout, " bulk operations.**\n");
fprintf(stdout, "\n");
fprintf(stdout, "** = only applies to bulk operations.\n");
} }
@ -160,6 +175,15 @@ app_resolve_switch(char *arg)
if (0 == strcmp(&arg[2], "verbosity")) if (0 == strcmp(&arg[2], "verbosity"))
return VERBOSITY_SWITCH; return VERBOSITY_SWITCH;
if (0 == strcmp(&arg[2], "nonetworking"))
return FORCE_NO_NETWORKING_SWITCH;
if (0 == strcmp(&arg[2], "prefercache"))
return PREFER_CACHE_SWITCH;
if (0 == strcmp(&arg[2], "dropcache"))
return DROP_CACHE_SWITCH;
} else { } else {
if (arglen == 2) { // short form if (arglen == 2) { // short form
switch (arg[1]) { switch (arg[1]) {
@ -237,6 +261,18 @@ App::ArgvReceived(int32 argc, char* argv[])
break; break;
case FORCE_NO_NETWORKING_SWITCH:
ServerSettings::SetForceNoNetwork(true);
break;
case PREFER_CACHE_SWITCH:
ServerSettings::SetPreferCache(true);
break;
case DROP_CACHE_SWITCH:
ServerSettings::SetDropCache(true);
break;
case NOT_SWITCH: case NOT_SWITCH:
{ {
BEntry entry(argv[i], true); BEntry entry(argv[i], true);

11
src/apps/haikudepot/ui/MainWindow.cpp
View File

@ -132,6 +132,7 @@ MainWindow::MainWindow(const BMessage& settings)
fLogInItem(NULL), fLogInItem(NULL),
fLogOutItem(NULL), fLogOutItem(NULL),
fModelListener(new MessageModelListener(BMessenger(this)), true), fModelListener(new MessageModelListener(BMessenger(this)), true),
fBulkLoadStateMachine(&fModel),
fTerminating(false), fTerminating(false),
fSinglePackageMode(false), fSinglePackageMode(false),
fModelWorker(B_BAD_THREAD_ID) fModelWorker(B_BAD_THREAD_ID)
@ -245,6 +246,7 @@ MainWindow::MainWindow(const BMessage& settings, const PackageInfoRef& package)
fLogInItem(NULL), fLogInItem(NULL),
fLogOutItem(NULL), fLogOutItem(NULL),
fModelListener(new MessageModelListener(BMessenger(this)), true), fModelListener(new MessageModelListener(BMessenger(this)), true),
fBulkLoadStateMachine(&fModel),
fTerminating(false), fTerminating(false),
fSinglePackageMode(true), fSinglePackageMode(true),
fModelWorker(B_BAD_THREAD_ID) fModelWorker(B_BAD_THREAD_ID)
@ -1062,7 +1064,7 @@ MainWindow::_RefreshPackageList(bool force)
bool wasEmpty = fModel.Depots().IsEmpty(); bool wasEmpty = fModel.Depots().IsEmpty();
if (force || wasEmpty) if (force || wasEmpty)
fModel.StopPopulatingAllPackages(); fBulkLoadStateMachine.Stop();
BAutolock lock(fModel.Lock()); BAutolock lock(fModel.Lock());
@ -1094,11 +1096,10 @@ MainWindow::_RefreshPackageList(bool force)
fModel.AddDepot(it->second); fModel.AddDepot(it->second);
} }
fModel.PopulateWebAppRepositoryCodes();
// start retrieving package icons and average ratings // start retrieving package icons and average ratings
if (force || wasEmpty) if (force || wasEmpty) {
fModel.PopulateAllPackages(); fBulkLoadStateMachine.Start();
}
// compute the OS package dependencies // compute the OS package dependencies
try { try {

4
src/apps/haikudepot/ui/MainWindow.h
View File

@ -2,6 +2,7 @@
* Copyright 2013-2014, Stephan Aßmus <superstippi@gmx.de>. * Copyright 2013-2014, Stephan Aßmus <superstippi@gmx.de>.
* Copyright 2013, Rene Gollent <rene@gollent.com>. * Copyright 2013, Rene Gollent <rene@gollent.com>.
* Copyright 2017, Julian Harnath <julian.harnath@rwth-aachen.de>. * Copyright 2017, Julian Harnath <julian.harnath@rwth-aachen.de>.
* Copyright 2017, Andrew Lindesay <apl@lindesay.co.nz>.
* All rights reserved. Distributed under the terms of the MIT License. * All rights reserved. Distributed under the terms of the MIT License.
*/ */
#ifndef MAIN_WINDOW_H #ifndef MAIN_WINDOW_H
@ -9,6 +10,7 @@
#include <Window.h> #include <Window.h>
#include "BulkLoadStateMachine.h"
#include "Model.h" #include "Model.h"
#include "PackageAction.h" #include "PackageAction.h"
#include "PackageActionHandler.h" #include "PackageActionHandler.h"
@ -119,6 +121,8 @@ private:
Model fModel; Model fModel;
ModelListenerRef fModelListener; ModelListenerRef fModelListener;
PackageList fVisiblePackages; PackageList fVisiblePackages;
BulkLoadStateMachine
fBulkLoadStateMachine;
bool fTerminating; bool fTerminating;
bool fSinglePackageMode; bool fSinglePackageMode;

45
src/apps/haikudepot/util/StorageUtils.cpp
View File

@ -5,13 +5,15 @@
#include "StorageUtils.h" #include "StorageUtils.h"
#include <stdio.h>
#include <errno.h>
#include <Directory.h> #include <Directory.h>
#include <File.h> #include <File.h>
#include <Entry.h> #include <Entry.h>
#include <String.h> #include <String.h>
#include <stdio.h> #include "Logger.h"
#include <errno.h>
#define FILE_TO_STRING_BUFFER_LEN 64 #define FILE_TO_STRING_BUFFER_LEN 64
@ -51,23 +53,27 @@ StorageUtils::RemoveDirectoryContents(BPath& path)
bool exists = false; bool exists = false;
bool isDirectory = false; bool isDirectory = false;
BPath directroyEntryPath; BPath directoryEntryPath;
result = directoryEntry.GetPath(&directroyEntryPath); result = directoryEntry.GetPath(&directoryEntryPath);
if (result == B_OK) if (result == B_OK) {
result = ExistsDirectory(directroyEntryPath, &exists, &isDirectory); result = ExistsObject(directoryEntryPath, &exists, &isDirectory,
NULL);
}
if (result == B_OK) { if (result == B_OK) {
if (isDirectory) if (isDirectory)
RemoveDirectoryContents(directroyEntryPath); RemoveDirectoryContents(directoryEntryPath);
if (remove(directroyEntryPath.Path()) == 0) { if (remove(directoryEntryPath.Path()) == 0) {
if (Logger::IsDebugEnabled()) {
fprintf(stdout, "did delete [%s]\n", fprintf(stdout, "did delete [%s]\n",
directroyEntryPath.Path()); directoryEntryPath.Path());
}
} else { } else {
fprintf(stderr, "unable to delete [%s]\n", fprintf(stderr, "unable to delete [%s]\n",
directroyEntryPath.Path()); directoryEntryPath.Path());
result = B_ERROR; result = B_ERROR;
} }
} }
@ -84,21 +90,34 @@ StorageUtils::RemoveDirectoryContents(BPath& path)
*/ */
status_t status_t
StorageUtils::ExistsDirectory(BPath& directory, StorageUtils::ExistsObject(BPath& path,
bool* exists, bool* exists,
bool* isDirectory) bool* isDirectory,
off_t* size)
{ {
struct stat s; struct stat s;
if (exists != NULL)
*exists = false; *exists = false;
if (isDirectory != NULL)
*isDirectory = false; *isDirectory = false;
if (-1 == stat(directory.Path(), &s)) { if (size != NULL)
*size = 0;
if (-1 == stat(path.Path(), &s)) {
if (ENOENT != errno) if (ENOENT != errno)
return B_ERROR; return B_ERROR;
} else { } else {
if (exists != NULL)
*exists = true; *exists = true;
if (isDirectory != NULL)
*isDirectory = S_ISDIR(s.st_mode); *isDirectory = S_ISDIR(s.st_mode);
if (size != NULL)
*size = s.st_size;
} }
return B_OK; return B_OK;

6
src/apps/haikudepot/util/StorageUtils.h
View File

@ -2,7 +2,6 @@
* Copyright 2017, Andrew Lindesay <apl@lindesay.co.nz>. * Copyright 2017, Andrew Lindesay <apl@lindesay.co.nz>.
* All rights reserved. Distributed under the terms of the MIT License. * All rights reserved. Distributed under the terms of the MIT License.
*/ */
#ifndef PATH_UTILS_H #ifndef PATH_UTILS_H
#define PATH_UTILS_H #define PATH_UTILS_H
@ -13,9 +12,10 @@ class StorageUtils {
public: public:
static status_t RemoveDirectoryContents(BPath& path); static status_t RemoveDirectoryContents(BPath& path);
static status_t AppendToString(BPath& path, BString& result); static status_t AppendToString(BPath& path, BString& result);
static status_t ExistsDirectory(BPath& directory, static status_t ExistsObject(BPath& directory,
bool* exists, bool* exists,
bool* isDirectory); bool* isDirectory,
off_t* size);
}; };
#endif // PATH_UTILS_H #endif // PATH_UTILS_H

9
src/apps/haikudepot/util/ToFileUrlProtocolListener.cpp
View File

@ -18,6 +18,7 @@ ToFileUrlProtocolListener::ToFileUrlProtocolListener(BPath path,
fTraceLoggingIdentifier = traceLoggingIdentifier; fTraceLoggingIdentifier = traceLoggingIdentifier;
fTraceLogging = traceLogging; fTraceLogging = traceLogging;
fShouldDownload = true; fShouldDownload = true;
fContentLength = 0;
} }
@ -70,6 +71,8 @@ void
ToFileUrlProtocolListener::DataReceived(BUrlRequest* caller, const char* data, ToFileUrlProtocolListener::DataReceived(BUrlRequest* caller, const char* data,
off_t position, ssize_t size) off_t position, ssize_t size)
{ {
fContentLength += size;
if (fShouldDownload && fDownloadIO != NULL && size > 0) { if (fShouldDownload && fDownloadIO != NULL && size > 0) {
size_t remaining = size; size_t remaining = size;
size_t written = 0; size_t written = 0;
@ -116,3 +119,9 @@ ToFileUrlProtocolListener::DebugMessage(BUrlRequest* caller,
} }
} }
ssize_t
ToFileUrlProtocolListener::ContentLength()
{
return fContentLength;
}

3
src/apps/haikudepot/util/ToFileUrlProtocolListener.h
View File

@ -13,6 +13,8 @@ public:
bool traceLogging); bool traceLogging);
virtual ~ToFileUrlProtocolListener(); virtual ~ToFileUrlProtocolListener();
ssize_t ContentLength();
void ConnectionOpened(BUrlRequest* caller); void ConnectionOpened(BUrlRequest* caller);
void HostnameResolved(BUrlRequest* caller, void HostnameResolved(BUrlRequest* caller,
const char* ip); const char* ip);
@ -37,6 +39,7 @@ private:
bool fTraceLogging; bool fTraceLogging;
BString fTraceLoggingIdentifier; BString fTraceLoggingIdentifier;
BPositionIO* fDownloadIO; BPositionIO* fDownloadIO;
ssize_t fContentLength;
}; };

6
src/tests/apps/haikudepot/DumpExportRepositoryJsonListenerTest.cpp
View File

@ -122,7 +122,7 @@ public:
TestBulkContainerItemListener(); TestBulkContainerItemListener();
virtual ~TestBulkContainerItemListener(); virtual ~TestBulkContainerItemListener();
void Handle(DumpExportRepository* item); bool Handle(DumpExportRepository* item);
void Complete(); void Complete();
BString ConcatenatedCodes(); BString ConcatenatedCodes();
@ -261,7 +261,7 @@ TestBulkContainerItemListener::~TestBulkContainerItemListener()
for this. for this.
*/ */
void bool
TestBulkContainerItemListener::Handle(DumpExportRepository* item) TestBulkContainerItemListener::Handle(DumpExportRepository* item)
{ {
int32 i; int32 i;
@ -278,6 +278,8 @@ TestBulkContainerItemListener::Handle(DumpExportRepository* item)
fConcatenatedSourcesUrl.Append( fConcatenatedSourcesUrl.Append(
item->RepositorySourcesItemAt(i)->Url()->String()); item->RepositorySourcesItemAt(i)->Url()->String());
} }
return true;
} }

2
src/tests/apps/haikudepot/HaikuDepotTestAddon.cpp
View File

@ -8,6 +8,7 @@
#include "StandardMetaDataJsonEventListenerTest.h" #include "StandardMetaDataJsonEventListenerTest.h"
#include "DumpExportRepositoryJsonListenerTest.h" #include "DumpExportRepositoryJsonListenerTest.h"
#include "ListTest.h"
BTestSuite* BTestSuite*
getTestSuite() getTestSuite()
@ -16,6 +17,7 @@ getTestSuite()
StandardMetaDataJsonEventListenerTest::AddTests(*suite); StandardMetaDataJsonEventListenerTest::AddTests(*suite);
DumpExportRepositoryJsonListenerTest::AddTests(*suite); DumpExportRepositoryJsonListenerTest::AddTests(*suite);
ListTest::AddTests(*suite);
return suite; return suite;
} }

20
src/tests/apps/haikudepot/Jamfile
View File

@ -9,6 +9,16 @@ SubDirHdrs [ FDirName $(HAIKU_TOP) src apps haikudepot server dumpexportreposito
UsePrivateHeaders shared ; UsePrivateHeaders shared ;
local sourceDirs =
server
server/dumpexportrepository
;
local sourceDir ;
for sourceDir in $(sourceDirs) {
SEARCH_SOURCE += [ FDirName $(HAIKU_TOP) src apps haikudepot $(sourceDir) ] ;
}
UnitTestLib haikudepottest.so : UnitTestLib haikudepottest.so :
HaikuDepotTestAddon.cpp HaikuDepotTestAddon.cpp
@ -17,15 +27,11 @@ UnitTestLib haikudepottest.so :
DumpExportRepositoryJsonListener.cpp DumpExportRepositoryJsonListener.cpp
DumpExportRepositoryJsonListenerTest.cpp DumpExportRepositoryJsonListenerTest.cpp
ListTest.cpp
StandardMetaData.cpp StandardMetaData.cpp
StandardMetaDataJsonEventListener.cpp StandardMetaDataJsonEventListener.cpp
StandardMetaDataJsonEventListenerTest.cpp StandardMetaDataJsonEventListenerTest.cpp
: be shared bnetapi [ TargetLibstdc++ ] [ TargetLibsupc++ ] : be shared bnetapi package [ TargetLibstdc++ ] [ TargetLibsupc++ ]
; ;
SEARCH on [ FGristFiles StandardMetaData.cpp StandardMetaDataJsonEventListener.cpp ]
= [ FDirName $(HAIKU_TOP) src apps haikudepot server ] ;
SEARCH on [ FGristFiles DumpExportRepositorySource.cpp DumpExportRepository.cpp DumpExportRepositoryJsonListener.cpp ]
= [ FDirName $(HAIKU_TOP) src apps haikudepot server dumpexportrepository ] ;