HaikuDepot: Improve icon download handling performance
Previously each icon would launch an independent HTTP request to pull down the HVIF icon data. This change means that the data will be pulled down in bulk across all packages as a .tgz and will then be kept in a cache locally. The client-server logic will use standard "If-Modified-Since" headers to check for updates each time the HaikuDepot desktop application starts up. This arrangement will bring down the HVIF as well as bitmap icons and use the best representation it can. Additionally, it is possible from a command-line option to log HTTP traffic verbosely and it is also possible to use an "-h" flag to display help on command-line arguments. The code-structure around this change also anticipates some future extensions to handle other client-server improvements. Fixes #11804
This commit is contained in:
Andrew Lindesay
parent
7c8d207203
commit
19c15fec85
|
|
@ -1,10 +1,10 @@
|
|||
SubDir HAIKU_TOP src apps haikudepot ;
|
||||
|
||||
UsePrivateHeaders interface shared package ;
|
||||
UsePrivateHeaders interface shared package support ;
|
||||
|
||||
# source directories
|
||||
local sourceDirs =
|
||||
edits_generic model textview ui ui_generic
|
||||
edits_generic model textview ui ui_generic server tar util
|
||||
;
|
||||
|
||||
local sourceDir ;
|
||||
|
|
@ -49,6 +49,7 @@ Application HaikuDepot :
|
|||
DecisionProvider.cpp
|
||||
FeaturedPackagesView.cpp
|
||||
FilterView.cpp
|
||||
LocalIconStore.cpp
|
||||
JobStateListener.cpp
|
||||
LinkView.cpp
|
||||
LinkedBitmapView.cpp
|
||||
|
|
@ -72,8 +73,22 @@ Application HaikuDepot :
|
|||
ScrollableGroupView.cpp
|
||||
SharedBitmap.cpp
|
||||
UserLoginWindow.cpp
|
||||
|
||||
# network + server
|
||||
AbstractServerProcess.cpp
|
||||
ServerSettings.cpp
|
||||
WebAppInterface.cpp
|
||||
|
||||
ServerIconExportUpdateProcess.cpp
|
||||
IconMetaData.cpp
|
||||
|
||||
# tar
|
||||
TarArchiveHeader.cpp
|
||||
TarArchiveService.cpp
|
||||
|
||||
#util
|
||||
ToFileUrlProtocolListener.cpp
|
||||
StorageUtils.cpp
|
||||
|
||||
# package_daemon
|
||||
ProblemWindow.cpp
|
||||
ResultWindow.cpp
|
||||
|
|
|
|||
|
|
@ -0,0 +1,127 @@
|
|||
/*
|
||||
* Copyright 2017, Andrew Lindesay <apl@lindesay.co.nz>.
|
||||
* All rights reserved. Distributed under the terms of the MIT License.
|
||||
*/
|
||||
|
||||
#include "LocalIconStore.h"
|
||||
|
||||
#include <Directory.h>
|
||||
#include <FindDirectory.h>
|
||||
|
||||
#include "ServerIconExportUpdateProcess.h"
|
||||
#include "StorageUtils.h"
|
||||
|
||||
|
||||
LocalIconStore::LocalIconStore()
|
||||
{
|
||||
if (_EnsureIconStoragePath(fIconStoragePath) != B_OK)
|
||||
fprintf(stdout, "unable to setup icon storage\n");
|
||||
}
|
||||
|
||||
|
||||
LocalIconStore::~LocalIconStore()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
bool
|
||||
LocalIconStore::_HasIconStoragePath() const
|
||||
{
|
||||
return fIconStoragePath.InitCheck() == B_OK;
|
||||
}
|
||||
|
||||
|
||||
/* This method will try to find an icon for the package name supplied. If an
|
||||
* icon was able to be found then the method will return B_OK and will update
|
||||
* the supplied path object with the path to the icon file.
|
||||
*/
|
||||
|
||||
status_t
|
||||
LocalIconStore::TryFindIconPath(const BString& pkgName, BPath& path) const
|
||||
{
|
||||
if (_HasIconStoragePath()) {
|
||||
BPath bestIconPath;
|
||||
BPath iconPkgPath(fIconStoragePath);
|
||||
bool exists;
|
||||
bool isDir;
|
||||
|
||||
if ( (iconPkgPath.Append("hicn") == B_OK)
|
||||
&& (iconPkgPath.Append(pkgName) == B_OK)
|
||||
&& (StorageUtils::ExistsDirectory(iconPkgPath, &exists, &isDir)
|
||||
== B_OK)
|
||||
&& exists
|
||||
&& isDir
|
||||
&& (_IdentifyBestIconFileAtDirectory(iconPkgPath, bestIconPath)
|
||||
== B_OK)
|
||||
) {
|
||||
path = bestIconPath;
|
||||
return B_OK;
|
||||
}
|
||||
}
|
||||
|
||||
path.Unset();
|
||||
return B_FILE_NOT_FOUND;
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
LocalIconStore::UpdateFromServerIfNecessary() const
|
||||
{
|
||||
if (_HasIconStoragePath()) {
|
||||
BPath iconStoragePath(fIconStoragePath);
|
||||
ServerIconExportUpdateProcess service(iconStoragePath);
|
||||
service.Run();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
status_t
|
||||
LocalIconStore::_EnsureIconStoragePath(BPath& path) const
|
||||
{
|
||||
BPath iconStoragePath;
|
||||
|
||||
if (find_directory(B_USER_CACHE_DIRECTORY, &iconStoragePath) == B_OK
|
||||
&& iconStoragePath.Append("HaikuDepot") == B_OK
|
||||
&& iconStoragePath.Append("__allicons") == B_OK
|
||||
&& create_directory(iconStoragePath.Path(), 0777) == B_OK) {
|
||||
path.SetTo(iconStoragePath.Path());
|
||||
return B_OK;
|
||||
}
|
||||
|
||||
path.Unset();
|
||||
fprintf(stdout, "unable to find the user cache directory for icons");
|
||||
return B_ERROR;
|
||||
}
|
||||
|
||||
|
||||
status_t
|
||||
LocalIconStore::_IdentifyBestIconFileAtDirectory(const BPath& directory,
|
||||
BPath& bestIconPath) const
|
||||
{
|
||||
StringList iconLeafnames;
|
||||
|
||||
iconLeafnames.Add("icon.hvif");
|
||||
iconLeafnames.Add("64.png");
|
||||
iconLeafnames.Add("32.png");
|
||||
iconLeafnames.Add("16.png");
|
||||
|
||||
bestIconPath.Unset();
|
||||
|
||||
for (int32 i = 0; i < iconLeafnames.CountItems(); i++) {
|
||||
BString iconLeafname = iconLeafnames.ItemAt(i);
|
||||
BPath workingPath(directory);
|
||||
bool exists;
|
||||
bool isDir;
|
||||
|
||||
if ( (workingPath.Append(iconLeafname) == B_OK
|
||||
&& StorageUtils::ExistsDirectory(
|
||||
workingPath, &exists, &isDir) == B_OK)
|
||||
&& exists
|
||||
&& !isDir) {
|
||||
bestIconPath.SetTo(workingPath.Path());
|
||||
return B_OK;
|
||||
}
|
||||
}
|
||||
|
||||
return B_FILE_NOT_FOUND;
|
||||
}
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
/*
|
||||
* Copyright 2017, Andrew Lindesay <apl@lindesay.co.nz>.
|
||||
* All rights reserved. Distributed under the terms of the MIT License.
|
||||
*/
|
||||
|
||||
|
||||
#ifndef LOCAL_ICON_STORE_H
|
||||
#define LOCAL_ICON_STORE_H
|
||||
|
||||
#include <String.h>
|
||||
#include <File.h>
|
||||
#include <Path.h>
|
||||
|
||||
#include "PackageInfo.h"
|
||||
|
||||
|
||||
class LocalIconStore {
|
||||
public:
|
||||
LocalIconStore();
|
||||
virtual ~LocalIconStore();
|
||||
status_t TryFindIconPath(const BString& pkgName,
|
||||
BPath& path) const;
|
||||
void UpdateFromServerIfNecessary() const;
|
||||
|
||||
private:
|
||||
bool _HasIconStoragePath() const;
|
||||
status_t _EnsureIconStoragePath(BPath& path) const;
|
||||
status_t _IdentifyBestIconFileAtDirectory(
|
||||
const BPath& directory,
|
||||
BPath& bestIconPath) const;
|
||||
|
||||
BPath fIconStoragePath;
|
||||
|
||||
};
|
||||
|
||||
|
||||
#endif // LOCAL_ICON_STORE_H
|
||||
|
|
@ -1,11 +1,12 @@
|
|||
/*
|
||||
* Copyright 2013-2014, Stephan Aßmus <superstippi@gmx.de>.
|
||||
* Copyright 2014, Axel Dörfler <axeld@pinc-software.de>.
|
||||
* Copyright 2016, Andrew Lindesay <apl@lindesay.co.nz>.
|
||||
* Copyright 2016-2017, Andrew Lindesay <apl@lindesay.co.nz>.
|
||||
* All rights reserved. Distributed under the terms of the MIT License.
|
||||
*/
|
||||
|
||||
#include "Model.h"
|
||||
#include "StorageUtils.h"
|
||||
|
||||
#include <ctime>
|
||||
#include <stdarg.h>
|
||||
|
|
@ -642,7 +643,7 @@ Model::SetShowDevelopPackages(bool show)
|
|||
}
|
||||
|
||||
|
||||
// #pragma mark - information retrival
|
||||
// #pragma mark - information retrieval
|
||||
|
||||
|
||||
void
|
||||
|
|
@ -905,17 +906,60 @@ Model::_PopulateAllPackagesEntry(void* cookie)
|
|||
Model* model = static_cast<Model*>(cookie);
|
||||
model->_PopulateAllPackagesThread(true);
|
||||
model->_PopulateAllPackagesThread(false);
|
||||
model->_PopulateAllPackagesIcons();
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
Model::_PopulateAllPackagesIcons()
|
||||
{
|
||||
fLocalIconStore.UpdateFromServerIfNecessary();
|
||||
|
||||
int32 depotIndex = 0;
|
||||
int32 packageIndex = 0;
|
||||
int32 countIconsSet = 0;
|
||||
|
||||
fprintf(stdout, "will populate all packages' icons\n");
|
||||
|
||||
while (true) {
|
||||
PackageInfoRef package;
|
||||
BAutolock locker(&fLock);
|
||||
|
||||
if (depotIndex > fDepots.CountItems()) {
|
||||
fprintf(stdout, "did populate %ld packages' icons\n",
|
||||
countIconsSet);
|
||||
return;
|
||||
}
|
||||
|
||||
const DepotInfo& depot = fDepots.ItemAt(depotIndex);
|
||||
const PackageList& packages = depot.Packages();
|
||||
|
||||
if (packageIndex >= packages.CountItems()) {
|
||||
// Need the next depot
|
||||
packageIndex = 0;
|
||||
depotIndex++;
|
||||
} else {
|
||||
package = packages.ItemAt(packageIndex);
|
||||
#ifdef DEBUG
|
||||
fprintf(stdout, "will populate package icon for [%s]\n",
|
||||
package->Name().String());
|
||||
#endif
|
||||
if (_PopulatePackageIcon(package) == B_OK)
|
||||
countIconsSet++;
|
||||
|
||||
packageIndex++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
Model::_PopulateAllPackagesThread(bool fromCacheOnly)
|
||||
{
|
||||
int32 depotIndex = 0;
|
||||
int32 packageIndex = 0;
|
||||
PackageList bulkPackageList;
|
||||
PackageList packagesWithIconsList;
|
||||
|
||||
while (!fStopPopulatingAllPackages) {
|
||||
// Obtain PackageInfoRef while keeping the depot and package lists
|
||||
|
|
@ -946,30 +990,15 @@ Model::_PopulateAllPackagesThread(bool fromCacheOnly)
|
|||
//_PopulatePackageInfo(package, fromCacheOnly);
|
||||
bulkPackageList.Add(package);
|
||||
if (bulkPackageList.CountItems() == 50) {
|
||||
_PopulatePackageInfos(bulkPackageList, fromCacheOnly,
|
||||
packagesWithIconsList);
|
||||
_PopulatePackageInfos(bulkPackageList, fromCacheOnly);
|
||||
bulkPackageList.Clear();
|
||||
}
|
||||
if (fromCacheOnly)
|
||||
_PopulatePackageIcon(package, fromCacheOnly);
|
||||
// TODO: Average user rating. It needs to be shown in the
|
||||
// list view, so without the user clicking the package.
|
||||
}
|
||||
|
||||
if (bulkPackageList.CountItems() > 0) {
|
||||
_PopulatePackageInfos(bulkPackageList, fromCacheOnly,
|
||||
packagesWithIconsList);
|
||||
}
|
||||
|
||||
if (!fromCacheOnly) {
|
||||
for (int i = packagesWithIconsList.CountItems() - 1; i >= 0; i--) {
|
||||
if (fStopPopulatingAllPackages)
|
||||
break;
|
||||
const PackageInfoRef& package = packagesWithIconsList.ItemAtFast(i);
|
||||
printf("Getting/Updating native icon for %s\n",
|
||||
package->Name().String());
|
||||
_PopulatePackageIcon(package, fromCacheOnly);
|
||||
}
|
||||
_PopulatePackageInfos(bulkPackageList, fromCacheOnly);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1012,8 +1041,7 @@ Model::_GetCacheFile(BPath& path, BFile& file, directory_which directory,
|
|||
|
||||
|
||||
void
|
||||
Model::_PopulatePackageInfos(PackageList& packages, bool fromCacheOnly,
|
||||
PackageList& packagesWithIcons)
|
||||
Model::_PopulatePackageInfos(PackageList& packages, bool fromCacheOnly)
|
||||
{
|
||||
if (fStopPopulatingAllPackages)
|
||||
return;
|
||||
|
|
@ -1035,8 +1063,6 @@ Model::_PopulatePackageInfos(PackageList& packages, bool fromCacheOnly,
|
|||
BMessage pkgInfo;
|
||||
if (pkgInfo.Unflatten(&file) == B_OK) {
|
||||
_PopulatePackageInfo(package, pkgInfo);
|
||||
if (_HasNativeIcon(pkgInfo))
|
||||
packagesWithIcons.Add(package);
|
||||
packages.Remove(i);
|
||||
}
|
||||
}
|
||||
|
|
@ -1101,8 +1127,6 @@ Model::_PopulatePackageInfos(PackageList& packages, bool fromCacheOnly,
|
|||
const PackageInfoRef& package = packages.ItemAtFast(i);
|
||||
if (pkgName == package->Name()) {
|
||||
_PopulatePackageInfo(package, pkgInfo);
|
||||
if (_HasNativeIcon(pkgInfo))
|
||||
packagesWithIcons.Add(package);
|
||||
|
||||
// Store in cache
|
||||
BFile file;
|
||||
|
|
@ -1136,8 +1160,8 @@ Model::_PopulatePackageInfos(PackageList& packages, bool fromCacheOnly,
|
|||
for (int i = count / 2; i < count; i++)
|
||||
secondHalf.Add(packages.ItemAtFast(i));
|
||||
packages.Clear();
|
||||
_PopulatePackageInfos(firstHalf, fromCacheOnly, packagesWithIcons);
|
||||
_PopulatePackageInfos(secondHalf, fromCacheOnly, packagesWithIcons);
|
||||
_PopulatePackageInfos(firstHalf, fromCacheOnly);
|
||||
_PopulatePackageInfos(secondHalf, fromCacheOnly);
|
||||
} else {
|
||||
while (packages.CountItems() > 0) {
|
||||
const PackageInfoRef& package = packages.ItemAtFast(0);
|
||||
|
|
@ -1337,41 +1361,33 @@ Model::_PopulatePackageInfo(const PackageInfoRef& package, const BMessage& data)
|
|||
}
|
||||
|
||||
|
||||
void
|
||||
Model::_PopulatePackageIcon(const PackageInfoRef& package, bool fromCacheOnly)
|
||||
status_t
|
||||
Model::_PopulatePackageIcon(const PackageInfoRef& package)
|
||||
{
|
||||
// See if there is a cached icon file
|
||||
BFile iconFile;
|
||||
BPath iconCachePath;
|
||||
BString iconName(package->Name());
|
||||
iconName << ".hvif";
|
||||
if (_GetCacheFile(iconCachePath, iconFile, B_USER_CACHE_DIRECTORY,
|
||||
"HaikuDepot", iconName, fromCacheOnly, 60 * 60)) {
|
||||
// Cache file is recent enough, just use it and return.
|
||||
BitmapRef bitmapRef(new(std::nothrow)SharedBitmap(iconFile), true);
|
||||
BPath bestIconPath;
|
||||
|
||||
if ( fLocalIconStore.TryFindIconPath(
|
||||
package->Name(), bestIconPath) == B_OK) {
|
||||
|
||||
BFile bestIconFile(bestIconPath.Path(), O_RDONLY);
|
||||
BitmapRef bitmapRef(new(std::nothrow)SharedBitmap(bestIconFile), true);
|
||||
BAutolock locker(&fLock);
|
||||
package->SetIcon(bitmapRef);
|
||||
return;
|
||||
|
||||
#ifdef DEBUG
|
||||
fprintf(stdout, "have set the package icon for [%s] from [%s]\n",
|
||||
package->Name().String(), bestIconPath.Path());
|
||||
#endif
|
||||
|
||||
return B_OK;
|
||||
}
|
||||
|
||||
if (fromCacheOnly)
|
||||
return;
|
||||
#ifdef DEBUG
|
||||
fprintf(stdout, "did not set the package icon for [%s]; no data\n",
|
||||
package->Name().String());
|
||||
#endif
|
||||
|
||||
// Retrieve icon from web-app
|
||||
BMallocIO buffer;
|
||||
|
||||
status_t status = fWebAppInterface.RetrievePackageIcon(package->Name(),
|
||||
&buffer);
|
||||
if (status == B_OK) {
|
||||
BitmapRef bitmapRef(new(std::nothrow)SharedBitmap(buffer), true);
|
||||
BAutolock locker(&fLock);
|
||||
package->SetIcon(bitmapRef);
|
||||
locker.Unlock();
|
||||
if (iconFile.SetTo(iconCachePath.Path(),
|
||||
B_WRITE_ONLY | B_CREATE_FILE | B_ERASE_FILE) == B_OK) {
|
||||
iconFile.Write(buffer.Buffer(), buffer.BufferLength());
|
||||
}
|
||||
}
|
||||
return B_FILE_NOT_FOUND;
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -1439,32 +1455,6 @@ Model::_PopulatePackageScreenshot(const PackageInfoRef& package,
|
|||
}
|
||||
|
||||
|
||||
bool
|
||||
Model::_HasNativeIcon(const BMessage& message) const
|
||||
{
|
||||
BMessage pkgIcons;
|
||||
if (message.FindMessage("pkgIcons", &pkgIcons) != B_OK)
|
||||
return false;
|
||||
|
||||
int32 index = 0;
|
||||
while (true) {
|
||||
BString name;
|
||||
name << index++;
|
||||
|
||||
BMessage typeCodeInfo;
|
||||
if (pkgIcons.FindMessage(name, &typeCodeInfo) != B_OK)
|
||||
break;
|
||||
|
||||
BString mediaTypeCode;
|
||||
if (typeCodeInfo.FindString("mediaTypeCode", &mediaTypeCode) == B_OK
|
||||
&& mediaTypeCode == "application/x-vnd.haiku-icon") {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
// #pragma mark - listener notification methods
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
/*
|
||||
* Copyright 2013-2014, Stephan Aßmus <superstippi@gmx.de>.
|
||||
* Copyright 2016, Andrew Lindesay <apl@lindesay.co.nz>.
|
||||
* Copyright 2016-2017, Andrew Lindesay <apl@lindesay.co.nz>.
|
||||
* All rights reserved. Distributed under the terms of the MIT License.
|
||||
*/
|
||||
#ifndef MODEL_H
|
||||
|
|
@ -9,6 +9,7 @@
|
|||
#include <FindDirectory.h>
|
||||
#include <Locker.h>
|
||||
|
||||
#include "LocalIconStore.h"
|
||||
#include "PackageInfo.h"
|
||||
#include "WebAppInterface.h"
|
||||
|
||||
|
|
@ -156,6 +157,8 @@ private:
|
|||
static int32 _PopulateAllPackagesEntry(void* cookie);
|
||||
void _PopulateAllPackagesThread(bool fromCacheOnly);
|
||||
|
||||
void _PopulateAllPackagesIcons();
|
||||
|
||||
bool _GetCacheFile(BPath& path, BFile& file,
|
||||
directory_which directory,
|
||||
const char* relativeLocation,
|
||||
|
|
@ -169,18 +172,15 @@ private:
|
|||
|
||||
void _PopulatePackageInfos(
|
||||
PackageList& packages,
|
||||
bool fromCacheOnly,
|
||||
PackageList& packagesWithIcons);
|
||||
bool fromCacheOnly);
|
||||
void _PopulatePackageInfo(
|
||||
const PackageInfoRef& package,
|
||||
bool fromCacheOnly);
|
||||
void _PopulatePackageInfo(
|
||||
const PackageInfoRef& package,
|
||||
const BMessage& data);
|
||||
void _PopulatePackageIcon(
|
||||
const PackageInfoRef& package,
|
||||
bool fromCacheOnly);
|
||||
bool _HasNativeIcon(const BMessage& message) const;
|
||||
status_t _PopulatePackageIcon(
|
||||
const PackageInfoRef& package);
|
||||
void _PopulatePackageScreenshot(
|
||||
const PackageInfoRef& package,
|
||||
const ScreenshotInfo& info,
|
||||
|
|
@ -194,6 +194,8 @@ private:
|
|||
|
||||
DepotList fDepots;
|
||||
|
||||
LocalIconStore fLocalIconStore;
|
||||
|
||||
CategoryRef fCategoryAudio;
|
||||
CategoryRef fCategoryBusiness;
|
||||
CategoryRef fCategoryDevelopment;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,12 @@
|
|||
/*
|
||||
* Copyright 2017, Andrew Lindesay <apl@lindesay.co.nz>.
|
||||
* All rights reserved. Distributed under the terms of the MIT License.
|
||||
*/
|
||||
#include "AbstractServerProcess.h"
|
||||
|
||||
|
||||
status_t
|
||||
AbstractServerProcess::Run()
|
||||
{
|
||||
return B_OK;
|
||||
}
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
/*
|
||||
* Copyright 2017, Andrew Lindesay <apl@lindesay.co.nz>.
|
||||
* All rights reserved. Distributed under the terms of the MIT License.
|
||||
*/
|
||||
|
||||
#ifndef ABSTRACT_SERVER_PROCESS_H
|
||||
#define ABSTRACT_SERVER_PROCESS_H
|
||||
|
||||
#include <String.h>
|
||||
|
||||
class AbstractServerProcess {
|
||||
public:
|
||||
status_t Run();
|
||||
};
|
||||
|
||||
#endif // ABSTRACT_SERVER_PROCESS_H
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
/*
|
||||
* Copyright 2017, Andrew Lindesay <apl@lindesay.co.nz>.
|
||||
* All rights reserved. Distributed under the terms of the MIT License.
|
||||
*/
|
||||
|
||||
#include "IconMetaData.h"
|
||||
|
||||
|
||||
uint64_t
|
||||
IconMetaData::GetCreateTimestamp()
|
||||
{
|
||||
return fCreateTimestamp;
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
IconMetaData::SetCreateTimestamp(uint64_t value)
|
||||
{
|
||||
fCreateTimestamp = value;
|
||||
}
|
||||
|
||||
|
||||
uint64_t
|
||||
IconMetaData::GetDataModifiedTimestamp()
|
||||
{
|
||||
return fDataModifiedTimestamp;
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
IconMetaData::SetDataModifiedTimestamp(uint64_t value)
|
||||
{
|
||||
fDataModifiedTimestamp = value;
|
||||
}
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
/*
|
||||
* Copyright 2017, Andrew Lindesay <apl@lindesay.co.nz>.
|
||||
* All rights reserved. Distributed under the terms of the MIT License.
|
||||
*/
|
||||
|
||||
|
||||
#ifndef ICON_META_DATA_H
|
||||
#define ICON_META_DATA_H
|
||||
|
||||
#include <File.h>
|
||||
#include <HttpHeaders.h>
|
||||
#include <Locker.h>
|
||||
#include <String.h>
|
||||
|
||||
|
||||
/* This class models (some of) the meta-data that is bundled into the tar file
|
||||
* that is downloaded to the HaikuDepot client from the HaikuDepotServer
|
||||
* application server system. The file is included in the tar-ball data.
|
||||
*/
|
||||
|
||||
|
||||
class IconMetaData {
|
||||
public:
|
||||
uint64_t GetCreateTimestamp();
|
||||
void SetCreateTimestamp(uint64_t value);
|
||||
|
||||
uint64_t GetDataModifiedTimestamp();
|
||||
void SetDataModifiedTimestamp(
|
||||
uint64_t value);
|
||||
|
||||
private:
|
||||
uint64_t fCreateTimestamp;
|
||||
uint64_t fDataModifiedTimestamp;
|
||||
};
|
||||
|
||||
|
||||
#endif // ICON_META_DATA_H
|
||||
|
|
@ -0,0 +1,294 @@
|
|||
/*
|
||||
* Copyright 2017, Andrew Lindesay <apl@lindesay.co.nz>.
|
||||
* All rights reserved. Distributed under the terms of the MIT License.
|
||||
*/
|
||||
|
||||
#include "ServerIconExportUpdateProcess.h"
|
||||
|
||||
#include <errno.h>
|
||||
#include <stdio.h>
|
||||
#include <sys/stat.h>
|
||||
#include <time.h>
|
||||
|
||||
#include <HttpRequest.h>
|
||||
#include <Json.h>
|
||||
#include <Url.h>
|
||||
#include <support/ZlibCompressionAlgorithm.h>
|
||||
|
||||
#include "ServerSettings.h"
|
||||
#include "StorageUtils.h"
|
||||
#include "TarArchiveService.h"
|
||||
#include "ToFileUrlProtocolListener.h"
|
||||
|
||||
|
||||
#define MAX_REDIRECTS 3
|
||||
#define MAX_FAILURES 2
|
||||
|
||||
#define HTTP_STATUS_OK 200
|
||||
#define HTTP_STATUS_FOUND 302
|
||||
#define HTTP_STATUS_NOT_MODIFIED 304
|
||||
|
||||
#define APP_ERR_NOT_MODIFIED (B_APP_ERROR_BASE + 452)
|
||||
|
||||
// 30 seconds
|
||||
#define TIMEOUT_MICROSECONDS 3e+7
|
||||
|
||||
|
||||
/*! This constructor will locate the cached data in a standardized location */
|
||||
|
||||
ServerIconExportUpdateProcess::ServerIconExportUpdateProcess(
|
||||
const BPath& localStorageDirectoryPath)
|
||||
{
|
||||
fLocalStorageDirectoryPath = localStorageDirectoryPath;
|
||||
}
|
||||
|
||||
|
||||
status_t
|
||||
ServerIconExportUpdateProcess::Run()
|
||||
{
|
||||
BPath tarGzFilePath(tmpnam(NULL));
|
||||
status_t result = B_OK;
|
||||
|
||||
fprintf(stdout, "will start fetching icons\n");
|
||||
|
||||
result = _Download(tarGzFilePath);
|
||||
|
||||
if (result != APP_ERR_NOT_MODIFIED) {
|
||||
if (result != B_OK)
|
||||
return result;
|
||||
|
||||
fprintf(stdout, "delete any existing stored data\n");
|
||||
StorageUtils::RemoveDirectoryContents(fLocalStorageDirectoryPath);
|
||||
|
||||
BFile *tarGzFile = new BFile(tarGzFilePath.Path(), O_RDONLY);
|
||||
BDataIO* tarIn;
|
||||
|
||||
BZlibDecompressionParameters* zlibDecompressionParameters
|
||||
= new BZlibDecompressionParameters();
|
||||
|
||||
result = BZlibCompressionAlgorithm()
|
||||
.CreateDecompressingInputStream(tarGzFile,
|
||||
zlibDecompressionParameters, tarIn);
|
||||
|
||||
if (result == B_OK) {
|
||||
result = TarArchiveService::Unpack(*tarIn,
|
||||
fLocalStorageDirectoryPath);
|
||||
|
||||
if (result == B_OK) {
|
||||
if (0 != remove(tarGzFilePath.Path())) {
|
||||
fprintf(stdout, "unable to delete the temporary tgz path; "
|
||||
"%s", tarGzFilePath.Path());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
delete tarGzFile;
|
||||
}
|
||||
|
||||
fprintf(stdout, "did complete fetching icons\n");
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
status_t
|
||||
ServerIconExportUpdateProcess::_IfModifiedSinceHeaderValue(BString& headerValue,
|
||||
BPath& iconMetaDataPath) const
|
||||
{
|
||||
headerValue.SetTo("");
|
||||
struct stat s;
|
||||
|
||||
if (-1 == stat(iconMetaDataPath.Path(), &s)) {
|
||||
if (ENOENT != errno)
|
||||
return B_ERROR;
|
||||
|
||||
return B_FILE_NOT_FOUND;
|
||||
}
|
||||
|
||||
IconMetaData iconMetaData;
|
||||
status_t result = _PopulateIconMetaData(iconMetaData, iconMetaDataPath);
|
||||
|
||||
if (result == B_OK) {
|
||||
_TimestampToRfc2822String(iconMetaData.GetDataModifiedTimestamp(),
|
||||
headerValue);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
status_t
|
||||
ServerIconExportUpdateProcess::_IfModifiedSinceHeaderValue(BString& headerValue)
|
||||
const
|
||||
{
|
||||
BPath iconMetaDataPath(fLocalStorageDirectoryPath);
|
||||
iconMetaDataPath.Append("hicn/info.json");
|
||||
return _IfModifiedSinceHeaderValue(headerValue, iconMetaDataPath);
|
||||
}
|
||||
|
||||
|
||||
status_t
|
||||
ServerIconExportUpdateProcess::_Download(BPath& tarGzFilePath)
|
||||
{
|
||||
BString urlString = ServerSettings::CreateFullUrl("/__pkgicon/all.tar.gz");
|
||||
return _Download(tarGzFilePath, BUrl(urlString), 0, 0);
|
||||
}
|
||||
|
||||
|
||||
status_t
|
||||
ServerIconExportUpdateProcess::_Download(BPath& tarGzFilePath, const BUrl& url,
|
||||
uint32 redirects, uint32 failures)
|
||||
{
|
||||
if (redirects > MAX_REDIRECTS) {
|
||||
fprintf(stdout, "exceeded %d redirects --> failure\n", MAX_REDIRECTS);
|
||||
return B_IO_ERROR;
|
||||
}
|
||||
|
||||
if (failures > MAX_FAILURES) {
|
||||
fprintf(stdout, "exceeded %d failures\n", MAX_FAILURES);
|
||||
return B_IO_ERROR;
|
||||
}
|
||||
|
||||
fprintf(stdout, "will stream '%s' to [%s]\n", url.UrlString().String(),
|
||||
tarGzFilePath.Path());
|
||||
|
||||
bool isSecure = url.Protocol() == BString("https");
|
||||
ToFileUrlProtocolListener listener(tarGzFilePath, "icon-export",
|
||||
ServerSettings::UrlConnectionTraceLoggingEnabled());
|
||||
BUrlContext context;
|
||||
|
||||
BHttpHeaders headers;
|
||||
ServerSettings::AugmentHeaders(headers);
|
||||
|
||||
BString ifModifiedSinceHeader;
|
||||
status_t ifModifiedSinceHeaderStatus = _IfModifiedSinceHeaderValue(
|
||||
ifModifiedSinceHeader);
|
||||
|
||||
if (ifModifiedSinceHeaderStatus == B_OK &&
|
||||
ifModifiedSinceHeader.Length() > 0) {
|
||||
headers.AddHeader("If-Modified-Since", ifModifiedSinceHeader);
|
||||
}
|
||||
|
||||
BHttpRequest request(url, isSecure, "HTTP", &listener, &context);
|
||||
request.SetMethod(B_HTTP_GET);
|
||||
request.SetHeaders(headers);
|
||||
request.SetTimeout(TIMEOUT_MICROSECONDS);
|
||||
|
||||
thread_id thread = request.Run();
|
||||
wait_for_thread(thread, NULL);
|
||||
|
||||
const BHttpResult& result = dynamic_cast<const BHttpResult&>(
|
||||
request.Result());
|
||||
|
||||
int32 statusCode = result.StatusCode();
|
||||
|
||||
switch (statusCode) {
|
||||
case HTTP_STATUS_OK:
|
||||
fprintf(stdout, "did complete streaming data\n");
|
||||
return B_OK;
|
||||
|
||||
case HTTP_STATUS_NOT_MODIFIED:
|
||||
fprintf(stdout, "remote data has not changed since [%s]\n",
|
||||
ifModifiedSinceHeader.String());
|
||||
return APP_ERR_NOT_MODIFIED;
|
||||
|
||||
case HTTP_STATUS_FOUND: // redirect
|
||||
{
|
||||
const BHttpHeaders responseHeaders = result.Headers();
|
||||
const char *locationValue = responseHeaders["Location"];
|
||||
|
||||
if (NULL != locationValue && 0 != strlen(locationValue)) {
|
||||
BUrl location(locationValue);
|
||||
fprintf(stdout, "will redirect to; %s\n",
|
||||
location.UrlString().String());
|
||||
return _Download(tarGzFilePath, location, redirects + 1, 0);
|
||||
}
|
||||
|
||||
fprintf(stdout, "unable to find 'Location' header for redirect\n");
|
||||
return B_IO_ERROR;
|
||||
}
|
||||
|
||||
default:
|
||||
if (0 == statusCode || 5 == (statusCode / 100)) {
|
||||
fprintf(stdout, "error response from server; %zu --> "
|
||||
"retry...\n", statusCode);
|
||||
return _Download(tarGzFilePath, url, redirects, failures + 1);
|
||||
}
|
||||
|
||||
fprintf(stdout, "unexpected response from server; %zu\n",
|
||||
statusCode);
|
||||
return B_IO_ERROR;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
status_t
|
||||
ServerIconExportUpdateProcess::_PopulateIconMetaData(IconMetaData& iconMetaData,
|
||||
BMessage& message) const
|
||||
{
|
||||
status_t result = B_OK;
|
||||
double value; // numeric resolution issue?
|
||||
|
||||
if (result == B_OK)
|
||||
result = message.FindDouble("createTimestamp", &value);
|
||||
|
||||
if (result == B_OK)
|
||||
iconMetaData.SetCreateTimestamp((uint64) value);
|
||||
|
||||
if (result == B_OK)
|
||||
result = message.FindDouble("dataModifiedTimestamp", &value);
|
||||
|
||||
if (result == B_OK)
|
||||
iconMetaData.SetDataModifiedTimestamp((uint64) value);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
status_t
|
||||
ServerIconExportUpdateProcess::_PopulateIconMetaData(IconMetaData& iconMetaData,
|
||||
BString& jsonString) const
|
||||
{
|
||||
BJson parser;
|
||||
BMessage infoMetaDataMessage;
|
||||
status_t result = parser.Parse(infoMetaDataMessage, jsonString);
|
||||
|
||||
if (result == B_OK)
|
||||
return _PopulateIconMetaData(iconMetaData, infoMetaDataMessage);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
status_t
|
||||
ServerIconExportUpdateProcess::_PopulateIconMetaData(IconMetaData& iconMetaData,
|
||||
BPath& path) const
|
||||
{
|
||||
|
||||
BString infoMetaDataStr;
|
||||
status_t result = StorageUtils::AppendToString(path, infoMetaDataStr);
|
||||
|
||||
if (result == B_OK)
|
||||
return _PopulateIconMetaData(iconMetaData, infoMetaDataStr);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
/* The output format for this is suitable for use in the "If-Modified-Since"
|
||||
* header. An example of this output would be;
|
||||
* 'Fri, 24 Oct 2014 19:32:27 +0000'
|
||||
*/
|
||||
|
||||
void
|
||||
ServerIconExportUpdateProcess::_TimestampToRfc2822String(
|
||||
uint64_t timestampMillis,
|
||||
BString& rfc2822String) const
|
||||
{
|
||||
char output[256];
|
||||
output[0] = 0;
|
||||
time_t t = (timestampMillis / 1000);
|
||||
strftime(output, 256, "%a, %d %b %Y %T %z", gmtime(&t));
|
||||
rfc2822String.SetTo(output);
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
/*
|
||||
* Copyright 2017, Andrew Lindesay <apl@lindesay.co.nz>.
|
||||
* All rights reserved. Distributed under the terms of the MIT License.
|
||||
*/
|
||||
|
||||
#ifndef SERVER_ICON_PROCESS_H
|
||||
#define SERVER_ICON_PROCESS_H
|
||||
|
||||
|
||||
#include "AbstractServerProcess.h"
|
||||
|
||||
#include <File.h>
|
||||
#include <Path.h>
|
||||
#include <String.h>
|
||||
#include <Url.h>
|
||||
|
||||
#include "IconMetaData.h"
|
||||
|
||||
|
||||
class ServerIconExportUpdateProcess : public AbstractServerProcess {
|
||||
public:
|
||||
|
||||
ServerIconExportUpdateProcess(
|
||||
const BPath& localStorageDirectoryPath);
|
||||
|
||||
status_t Run();
|
||||
|
||||
private:
|
||||
status_t _Download(BPath& tarGzFilePath);
|
||||
status_t _Download(BPath& tarGzFilePath, const BUrl& url,
|
||||
uint32 redirects, uint32 failures);
|
||||
BString _FormFullUrl(const BString& suffix) const;
|
||||
status_t _IfModifiedSinceHeaderValue(
|
||||
BString& headerValue) const;
|
||||
status_t _IfModifiedSinceHeaderValue(
|
||||
BString& headerValue,
|
||||
BPath& iconMetaDataPath) const;
|
||||
|
||||
status_t _PopulateIconMetaData(
|
||||
IconMetaData& iconMetaData, BPath& path)
|
||||
const;
|
||||
status_t _PopulateIconMetaData(
|
||||
IconMetaData& iconMetaData,
|
||||
BString& jsonString) const;
|
||||
status_t _PopulateIconMetaData(
|
||||
IconMetaData& iconMetaData,
|
||||
BMessage& message) const;
|
||||
|
||||
void _TimestampToRfc2822String(
|
||||
uint64_t timestampMillis,
|
||||
BString& rfc2822String) const;
|
||||
|
||||
BString fBaseUrl;
|
||||
BPath fLocalStorageDirectoryPath;
|
||||
|
||||
};
|
||||
|
||||
#endif // SERVER_ICON_PROCESS_H
|
||||
|
|
@ -0,0 +1,128 @@
|
|||
/*
|
||||
* Copyright 2017, Andrew Lindesay <apl@lindesay.co.nz>.
|
||||
* All rights reserved. Distributed under the terms of the MIT License.
|
||||
*/
|
||||
|
||||
#include "ServerSettings.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
#include <AppFileInfo.h>
|
||||
#include <Application.h>
|
||||
#include <Roster.h>
|
||||
#include <Url.h>
|
||||
|
||||
#include "AutoLocker.h"
|
||||
|
||||
|
||||
#define BASEURL_DEFAULT "https://depot.haiku-os.org"
|
||||
#define USERAGENT_FALLBACK_VERSION "0.0.0"
|
||||
|
||||
|
||||
BString ServerSettings::fBaseUrl = BString(BASEURL_DEFAULT);
|
||||
BString ServerSettings::fUserAgent = BString();
|
||||
BLocker ServerSettings::fUserAgentLocker;
|
||||
bool ServerSettings::fUrlConnectionTraceLogging = false;
|
||||
|
||||
|
||||
status_t
|
||||
ServerSettings::SetBaseUrl(const BString& value)
|
||||
{
|
||||
BUrl url(value);
|
||||
|
||||
if (!url.IsValid()) {
|
||||
fprintf(stderr, "the url is not valid\n");
|
||||
return B_BAD_VALUE;
|
||||
}
|
||||
|
||||
if (url.Protocol() != "http" && url.Protocol() != "https") {
|
||||
fprintf(stderr, "the url protocol must be 'http' or 'https'\n");
|
||||
return B_BAD_VALUE;
|
||||
}
|
||||
|
||||
fBaseUrl.SetTo(value);
|
||||
|
||||
if (fBaseUrl.EndsWith("/")) {
|
||||
fprintf(stderr, "will remove trailing '/' character in url base\n");
|
||||
fBaseUrl.Remove(fBaseUrl.Length() - 1, 1);
|
||||
}
|
||||
|
||||
return B_OK;
|
||||
}
|
||||
|
||||
|
||||
BString
|
||||
ServerSettings::CreateFullUrl(const BString urlPathComponents)
|
||||
{
|
||||
return BString(fBaseUrl) << urlPathComponents;
|
||||
}
|
||||
|
||||
|
||||
const BString
|
||||
ServerSettings::GetUserAgent()
|
||||
{
|
||||
AutoLocker<BLocker> lock(&fUserAgentLocker);
|
||||
|
||||
if (fUserAgent.IsEmpty()) {
|
||||
fUserAgent.SetTo("HaikuDepot/");
|
||||
fUserAgent.Append(_GetUserAgentVersionString());
|
||||
}
|
||||
|
||||
return fUserAgent;
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
ServerSettings::EnableUrlConnectionTraceLogging() {
|
||||
fUrlConnectionTraceLogging = true;
|
||||
}
|
||||
|
||||
bool
|
||||
ServerSettings::UrlConnectionTraceLoggingEnabled() {
|
||||
return fUrlConnectionTraceLogging;
|
||||
}
|
||||
|
||||
|
||||
const BString
|
||||
ServerSettings::_GetUserAgentVersionString()
|
||||
{
|
||||
app_info info;
|
||||
|
||||
if (be_app->GetAppInfo(&info) != B_OK) {
|
||||
fprintf(stderr, "Unable to get the application info\n");
|
||||
be_app->Quit();
|
||||
return BString(USERAGENT_FALLBACK_VERSION);
|
||||
}
|
||||
|
||||
BFile file(&info.ref, B_READ_ONLY);
|
||||
|
||||
if (file.InitCheck() != B_OK) {
|
||||
fprintf(stderr, "Unable to access the application info file\n");
|
||||
be_app->Quit();
|
||||
return BString(USERAGENT_FALLBACK_VERSION);
|
||||
}
|
||||
|
||||
BAppFileInfo appFileInfo(&file);
|
||||
version_info versionInfo;
|
||||
|
||||
if (appFileInfo.GetVersionInfo(
|
||||
&versionInfo, B_APP_VERSION_KIND) != B_OK) {
|
||||
fprintf(stderr, "Unable to establish the application version\n");
|
||||
be_app->Quit();
|
||||
return BString(USERAGENT_FALLBACK_VERSION);
|
||||
}
|
||||
|
||||
BString result;
|
||||
result.SetToFormat("%" B_PRId32 ".%" B_PRId32 ".%" B_PRId32,
|
||||
versionInfo.major, versionInfo.middle, versionInfo.minor);
|
||||
return result;
|
||||
}
|
||||
|
||||
void
|
||||
ServerSettings::AugmentHeaders(BHttpHeaders& headers)
|
||||
{
|
||||
headers.AddHeader("User-Agent", GetUserAgent());
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
/*
|
||||
* Copyright 2017, Andrew Lindesay <apl@lindesay.co.nz>.
|
||||
* All rights reserved. Distributed under the terms of the MIT License.
|
||||
*/
|
||||
|
||||
#ifndef SERVER_SETTINGS_H
|
||||
#define SERVER_SETTINGS_H
|
||||
|
||||
#include <File.h>
|
||||
#include <HttpHeaders.h>
|
||||
#include <Locker.h>
|
||||
#include <String.h>
|
||||
|
||||
|
||||
class ServerSettings {
|
||||
public:
|
||||
static status_t SetBaseUrl(const BString& baseUrl);
|
||||
static const BString GetUserAgent();
|
||||
static void AugmentHeaders(BHttpHeaders& headers);
|
||||
static BString CreateFullUrl(
|
||||
const BString urlPathComponents);
|
||||
static void EnableUrlConnectionTraceLogging();
|
||||
static bool UrlConnectionTraceLoggingEnabled();
|
||||
|
||||
private:
|
||||
static const BString _GetUserAgentVersionString();
|
||||
|
||||
static BString fBaseUrl;
|
||||
static BString fUserAgent;
|
||||
static BLocker fUserAgentLocker;
|
||||
static bool fUrlConnectionTraceLogging;
|
||||
};
|
||||
|
||||
#endif // SERVER_SETTINGS_H
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
/*
|
||||
* Copyright 2014, Stephan Aßmus <superstippi@gmx.de>.
|
||||
* Copyright 2016, Andrew Lindesay <apl@lindesay.co.nz>.
|
||||
* Copyright 2016-2017, Andrew Lindesay <apl@lindesay.co.nz>.
|
||||
* All rights reserved. Distributed under the terms of the MIT License.
|
||||
*/
|
||||
|
||||
|
|
@ -25,6 +25,7 @@
|
|||
#include "AutoLocker.h"
|
||||
#include "List.h"
|
||||
#include "PackageInfo.h"
|
||||
#include "ServerSettings.h"
|
||||
|
||||
|
||||
#define BASEURL_DEFAULT "https://depot.haiku-os.org"
|
||||
|
|
@ -186,43 +187,36 @@ private:
|
|||
|
||||
class ProtocolListener : public BUrlProtocolListener {
|
||||
public:
|
||||
ProtocolListener()
|
||||
ProtocolListener(bool traceLogging)
|
||||
:
|
||||
fDownloadIO(NULL),
|
||||
fDebug(false)
|
||||
fTraceLogging(traceLogging)
|
||||
{
|
||||
}
|
||||
|
||||
virtual ~ProtocolListener()
|
||||
{
|
||||
}
|
||||
|
||||
virtual void ConnectionOpened(BUrlRequest* caller)
|
||||
{
|
||||
// printf("ConnectionOpened(%p)\n", caller);
|
||||
}
|
||||
|
||||
virtual void HostnameResolved(BUrlRequest* caller, const char* ip)
|
||||
{
|
||||
// printf("HostnameResolved(%p): %s\n", caller, ip);
|
||||
}
|
||||
|
||||
virtual void ResponseStarted(BUrlRequest* caller)
|
||||
{
|
||||
if (fDebug)
|
||||
printf("ResponseStarted(%p)\n", caller);
|
||||
}
|
||||
|
||||
virtual void HeadersReceived(BUrlRequest* caller)
|
||||
{
|
||||
if (fDebug)
|
||||
printf("HeadersReceived(%p)\n", caller);
|
||||
}
|
||||
|
||||
virtual void DataReceived(BUrlRequest* caller, const char* data,
|
||||
off_t position, ssize_t size)
|
||||
{
|
||||
if (fDebug) {
|
||||
printf("DataReceived(%p): %ld bytes\n", caller, size);
|
||||
printf("%.*s", (int)size, data);
|
||||
}
|
||||
|
||||
if (fDownloadIO != NULL)
|
||||
fDownloadIO->Write(data, size);
|
||||
}
|
||||
|
|
@ -230,27 +224,22 @@ public:
|
|||
virtual void DownloadProgress(BUrlRequest* caller, ssize_t bytesReceived,
|
||||
ssize_t bytesTotal)
|
||||
{
|
||||
// printf("DownloadProgress(%p): %ld/%ld\n", caller, bytesReceived,
|
||||
// bytesTotal);
|
||||
}
|
||||
|
||||
virtual void UploadProgress(BUrlRequest* caller, ssize_t bytesSent,
|
||||
ssize_t bytesTotal)
|
||||
{
|
||||
if (fDebug)
|
||||
printf("UploadProgress(%p): %ld/%ld\n", caller, bytesSent, bytesTotal);
|
||||
}
|
||||
|
||||
virtual void RequestCompleted(BUrlRequest* caller, bool success)
|
||||
{
|
||||
if (fDebug)
|
||||
printf("RequestCompleted(%p): %d\n", caller, success);
|
||||
}
|
||||
|
||||
virtual void DebugMessage(BUrlRequest* caller,
|
||||
BUrlProtocolDebugMessage type, const char* text)
|
||||
{
|
||||
// printf("DebugMessage(%p): %s\n", caller, text);
|
||||
if (fTraceLogging)
|
||||
printf("jrpc: %s\n", text);
|
||||
}
|
||||
|
||||
void SetDownloadIO(BDataIO* downloadIO)
|
||||
|
|
@ -258,14 +247,9 @@ public:
|
|||
fDownloadIO = downloadIO;
|
||||
}
|
||||
|
||||
void SetDebug(bool debug)
|
||||
{
|
||||
fDebug = debug;
|
||||
}
|
||||
|
||||
private:
|
||||
BDataIO* fDownloadIO;
|
||||
bool fDebug;
|
||||
bool fTraceLogging;
|
||||
};
|
||||
|
||||
|
||||
|
|
@ -275,14 +259,9 @@ WebAppInterface::fRequestIndex = 0;
|
|||
|
||||
enum {
|
||||
NEEDS_AUTHORIZATION = 1 << 0,
|
||||
ENABLE_DEBUG = 1 << 1,
|
||||
};
|
||||
|
||||
|
||||
BString WebAppInterface::fBaseUrl = BString(BASEURL_DEFAULT);
|
||||
BString WebAppInterface::fUserAgent = BString();
|
||||
BLocker WebAppInterface::fUserAgentLocker;
|
||||
|
||||
WebAppInterface::WebAppInterface()
|
||||
:
|
||||
fLanguage("en")
|
||||
|
|
@ -327,108 +306,6 @@ WebAppInterface::SetAuthorization(const BString& username,
|
|||
}
|
||||
|
||||
|
||||
static bool
|
||||
arguments_is_url_valid(const BString& value)
|
||||
{
|
||||
if (value.Length() < 8) {
|
||||
fprintf(stderr, "the url is less than 8 characters in length\n");
|
||||
return false;
|
||||
}
|
||||
|
||||
int32 schemeEnd = value.FindFirst("://");
|
||||
|
||||
if (schemeEnd < B_OK) {
|
||||
fprintf(stderr, "the url does not contain the '://' string\n");
|
||||
return false;
|
||||
}
|
||||
|
||||
BString scheme;
|
||||
value.CopyInto(scheme, 0, schemeEnd);
|
||||
|
||||
if (scheme != "http" && scheme != "https") {
|
||||
fprintf(stderr, "the url scheme should be 'http' or 'https'\n");
|
||||
return false;
|
||||
}
|
||||
|
||||
if (value.Length() - 1 == value.FindLast("/")) {
|
||||
fprintf(stderr, "the url should be be terminated with a '/'\n");
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
/*! This method will set the web app base URL, returning a status to
|
||||
indicate if the URL was acceptable.
|
||||
\return B_OK if the base URL was valid and B_BAD_VALUE if not.
|
||||
*/
|
||||
status_t
|
||||
WebAppInterface::SetBaseUrl(const BString& url)
|
||||
{
|
||||
if (!arguments_is_url_valid(url))
|
||||
return B_BAD_VALUE;
|
||||
|
||||
fBaseUrl.SetTo(url);
|
||||
|
||||
return B_OK;
|
||||
}
|
||||
|
||||
|
||||
const BString
|
||||
WebAppInterface::_GetUserAgentVersionString()
|
||||
{
|
||||
app_info info;
|
||||
|
||||
if (be_app->GetAppInfo(&info) != B_OK) {
|
||||
fprintf(stderr, "Unable to get the application info\n");
|
||||
be_app->Quit();
|
||||
return BString(USERAGENT_FALLBACK_VERSION);
|
||||
}
|
||||
|
||||
BFile file(&info.ref, B_READ_ONLY);
|
||||
|
||||
if (file.InitCheck() != B_OK) {
|
||||
fprintf(stderr, "Unable to access the application info file\n");
|
||||
be_app->Quit();
|
||||
return BString(USERAGENT_FALLBACK_VERSION);
|
||||
}
|
||||
|
||||
BAppFileInfo appFileInfo(&file);
|
||||
version_info versionInfo;
|
||||
|
||||
if (appFileInfo.GetVersionInfo(
|
||||
&versionInfo, B_APP_VERSION_KIND) != B_OK) {
|
||||
fprintf(stderr, "Unable to establish the application version\n");
|
||||
be_app->Quit();
|
||||
return BString(USERAGENT_FALLBACK_VERSION);
|
||||
}
|
||||
|
||||
BString result;
|
||||
result.SetToFormat("%" B_PRId32 ".%" B_PRId32 ".%" B_PRId32,
|
||||
versionInfo.major, versionInfo.middle, versionInfo.minor);
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
/*! This method will devise a suitable User-Agent header value that
|
||||
can be transmitted with HTTP requests to the server in order
|
||||
to identify this client.
|
||||
*/
|
||||
const BString
|
||||
WebAppInterface::_GetUserAgent()
|
||||
{
|
||||
AutoLocker<BLocker> lock(&fUserAgentLocker);
|
||||
|
||||
if (fUserAgent.IsEmpty()) {
|
||||
fUserAgent.SetTo("HaikuDepot/");
|
||||
fUserAgent.Append(_GetUserAgentVersionString());
|
||||
}
|
||||
|
||||
return fUserAgent;
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
WebAppInterface::SetPreferredLanguage(const BString& language)
|
||||
{
|
||||
|
|
@ -509,7 +386,6 @@ WebAppInterface::RetrieveBulkPackageInfo(const StringList& packageNames,
|
|||
.AddArray("filter")
|
||||
.AddItem("PKGCATEGORIES")
|
||||
.AddItem("PKGSCREENSHOTS")
|
||||
.AddItem("PKGICONS")
|
||||
.AddItem("PKGVERSIONLOCALIZATIONDESCRIPTIONS")
|
||||
.AddItem("PKGCHANGELOG")
|
||||
.EndArray()
|
||||
|
|
@ -521,37 +397,6 @@ WebAppInterface::RetrieveBulkPackageInfo(const StringList& packageNames,
|
|||
}
|
||||
|
||||
|
||||
status_t
|
||||
WebAppInterface::RetrievePackageIcon(const BString& packageName,
|
||||
BDataIO* stream)
|
||||
{
|
||||
BString urlString = _FormFullUrl(BString("/__pkgicon/") << packageName
|
||||
<< ".hvif");
|
||||
bool isSecure = 0 == urlString.FindFirst("https://");
|
||||
|
||||
BUrl url(urlString);
|
||||
|
||||
ProtocolListener listener;
|
||||
listener.SetDownloadIO(stream);
|
||||
|
||||
BHttpRequest request(url, isSecure, "HTTP", &listener);
|
||||
request.SetMethod(B_HTTP_GET);
|
||||
|
||||
thread_id thread = request.Run();
|
||||
wait_for_thread(thread, NULL);
|
||||
|
||||
const BHttpResult& result = dynamic_cast<const BHttpResult&>(
|
||||
request.Result());
|
||||
|
||||
int32 statusCode = result.StatusCode();
|
||||
|
||||
if (statusCode == 200)
|
||||
return B_OK;
|
||||
|
||||
return B_ERROR;
|
||||
}
|
||||
|
||||
|
||||
status_t
|
||||
WebAppInterface::RetrieveUserRatings(const BString& packageName,
|
||||
const BString& architecture, int resultOffset, int maxResults,
|
||||
|
|
@ -671,17 +516,18 @@ status_t
|
|||
WebAppInterface::RetrieveScreenshot(const BString& code,
|
||||
int32 width, int32 height, BDataIO* stream)
|
||||
{
|
||||
BString urlString = _FormFullUrl(BString("/__pkgscreenshot/") << code
|
||||
BString urlString = ServerSettings::CreateFullUrl(BString("/__pkgscreenshot/") << code
|
||||
<< ".png" << "?tw=" << width << "&th=" << height);
|
||||
bool isSecure = 0 == urlString.FindFirst("https://");
|
||||
|
||||
BUrl url(urlString);
|
||||
|
||||
ProtocolListener listener;
|
||||
ProtocolListener listener(
|
||||
ServerSettings::UrlConnectionTraceLoggingEnabled());
|
||||
listener.SetDownloadIO(stream);
|
||||
|
||||
BHttpHeaders headers;
|
||||
headers.AddHeader("User-Agent", _GetUserAgent());
|
||||
ServerSettings::AugmentHeaders(headers);
|
||||
|
||||
BHttpRequest request(url, isSecure, "HTTP", &listener);
|
||||
request.SetMethod(B_HTTP_GET);
|
||||
|
|
@ -776,35 +622,24 @@ WebAppInterface::AuthenticateUser(const BString& nickName,
|
|||
// #pragma mark - private
|
||||
|
||||
|
||||
BString
|
||||
WebAppInterface::_FormFullUrl(const BString& suffix) const
|
||||
{
|
||||
if (fBaseUrl.IsEmpty()) {
|
||||
fprintf(stderr, "illegal state - missing web app base url\n");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
return BString(fBaseUrl) << suffix;
|
||||
}
|
||||
|
||||
|
||||
status_t
|
||||
WebAppInterface::_SendJsonRequest(const char* domain, BString jsonString,
|
||||
uint32 flags, BMessage& reply) const
|
||||
{
|
||||
if ((flags & ENABLE_DEBUG) != 0)
|
||||
if (ServerSettings::UrlConnectionTraceLoggingEnabled())
|
||||
printf("_SendJsonRequest(%s)\n", jsonString.String());
|
||||
|
||||
BString urlString = _FormFullUrl(BString("/__api/v1/") << domain);
|
||||
BString urlString = ServerSettings::CreateFullUrl(BString("/__api/v1/") << domain);
|
||||
bool isSecure = 0 == urlString.FindFirst("https://");
|
||||
BUrl url(urlString);
|
||||
|
||||
ProtocolListener listener;
|
||||
ProtocolListener listener(
|
||||
ServerSettings::UrlConnectionTraceLoggingEnabled());
|
||||
BUrlContext context;
|
||||
|
||||
BHttpHeaders headers;
|
||||
headers.AddHeader("Content-Type", "application/json");
|
||||
headers.AddHeader("User-Agent", _GetUserAgent());
|
||||
ServerSettings::AugmentHeaders(headers);
|
||||
|
||||
BHttpRequest request(url, isSecure, "HTTP", &listener, &context);
|
||||
request.SetMethod(B_HTTP_POST);
|
||||
|
|
@ -827,7 +662,6 @@ WebAppInterface::_SendJsonRequest(const char* domain, BString jsonString,
|
|||
|
||||
BMallocIO replyData;
|
||||
listener.SetDownloadIO(&replyData);
|
||||
listener.SetDebug((flags & ENABLE_DEBUG) != 0);
|
||||
|
||||
thread_id thread = request.Run();
|
||||
wait_for_thread(thread, NULL);
|
||||
|
|
@ -848,7 +682,8 @@ WebAppInterface::_SendJsonRequest(const char* domain, BString jsonString,
|
|||
|
||||
BJson parser;
|
||||
status_t status = parser.Parse(reply, jsonString);
|
||||
if ((flags & ENABLE_DEBUG) != 0 && status == B_BAD_DATA) {
|
||||
if (ServerSettings::UrlConnectionTraceLoggingEnabled() &&
|
||||
status == B_BAD_DATA) {
|
||||
printf("Parser choked on JSON:\n%s\n", jsonString.String());
|
||||
}
|
||||
return status;
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
/*
|
||||
* Copyright 2014, Stephan Aßmus <superstippi@gmx.de>.
|
||||
* Copyright 2016, Andrew Lindesay <apl@lindesay.co.nz>.
|
||||
* Copyright 2016-2017, Andrew Lindesay <apl@lindesay.co.nz>.
|
||||
* All rights reserved. Distributed under the terms of the MIT License.
|
||||
*/
|
||||
#ifndef WEB_APP_INTERFACE_H
|
||||
|
|
@ -34,7 +34,6 @@ public:
|
|||
const BString& Username() const
|
||||
{ return fUsername; }
|
||||
|
||||
static status_t SetBaseUrl(const BString& url);
|
||||
void SetPreferredLanguage(const BString& language);
|
||||
void SetArchitecture(const BString& architecture);
|
||||
|
||||
|
|
@ -54,10 +53,6 @@ public:
|
|||
const StringList& repositoryCodes,
|
||||
BMessage& message);
|
||||
|
||||
status_t RetrievePackageIcon(
|
||||
const BString& packageName,
|
||||
BDataIO* stream);
|
||||
|
||||
status_t RetrieveUserRatings(
|
||||
const BString& packageName,
|
||||
const BString& architecture,
|
||||
|
|
@ -110,17 +105,11 @@ public:
|
|||
BMessage& message);
|
||||
|
||||
private:
|
||||
static const BString _GetUserAgentVersionString();
|
||||
static const BString _GetUserAgent();
|
||||
BString _FormFullUrl(const BString& suffix) const;
|
||||
status_t _SendJsonRequest(const char* domain,
|
||||
BString jsonString, uint32 flags,
|
||||
BMessage& reply) const;
|
||||
|
||||
private:
|
||||
static BString fBaseUrl;
|
||||
static BString fUserAgent;
|
||||
static BLocker fUserAgentLocker;
|
||||
BString fUsername;
|
||||
BString fPassword;
|
||||
BString fLanguage;
|
||||
|
|
@ -0,0 +1,148 @@
|
|||
/*
|
||||
* Copyright 2017, Andrew Lindesay <apl@lindesay.co.nz>.
|
||||
* All rights reserved. Distributed under the terms of the MIT License.
|
||||
*/
|
||||
|
||||
#include "TarArchiveHeader.h"
|
||||
|
||||
#include <stdio.h>
|
||||
|
||||
|
||||
#define OFFSET_FILENAME 0
|
||||
#define OFFSET_LENGTH 124
|
||||
#define OFFSET_CHECKSUM 148
|
||||
#define OFFSET_FILETYPE 156
|
||||
|
||||
#define LENGTH_FILENAME 100
|
||||
#define LENGTH_LENGTH 12
|
||||
#define LENGTH_CHECKSUM 8
|
||||
#define LENGTH_BLOCK 512
|
||||
|
||||
|
||||
TarArchiveHeader::TarArchiveHeader(const BString& fileName, uint64 length,
|
||||
tar_file_type fileType)
|
||||
:
|
||||
fFileName(fileName),
|
||||
fLength(length),
|
||||
fFileType(fileType)
|
||||
{
|
||||
}
|
||||
|
||||
tar_file_type
|
||||
TarArchiveHeader::_ReadFileType(unsigned char data) {
|
||||
switch (data) {
|
||||
case 0:
|
||||
case '0':
|
||||
return TAR_FILE_TYPE_NORMAL;
|
||||
|
||||
default:
|
||||
return TAR_FILE_TYPE_OTHER;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
const BString
|
||||
TarArchiveHeader::_ReadString(const unsigned char *data, size_t dataLength)
|
||||
{
|
||||
uint32 actualLength = 0;
|
||||
|
||||
while (actualLength < dataLength && 0 != data[actualLength])
|
||||
actualLength++;
|
||||
|
||||
return BString((const char *) data, actualLength);
|
||||
}
|
||||
|
||||
/*
|
||||
* This is an octal value represented as an ASCII string.
|
||||
*/
|
||||
|
||||
static bool tar_is_octal_digit(unsigned char c)
|
||||
{
|
||||
switch (c) {
|
||||
case '0':
|
||||
case '1':
|
||||
case '2':
|
||||
case '3':
|
||||
case '4':
|
||||
case '5':
|
||||
case '6':
|
||||
case '7':
|
||||
return true;
|
||||
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
uint32
|
||||
TarArchiveHeader::_ReadNumeric(const unsigned char *data, size_t dataLength)
|
||||
{
|
||||
uint32 actualLength = 0;
|
||||
|
||||
while (actualLength < dataLength && tar_is_octal_digit(data[actualLength]))
|
||||
actualLength++;
|
||||
|
||||
uint32 factor = 1;
|
||||
uint32 result = 0;
|
||||
|
||||
for (uint32 i = 0; i < actualLength; i++) {
|
||||
result += (data[actualLength - (1 + i)] - '0') * factor;
|
||||
factor *= 8;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
uint32
|
||||
TarArchiveHeader::_CalculateChecksum(const unsigned char* block)
|
||||
{
|
||||
uint32 result = 0;
|
||||
|
||||
for (uint32 i = 0; i < LENGTH_BLOCK; i++) {
|
||||
if (i >= OFFSET_CHECKSUM && i < OFFSET_CHECKSUM + LENGTH_CHECKSUM)
|
||||
result += 32;
|
||||
else
|
||||
result += block[i];
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
TarArchiveHeader*
|
||||
TarArchiveHeader::CreateFromBlock(const unsigned char* block)
|
||||
{
|
||||
uint32 actualChecksum = _CalculateChecksum(block);
|
||||
uint32 expectedChecksum = _ReadNumeric(&block[OFFSET_CHECKSUM],
|
||||
LENGTH_CHECKSUM);
|
||||
|
||||
if(actualChecksum != expectedChecksum) {
|
||||
fprintf(stderr, "tar archive header has bad checksum;"
|
||||
"expected %zu actual %zu\n", expectedChecksum, actualChecksum);
|
||||
} else {
|
||||
return new TarArchiveHeader(
|
||||
_ReadString(&block[OFFSET_FILENAME], LENGTH_FILENAME),
|
||||
_ReadNumeric(&block[OFFSET_LENGTH], LENGTH_LENGTH),
|
||||
_ReadFileType(block[OFFSET_FILETYPE]));
|
||||
}
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
const BString&
|
||||
TarArchiveHeader::GetFileName() const
|
||||
{
|
||||
return fFileName;
|
||||
}
|
||||
|
||||
size_t
|
||||
TarArchiveHeader::GetLength() const
|
||||
{
|
||||
return fLength;
|
||||
}
|
||||
|
||||
|
||||
tar_file_type
|
||||
TarArchiveHeader::GetFileType() const
|
||||
{
|
||||
return fFileType;
|
||||
}
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
/*
|
||||
* Copyright 2017, Andrew Lindesay <apl@lindesay.co.nz>.
|
||||
* All rights reserved. Distributed under the terms of the MIT License.
|
||||
*/
|
||||
|
||||
#ifndef TAR_ARCHIVE_HEADER_H
|
||||
#define TAR_ARCHIVE_HEADER_H
|
||||
|
||||
#include <String.h>
|
||||
|
||||
|
||||
enum tar_file_type {
|
||||
TAR_FILE_TYPE_NORMAL,
|
||||
TAR_FILE_TYPE_OTHER
|
||||
};
|
||||
|
||||
|
||||
/* Each file in a tar-archive has a header on it describing the next entry in
|
||||
* the stream. This class models the data in the header.
|
||||
*/
|
||||
|
||||
class TarArchiveHeader {
|
||||
public:
|
||||
TarArchiveHeader(const BString& fileName,
|
||||
uint64 length, tar_file_type fileType);
|
||||
|
||||
static TarArchiveHeader* CreateFromBlock(const unsigned char *block);
|
||||
|
||||
const BString& GetFileName() const;
|
||||
size_t GetLength() const;
|
||||
tar_file_type GetFileType() const;
|
||||
|
||||
private:
|
||||
static uint32 _CalculateChecksum(
|
||||
const unsigned char* data);
|
||||
static const BString _ReadString(const unsigned char* data,
|
||||
size_t dataLength);
|
||||
static uint32 _ReadNumeric(const unsigned char* data,
|
||||
size_t dataLength);
|
||||
static tar_file_type _ReadFileType(unsigned char data);
|
||||
|
||||
const BString fFileName;
|
||||
uint64 fLength;
|
||||
tar_file_type fFileType;
|
||||
|
||||
};
|
||||
|
||||
#endif // TAR_ARCHIVE_HEADER_H
|
||||
|
|
@ -0,0 +1,185 @@
|
|||
/*
|
||||
* Copyright 2017, Andrew Lindesay <apl@lindesay.co.nz>.
|
||||
* All rights reserved. Distributed under the terms of the MIT License.
|
||||
*/
|
||||
|
||||
#include "TarArchiveService.h"
|
||||
#include "StorageUtils.h"
|
||||
|
||||
#include <stdio.h>
|
||||
|
||||
#include <Directory.h>
|
||||
#include <File.h>
|
||||
#include <StringList.h>
|
||||
|
||||
|
||||
#define LENGTH_BLOCK 512
|
||||
|
||||
|
||||
status_t
|
||||
TarArchiveService::Unpack(BDataIO& tarDataIo, BPath& targetDirectory)
|
||||
{
|
||||
uint8 buffer[LENGTH_BLOCK];
|
||||
uint8 zero_buffer[LENGTH_BLOCK];
|
||||
status_t result = B_OK;
|
||||
uint32_t count_items_read = 0;
|
||||
|
||||
fprintf(stdout, "will unpack to [%s]\n", targetDirectory.Path());
|
||||
|
||||
memset(zero_buffer, 0, sizeof zero_buffer);
|
||||
|
||||
while (B_OK == result && B_OK == (result = tarDataIo.ReadExactly(buffer,
|
||||
LENGTH_BLOCK))) {
|
||||
|
||||
count_items_read++;
|
||||
|
||||
if (0 == memcmp(zero_buffer, buffer, sizeof zero_buffer)) {
|
||||
fprintf(stdout, "detected end of tar-ball\n");
|
||||
return B_OK; // end of tar-ball.
|
||||
} else {
|
||||
TarArchiveHeader* header = TarArchiveHeader::CreateFromBlock(
|
||||
buffer);
|
||||
|
||||
if (NULL == header) {
|
||||
fprintf(stderr, "unable to parse a tar header\n");
|
||||
result = B_ERROR;
|
||||
}
|
||||
|
||||
if (B_OK == result)
|
||||
result = _UnpackItem(tarDataIo, targetDirectory, *header);
|
||||
|
||||
delete header;
|
||||
}
|
||||
}
|
||||
|
||||
fprintf(stdout, "did unpack %d tar items\n", count_items_read);
|
||||
|
||||
if (B_OK != result) {
|
||||
fprintf(stdout, "error occurred unpacking tar items; %s\n",
|
||||
strerror(result));
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
status_t
|
||||
TarArchiveService::_EnsurePathToTarItemFile(
|
||||
BPath& targetDirectoryPath, BString &tarItemPath)
|
||||
{
|
||||
if (tarItemPath.Length() == 0)
|
||||
return B_ERROR;
|
||||
|
||||
BStringList components;
|
||||
tarItemPath.Split("/", false, components);
|
||||
|
||||
for (int32 i = 0; i < components.CountStrings(); i++) {
|
||||
BString component = components.StringAt(i);
|
||||
|
||||
if (_ValidatePathComponent(component) != B_OK) {
|
||||
fprintf(stdout, "malformed component; [%s]\n", component.String());
|
||||
return B_ERROR;
|
||||
}
|
||||
}
|
||||
|
||||
BPath parentPath;
|
||||
BPath assembledPath(targetDirectoryPath);
|
||||
|
||||
status_t result = assembledPath.Append(tarItemPath);
|
||||
|
||||
if (result == B_OK)
|
||||
result = assembledPath.GetParent(&parentPath);
|
||||
|
||||
if (result == B_OK)
|
||||
result = create_directory(parentPath.Path(), 0777);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
status_t
|
||||
TarArchiveService::_UnpackItem(BDataIO& tarDataIo,
|
||||
BPath& targetDirectoryPath,
|
||||
TarArchiveHeader& header)
|
||||
{
|
||||
status_t result = B_OK;
|
||||
BString entryFileName = header.GetFileName();
|
||||
uint32 entryLength = header.GetLength();
|
||||
|
||||
fprintf(stdout, "will unpack item [%s] length [%zu]b\n",
|
||||
entryFileName.String(), entryLength);
|
||||
|
||||
// if the path ends in "/" then it is a directory and there's no need to
|
||||
// unpack it although if there is a length, it will need to be skipped.
|
||||
|
||||
if (!entryFileName.EndsWith("/") ||
|
||||
header.GetFileType() != TAR_FILE_TYPE_NORMAL) {
|
||||
|
||||
result = _EnsurePathToTarItemFile(targetDirectoryPath,
|
||||
entryFileName);
|
||||
|
||||
if (result == B_OK) {
|
||||
BPath targetFilePath(targetDirectoryPath);
|
||||
targetFilePath.Append(entryFileName, false);
|
||||
result = _UnpackItemData(tarDataIo, targetFilePath, entryLength);
|
||||
}
|
||||
} else {
|
||||
off_t blocksToSkip = (entryLength / LENGTH_BLOCK);
|
||||
|
||||
if (entryLength % LENGTH_BLOCK > 0)
|
||||
blocksToSkip += 1;
|
||||
|
||||
if (0 != blocksToSkip) {
|
||||
uint8 buffer[LENGTH_BLOCK];
|
||||
|
||||
for (uint32 i = 0; B_OK == result && i < blocksToSkip; i++)
|
||||
tarDataIo.ReadExactly(buffer, LENGTH_BLOCK);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
status_t
|
||||
TarArchiveService::_UnpackItemData(BDataIO& tarDataIo,
|
||||
BPath& targetFilePath, uint32 length)
|
||||
{
|
||||
uint8 buffer[LENGTH_BLOCK];
|
||||
size_t remainingInItem = length;
|
||||
status_t result = B_OK;
|
||||
BFile targetFile(targetFilePath.Path(), O_WRONLY | O_CREAT);
|
||||
|
||||
while (remainingInItem > 0 &&
|
||||
B_OK == result &&
|
||||
B_OK == (result = tarDataIo.ReadExactly(buffer, LENGTH_BLOCK))) {
|
||||
|
||||
size_t writeFromBuffer = LENGTH_BLOCK;
|
||||
|
||||
if (remainingInItem < LENGTH_BLOCK)
|
||||
writeFromBuffer = remainingInItem;
|
||||
|
||||
result = targetFile.WriteExactly(buffer, writeFromBuffer);
|
||||
remainingInItem -= writeFromBuffer;
|
||||
}
|
||||
|
||||
if (result != B_OK)
|
||||
fprintf(stdout, "unable to unpack item data to; [%s]\n",
|
||||
targetFilePath.Path());
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
status_t
|
||||
TarArchiveService::_ValidatePathComponent(const BString& component)
|
||||
{
|
||||
if (component.Length() == 0)
|
||||
return B_ERROR;
|
||||
|
||||
if (component == ".." || component == "." || component == "~")
|
||||
return B_ERROR;
|
||||
|
||||
return B_OK;
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
/*
|
||||
* Copyright 2017, Andrew Lindesay <apl@lindesay.co.nz>.
|
||||
* All rights reserved. Distributed under the terms of the MIT License.
|
||||
*/
|
||||
|
||||
#ifndef TAR_ARCHIVE_SERVICE_H
|
||||
#define TAR_ARCHIVE_SERVICE_H
|
||||
|
||||
#include "AbstractServerProcess.h"
|
||||
#include "TarArchiveHeader.h"
|
||||
|
||||
#include <String.h>
|
||||
#include <Path.h>
|
||||
|
||||
|
||||
class TarArchiveService {
|
||||
public:
|
||||
static status_t Unpack(BDataIO& tarDataIo,
|
||||
BPath& targetDirectoryPath);
|
||||
|
||||
private:
|
||||
static status_t _EnsurePathToTarItemFile(
|
||||
BPath& targetDirectoryPath,
|
||||
BString &tarItemPath);
|
||||
static status_t _ValidatePathComponent(
|
||||
const BString& component);
|
||||
static status_t _UnpackItem(BDataIO& tarDataIo,
|
||||
BPath& targetDirectory,
|
||||
TarArchiveHeader& header);
|
||||
static status_t _UnpackItemData(BDataIO& tarDataIo,
|
||||
BPath& targetFilePath,
|
||||
uint32 length);
|
||||
|
||||
};
|
||||
|
||||
#endif // TAR_ARCHIVE_SERVICE_H
|
||||
|
|
@ -5,6 +5,7 @@
|
|||
|
||||
|
||||
#include "App.h"
|
||||
#include "ServerSettings.h"
|
||||
|
||||
#include <stdio.h>
|
||||
|
||||
|
|
@ -117,29 +118,113 @@ App::RefsReceived(BMessage* message)
|
|||
}
|
||||
|
||||
|
||||
enum arg_switch {
|
||||
UNKNOWN_SWITCH,
|
||||
NOT_SWITCH,
|
||||
HELP_SWITCH,
|
||||
WEB_APP_BASE_URL_SWITCH,
|
||||
URL_CONNECTION_TRACE_LOGGING_SWITCH,
|
||||
};
|
||||
|
||||
|
||||
static void
|
||||
app_print_help()
|
||||
{
|
||||
fprintf(stdout, "HaikuDepot ");
|
||||
fprintf(stdout, "[-u|--webappbaseurl <web-app-base-url>] ");
|
||||
fprintf(stdout, "[-h] ");
|
||||
fprintf(stdout, "[-t|--urlconnectiontracelogging]\n");
|
||||
}
|
||||
|
||||
|
||||
static arg_switch
|
||||
app_resolve_switch(char *arg)
|
||||
{
|
||||
int arglen = strlen(arg);
|
||||
|
||||
if (arglen > 0 && arg[0] == '-') {
|
||||
|
||||
if (arglen > 3 && arg[1] == '-') { // long form
|
||||
if (0 == strcmp(&arg[2], "webappbaseurl"))
|
||||
return WEB_APP_BASE_URL_SWITCH;
|
||||
|
||||
if (0 == strcmp(&arg[2], "help"))
|
||||
return HELP_SWITCH;
|
||||
|
||||
if (0 == strcmp(&arg[2], "urlconnectiontracelogging"))
|
||||
return URL_CONNECTION_TRACE_LOGGING_SWITCH;
|
||||
} else {
|
||||
if (arglen == 2) { // short form
|
||||
switch (arg[1]) {
|
||||
case 'u':
|
||||
return WEB_APP_BASE_URL_SWITCH;
|
||||
|
||||
case 'h':
|
||||
return HELP_SWITCH;
|
||||
|
||||
case 't':
|
||||
return URL_CONNECTION_TRACE_LOGGING_SWITCH;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return UNKNOWN_SWITCH;
|
||||
}
|
||||
|
||||
return NOT_SWITCH;
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
App::ArgvReceived(int32 argc, char* argv[])
|
||||
{
|
||||
for (int i = 1; i < argc;) {
|
||||
if (0 == strcmp("--webappbaseurl", argv[i])) {
|
||||
if (i == argc-1) {
|
||||
fprintf(stderr, "unexpected end of arguments; missing web app base url\n");
|
||||
switch (app_resolve_switch(argv[i])) {
|
||||
|
||||
case URL_CONNECTION_TRACE_LOGGING_SWITCH:
|
||||
ServerSettings::EnableUrlConnectionTraceLogging();
|
||||
break;
|
||||
|
||||
case HELP_SWITCH:
|
||||
app_print_help();
|
||||
Quit();
|
||||
break;
|
||||
|
||||
case WEB_APP_BASE_URL_SWITCH:
|
||||
if (i == argc-1) {
|
||||
fprintf(stdout, "unexpected end of arguments; missing "
|
||||
"web-app base url\n");
|
||||
Quit();
|
||||
}
|
||||
|
||||
if (ServerSettings::SetBaseUrl(argv[i + 1]) != B_OK) {
|
||||
fprintf(stdout, "malformed web app base url; %s\n",
|
||||
argv[i + 1]);
|
||||
Quit();
|
||||
}
|
||||
else {
|
||||
fprintf(stdout, "did configure the web base url; %s\n",
|
||||
argv[i + 1]);
|
||||
}
|
||||
|
||||
i++; // also move past the url value
|
||||
|
||||
break;
|
||||
|
||||
case NOT_SWITCH:
|
||||
{
|
||||
BEntry entry(argv[i], true);
|
||||
_Open(entry);
|
||||
break;
|
||||
}
|
||||
|
||||
if (WebAppInterface::SetBaseUrl(argv[i + 1]) != B_OK) {
|
||||
fprintf(stderr, "malformed web app base url; %s\n", argv[i + 1]);
|
||||
case UNKNOWN_SWITCH:
|
||||
fprintf(stdout, "unknown switch; %s\n", argv[i]);
|
||||
Quit();
|
||||
}
|
||||
else
|
||||
fprintf(stderr, "did configure the web base url; %s\n",argv[i + 1]);
|
||||
|
||||
i += 2;
|
||||
} else {
|
||||
BEntry entry(argv[i], true);
|
||||
_Open(entry);
|
||||
i++;
|
||||
break;
|
||||
}
|
||||
|
||||
i++; // move on at least one arg
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,105 @@
|
|||
/*
|
||||
* Copyright 2017, Andrew Lindesay <apl@lindesay.co.nz>.
|
||||
* All rights reserved. Distributed under the terms of the MIT License.
|
||||
*/
|
||||
|
||||
#include "StorageUtils.h"
|
||||
|
||||
#include <Directory.h>
|
||||
#include <File.h>
|
||||
#include <Entry.h>
|
||||
#include <String.h>
|
||||
|
||||
#include <stdio.h>
|
||||
#include <errno.h>
|
||||
|
||||
#define FILE_TO_STRING_BUFFER_LEN 64
|
||||
|
||||
|
||||
/* This method will append the contents of the file at the supplied path to the
|
||||
* string provided.
|
||||
*/
|
||||
|
||||
status_t
|
||||
StorageUtils::AppendToString(BPath& path, BString& result)
|
||||
{
|
||||
BFile file(path.Path(), O_RDONLY);
|
||||
uint8_t buffer[FILE_TO_STRING_BUFFER_LEN];
|
||||
size_t buffer_read;
|
||||
|
||||
while((buffer_read = file.Read(buffer, FILE_TO_STRING_BUFFER_LEN)) > 0)
|
||||
result.Append((char *) buffer, buffer_read);
|
||||
|
||||
return (status_t) buffer_read;
|
||||
}
|
||||
|
||||
|
||||
/* This method will traverse the directory structure and will remove all of the
|
||||
* files that are present in the directories as well as the directories
|
||||
* themselves.
|
||||
*/
|
||||
|
||||
status_t
|
||||
StorageUtils::RemoveDirectoryContents(BPath& path)
|
||||
{
|
||||
BDirectory directory(path.Path());
|
||||
BEntry directoryEntry;
|
||||
status_t result = B_OK;
|
||||
|
||||
while (result == B_OK &&
|
||||
directory.GetNextEntry(&directoryEntry) != B_ENTRY_NOT_FOUND) {
|
||||
|
||||
bool exists = false;
|
||||
bool isDirectory = false;
|
||||
BPath directroyEntryPath;
|
||||
|
||||
result = directoryEntry.GetPath(&directroyEntryPath);
|
||||
|
||||
if (result == B_OK)
|
||||
result = ExistsDirectory(directroyEntryPath, &exists, &isDirectory);
|
||||
|
||||
if (result == B_OK) {
|
||||
if (isDirectory)
|
||||
RemoveDirectoryContents(directroyEntryPath);
|
||||
|
||||
if (remove(directroyEntryPath.Path()) == 0) {
|
||||
fprintf(stdout, "did delete [%s]\n",
|
||||
directroyEntryPath.Path());
|
||||
} else {
|
||||
fprintf(stderr, "unable to delete [%s]\n",
|
||||
directroyEntryPath.Path());
|
||||
result = B_ERROR;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
/* This method checks to see if a file object exists at the path specified. If
|
||||
* something does exist then the value of the 'exists' pointer is set to true.
|
||||
* If the object is a directory then this value is also set to true.
|
||||
*/
|
||||
|
||||
status_t
|
||||
StorageUtils::ExistsDirectory(BPath& directory,
|
||||
bool* exists,
|
||||
bool* isDirectory)
|
||||
{
|
||||
struct stat s;
|
||||
|
||||
*exists = false;
|
||||
*isDirectory = false;
|
||||
|
||||
if (-1 == stat(directory.Path(), &s)) {
|
||||
if (ENOENT != errno)
|
||||
return B_ERROR;
|
||||
} else {
|
||||
*exists = true;
|
||||
*isDirectory = S_ISDIR(s.st_mode);
|
||||
}
|
||||
|
||||
return B_OK;
|
||||
}
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
/*
|
||||
* Copyright 2017, Andrew Lindesay <apl@lindesay.co.nz>.
|
||||
* All rights reserved. Distributed under the terms of the MIT License.
|
||||
*/
|
||||
|
||||
#ifndef PATH_UTILS_H
|
||||
#define PATH_UTILS_H
|
||||
|
||||
#include <Path.h>
|
||||
|
||||
class StorageUtils {
|
||||
|
||||
public:
|
||||
static status_t RemoveDirectoryContents(BPath& path);
|
||||
static status_t AppendToString(BPath& path, BString& result);
|
||||
static status_t ExistsDirectory(BPath& directory,
|
||||
bool* exists,
|
||||
bool* isDirectory);
|
||||
};
|
||||
|
||||
#endif // PATH_UTILS_H
|
||||
|
|
@ -0,0 +1,100 @@
|
|||
/*
|
||||
* Copyright 2017, Andrew Lindesay <apl@lindesay.co.nz>.
|
||||
* All rights reserved. Distributed under the terms of the MIT License.
|
||||
*/
|
||||
|
||||
#include "ToFileUrlProtocolListener.h"
|
||||
|
||||
#include <File.h>
|
||||
|
||||
#include <stdio.h>
|
||||
|
||||
|
||||
ToFileUrlProtocolListener::ToFileUrlProtocolListener(BPath path,
|
||||
BString traceLoggingIdentifier, bool traceLogging)
|
||||
{
|
||||
fDownloadIO = new BFile(path.Path(), O_WRONLY | O_CREAT);
|
||||
fTraceLoggingIdentifier = traceLoggingIdentifier;
|
||||
fTraceLogging = traceLogging;
|
||||
}
|
||||
|
||||
|
||||
ToFileUrlProtocolListener::~ToFileUrlProtocolListener()
|
||||
{
|
||||
delete fDownloadIO;
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
ToFileUrlProtocolListener::ConnectionOpened(BUrlRequest* caller)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
ToFileUrlProtocolListener::HostnameResolved(BUrlRequest* caller,
|
||||
const char* ip)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
ToFileUrlProtocolListener::ResponseStarted(BUrlRequest* caller)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
ToFileUrlProtocolListener::HeadersReceived(BUrlRequest* caller)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
ToFileUrlProtocolListener::DataReceived(BUrlRequest* caller, const char* data,
|
||||
off_t position, ssize_t size)
|
||||
{
|
||||
if (fDownloadIO != NULL && size > 0) {
|
||||
size_t remaining = size;
|
||||
size_t written = 0;
|
||||
|
||||
do {
|
||||
written = fDownloadIO->Write(&data[size - remaining], remaining);
|
||||
remaining -= written;
|
||||
} while (remaining > 0 && written > 0);
|
||||
|
||||
if (remaining > 0)
|
||||
fprintf(stdout, "unable to write all of the data to the file\n");
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
ToFileUrlProtocolListener::DownloadProgress(BUrlRequest* caller,
|
||||
ssize_t bytesReceived, ssize_t bytesTotal)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
ToFileUrlProtocolListener::UploadProgress(BUrlRequest* caller,
|
||||
ssize_t bytesSent, ssize_t bytesTotal)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
ToFileUrlProtocolListener::RequestCompleted(BUrlRequest* caller, bool success)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
ToFileUrlProtocolListener::DebugMessage(BUrlRequest* caller,
|
||||
BUrlProtocolDebugMessage type, const char* text)
|
||||
{
|
||||
if (fTraceLogging) {
|
||||
fprintf(stdout, "url->file <%s>; %s\n",
|
||||
fTraceLoggingIdentifier.String(), text);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
/*
|
||||
* Copyright 2017, Andrew Lindesay <apl@lindesay.co.nz>.
|
||||
* All rights reserved. Distributed under the terms of the MIT License.
|
||||
*/
|
||||
|
||||
#include <UrlProtocolListener.h>
|
||||
#include <UrlRequest.h>
|
||||
|
||||
class ToFileUrlProtocolListener : public BUrlProtocolListener {
|
||||
public:
|
||||
ToFileUrlProtocolListener(BPath path,
|
||||
BString traceLoggingIdentifier,
|
||||
bool traceLogging);
|
||||
virtual ~ToFileUrlProtocolListener();
|
||||
|
||||
void ConnectionOpened(BUrlRequest* caller);
|
||||
void HostnameResolved(BUrlRequest* caller,
|
||||
const char* ip);
|
||||
void ResponseStarted(BUrlRequest* caller);
|
||||
void HeadersReceived(BUrlRequest* caller);
|
||||
void DataReceived(BUrlRequest* caller,
|
||||
const char* data, off_t position,
|
||||
ssize_t size);
|
||||
void DownloadProgress(BUrlRequest* caller,
|
||||
ssize_t bytesReceived, ssize_t bytesTotal);
|
||||
void UploadProgress(BUrlRequest* caller,
|
||||
ssize_t bytesSent, ssize_t bytesTotal);
|
||||
void RequestCompleted(BUrlRequest* caller,
|
||||
bool success);
|
||||
void DebugMessage(BUrlRequest* caller,
|
||||
BUrlProtocolDebugMessage type,
|
||||
const char* text);
|
||||
|
||||
private:
|
||||
bool fTraceLogging;
|
||||
BString fTraceLoggingIdentifier;
|
||||
BDataIO* fDownloadIO;
|
||||
|
||||
|
||||
};
|
||||
Loading…
Reference in New Issue