Aren/haiku
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

repo rework: Remove need for repos to be self-aware

Browse Source 
* See #12917 for details.
* Squashed to one commit to make revert easy if we
  run into any issues.
* pkgman is now pre-attached to the 'current' repo
  version within nightly images so they can be updated
  by default.
* This shouldn't impact us keeping older sets of package
  versions by commit hash for building older hrevs.
* There are XXX stubs with "Kill me". These will need
  to be dropped after users are given sufficent time to
  upgrade. We're dropping a previously required field (url)
  so making this a slowish roll out.
* Makes the repos a lot less restrictive which should
  help PM package building automation be a bit easier.
* Once this stuff smooths out, we'll add UUID's to the
  repo definitions for duplicate repo detection.
This commit is contained in:
Alexander von Gluck IV 2016-12-02 00:41:06 -06:00
parent 28fb504bb8
commit 5ffaf72c8a
15 changed files with 184 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
build/jam/ImageRules
View File

@ -901,7 +901,7 @@ rule AddPackagesAndRepositoryVariablesToContainerScript script : container
local repositoryFiles ;
for repository in $(HAIKU_REPOSITORIES) {
repositoryFiles
+= [ on $(repository) return $(HAIKU_REPOSITORY_CACHE_FILE) ] ;
+= [ on $(repository) return $(HAIKU_REPOSITORY_CONFIG_FILE) ] ;
}
AddTargetVariableToScript $(script) : $(repositoryFiles) : repositories ;

9
build/jam/RepositoryRules
View File

@ -761,10 +761,10 @@ actions BuildRemoteHaikuPortsRepository1
}
rule HaikuRepository repository : repoInfoTemplate : packages
rule HaikuRepository repository : repoInfoTemplate : packages : url
{
# HaikuRepository <repository> : <repoInfoTemplate> : <packages>
# [ : <url> [ : <versionFile> ] ] ;
# HaikuRepository <repository> : <repoInfoTemplate> : <packages> : <url>
# [ : <versionFile> ] ;
# Builds the Haiku repository from the given packages and repository info
# template. <repository> must already be located.
#
@ -773,6 +773,7 @@ rule HaikuRepository repository : repoInfoTemplate : packages
# meta data files.
# <repoInfoTemplate> - The repository info template file to be used.
# <packages> - The packages to be added to the repository.
# <url> - The remote location of the initial update repository
local architecture = $(HAIKU_PACKAGING_ARCH) ;
local secondaryArchitecture ;
@ -791,7 +792,7 @@ rule HaikuRepository repository : repoInfoTemplate : packages
# build the respository config
local repositoryConfig = $(repository:G=repository-config)-config ;
MakeLocate $(repositoryConfig) : $(repositoriesDir) ;
RepositoryConfig $(repositoryConfig) : $(repositoryInfo) ;
RepositoryConfig $(repositoryConfig) : $(repositoryInfo) : $(url) ;
HAIKU_REPOSITORY_CONFIG_FILE on $(repository) = $(repositoryConfig) ;
# setup the repository cache file

6
build/jam/repositories/Haiku
View File

@ -52,4 +52,8 @@ if $(webPositiveIsAvailable) {
packages += webpositive ;
}
HaikuRepository $(haikuRepository) : $(repoInfo) : $(packages:S=.hpkg) ;
local version = current ;
#local version = $(HAIKU_VERSION) ;
local remoteURL = http://packages.haiku-os.org/haiku/master/$(HAIKU_PACKAGING_ARCH)/$(version) ;
HaikuRepository $(haikuRepository) : $(repoInfo) : $(packages:S=.hpkg) : $(remoteURL) ;

2
build/scripts/build_haiku_image
View File

@ -7,7 +7,7 @@ set -o errexit
# tmpDir
# addBuildCompatibilityLibDir
# systemPackages - lists of the hpkg packages copied/updated
# repositories - all repository files
# repositories - all repository configuration files
# downloadDir
# The following are only for image types:
# installDir

7
headers/os/package/RepositoryInfo.h
View File

@ -33,7 +33,6 @@ public:
status_t InitCheck() const;
const BString& Name() const;
const BString& OriginalBaseURL() const;
const BString& Vendor() const;
const BString& Summary() const;
uint8 Priority() const;
@ -42,7 +41,6 @@ public:
const BStringList& LicenseTexts() const;
void SetName(const BString& name);
void SetOriginalBaseURL(const BString& url);
void SetVendor(const BString& vendor);
void SetSummary(const BString& summary);
void SetPriority(uint8 priority);
@ -57,8 +55,10 @@ public:
static const uint8 kDefaultPriority;
// XXX: Kill me after everyone upgrades
static const char* const kURLField;
static const char* const kNameField;
static const char* const kURLField;
static const char* const kVendorField;
static const char* const kSummaryField;
static const char* const kPriorityField;
@ -74,7 +74,6 @@ private:
status_t fInitStatus;
BString fName;
BString fOriginalBaseURL;
BString fVendor;
BString fSummary;
uint8 fPriority;

1
src/bin/package_repo/command_list.cpp
View File

@ -69,7 +69,6 @@ struct RepositoryContentListHandler : BRepositoryContentHandler {
printf("repository-info:\n");
printf("\tname: %s\n", repositoryInfo.Name().String());
printf("\tsummary: %s\n", repositoryInfo.Summary().String());
printf("\turl: %s\n", repositoryInfo.OriginalBaseURL().String());
printf("\tvendor: %s\n", repositoryInfo.Vendor().String());
printf("\tpriority: %u\n", repositoryInfo.Priority());
printf("\tarchitecture: %s\n",

2
src/bin/pkgman/command_list_repos.cpp
View File

@ -118,8 +118,6 @@ ListReposCommand::Execute(int argc, const char* const* argv)
repoCache.Info().Architecture()]);
printf("\t\tpkg-count: %" B_PRIu32 "\n",
repoCache.CountPackages());
printf("\t\torig-url: %s\n",
repoCache.Info().OriginalBaseURL().String());
printf("\t\torig-prio: %u\n", repoCache.Info().Priority());
} else
printf("\t\t<no repository cache found>\n");

1
src/data/repository_infos/haiku
View File

@ -2,5 +2,4 @@ name Haiku
vendor "Haiku Project"
summary "The Haiku repository (for Haiku %HAIKU_VERSION_NO_REVISION%)"
priority 1
url http://packages.haiku-os.org/haiku/master/%HAIKU_PACKAGING_ARCH%/%HAIKU_VERSION_NO_REVISION%
architecture %HAIKU_PACKAGING_ARCH%

1
src/data/repository_infos/haikuports
View File

@ -2,5 +2,4 @@ name HaikuPorts
vendor "Haiku Project"
summary "The HaikuPorts repository (for Haiku %HAIKU_VERSION_NO_REVISION%)"
priority 1
url http://packages.haiku-os.org/haikuports/master/repo/%HAIKU_PACKAGING_ARCH%/dummy
architecture %HAIKU_PACKAGING_ARCH%

36
src/kits/package/RepositoryInfo.cpp
View File

@ -1,9 +1,10 @@
/*
* Copyright 2011, Haiku, Inc. All Rights Reserved.
* Copyright 2011-2016, Haiku, Inc. All Rights Reserved.
* Distributed under the terms of the MIT License.
*
* Authors:
* Oliver Tappe <zooey@hirschkaefer.de>
* Alexander von Gluck IV <kallisti5@unixzen.com>
*/
@ -27,7 +28,6 @@ namespace BPackageKit {
const uint8 BRepositoryInfo::kDefaultPriority = 50;
const char* const BRepositoryInfo::kNameField = "name";
const char* const BRepositoryInfo::kURLField = "url";
const char* const BRepositoryInfo::kVendorField = "vendor";
const char* const BRepositoryInfo::kSummaryField = "summary";
const char* const BRepositoryInfo::kPriorityField = "priority";
@ -35,6 +35,9 @@ const char* const BRepositoryInfo::kArchitectureField = "architecture";
const char* const BRepositoryInfo::kLicenseNameField = "licenseName";
const char* const BRepositoryInfo::kLicenseTextField = "licenseText";
// XXX: Kill me after everyone upgrades.
const char* const BRepositoryInfo::kURLField = "url";
BRepositoryInfo::BRepositoryInfo()
:
@ -82,14 +85,15 @@ BRepositoryInfo::Archive(BMessage* data, bool deep) const
if (result != B_OK)
return result;
if ((result = data->AddString(kNameField, fName)) != B_OK)
// XXX: Kill me after everyone upgrades
if ((result = data->AddString(kURLField, "STUB")) != B_OK)
return result;
if ((result = data->AddString(kURLField, fOriginalBaseURL)) != B_OK)
if ((result = data->AddString(kNameField, fName)) != B_OK)
return result;
if ((result = data->AddString(kVendorField, fVendor)) != B_OK)
return result;
result = data->AddString(kSummaryField, fSummary);
if (result != B_OK)
if ((result = data->AddString(kSummaryField, fSummary)) != B_OK)
return result;
if ((result = data->AddUInt8(kPriorityField, fPriority)) != B_OK)
return result;
@ -138,13 +142,6 @@ BRepositoryInfo::Name() const
}
const BString&
BRepositoryInfo::OriginalBaseURL() const
{
return fOriginalBaseURL;
}
const BString&
BRepositoryInfo::Vendor() const
{
@ -194,13 +191,6 @@ BRepositoryInfo::SetName(const BString& name)
}
void
BRepositoryInfo::SetOriginalBaseURL(const BString& url)
{
fOriginalBaseURL = url;
}
void
BRepositoryInfo::SetVendor(const BString& vendor)
{
@ -257,8 +247,6 @@ BRepositoryInfo::_SetTo(const BMessage* data)
status_t result;
if ((result = data->FindString(kNameField, &fName)) != B_OK)
return result;
if ((result = data->FindString(kURLField, &fOriginalBaseURL)) != B_OK)
return result;
if ((result = data->FindString(kVendorField, &fVendor)) != B_OK)
return result;
result = data->FindString(kSummaryField, &fSummary);
@ -318,7 +306,6 @@ BRepositoryInfo::_SetTo(const BEntry& entry)
&unload_driver_settings);
const char* name = get_driver_parameter(settingsHandle, "name", NULL, NULL);
const char* url = get_driver_parameter(settingsHandle, "url", NULL, NULL);
const char* vendor
= get_driver_parameter(settingsHandle, "vendor", NULL, NULL);
const char* summary
@ -328,7 +315,7 @@ BRepositoryInfo::_SetTo(const BEntry& entry)
const char* architectureString
= get_driver_parameter(settingsHandle, "architecture", NULL, NULL);
if (name == NULL || *name == '\0' || url == NULL || *url == '\0'
if (name == NULL || *name == '\0'
|| vendor == NULL || *vendor == '\0'
|| summary == NULL || *summary == '\0'
|| priorityString == NULL || *priorityString == '\0'
@ -343,7 +330,6 @@ BRepositoryInfo::_SetTo(const BEntry& entry)
}
fName = name;
fOriginalBaseURL = url;
fVendor = vendor;
fSummary = summary;
fPriority = atoi(priorityString);

17
src/tools/create_repository_config/create_repository_config.cpp
View File

@ -1,9 +1,10 @@
/*
* Copyright 2013, Haiku, Inc. All Rights Reserved.
* Copyright 2013-2016, Haiku, Inc. All Rights Reserved.
* Distributed under the terms of the MIT License.
*
* Authors:
* Ingo Weinhold <ingo_weinhold@gmx.de>
* Alexander von Gluck IV <kallisti5@unixzen.com>
*/
@ -37,11 +38,10 @@ void
print_usage_and_exit(bool error)
{
fprintf(error ? stderr : stdout,
"Usage: %s [ <URL> ] <repository info> <repository config>\n"
"Usage: %s <URL> <repository info> <repository config>\n"
"Creates a repository config file from a given repository info and\n"
"the base URL (the directory in which the \"repo\", \"repo.info\',\n"
"and \"repo.sha256 files can be found). If the URL is not specified,\n"
"the one from the repository info is used.",
"the base URL (the directory in which the 'repo', 'repo.info',\n"
"and 'repo.sha256' files can be found).\n",
sProgramName);
exit(error ? 1 : 0);
}
@ -50,7 +50,7 @@ print_usage_and_exit(bool error)
int
main(int argc, const char* const* argv)
{
if (argc < 3 || argc > 4) {
if (argc != 4) {
if (argc == 2
&& (strcmp(argv[1], "-h") == 0 || strcmp(argv[1], "--help") == 0)) {
print_usage_and_exit(false);
@ -59,7 +59,7 @@ main(int argc, const char* const* argv)
}
int argi = 1;
const char* url = argc == 4 ? argv[argi++] : NULL;
const char* url = argv[argi++];
const char* infoPath = argv[argi++];
const char* configPath = argv[argi++];
@ -68,9 +68,6 @@ main(int argc, const char* const* argv)
DIE_ON_ERROR(repoInfo.SetTo(infoPath),
"failed to read repository info file \"%s\"", infoPath);
if (url == NULL)
url = repoInfo.OriginalBaseURL();
// init and write the config
BPackageKit::BRepositoryConfig repoConfig;
repoConfig.SetName(repoInfo.Name());

61
src/tools/get_package_dependencies/HTTPClient.cpp Normal file
View File

@ -0,0 +1,61 @@
/*
* Copyright 2016, Haiku, Inc. All Rights Reserved.
* Distributed under the terms of the MIT License.
*
* Authors:
* Alexander von Gluck IV <kallisti5@unixzen.com>
*/
#include "HTTPClient.h"
#include <File.h>
HTTPClient::HTTPClient()
:
fCurl(NULL)
{
fCurl = curl_easy_init();
curl_easy_setopt(fCurl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(fCurl, CURLOPT_NOSIGNAL, 1);
curl_easy_setopt(fCurl, CURLOPT_ACCEPT_ENCODING, "deflate");
curl_easy_setopt(fCurl, CURLOPT_WRITEFUNCTION, &_WriteCallback);
}
HTTPClient::~HTTPClient()
{
curl_easy_cleanup(fCurl);
}
size_t
HTTPClient::_WriteCallback(void *buffer, size_t size, size_t nmemb,
void *userp)
{
BFile* target = reinterpret_cast<BFile*>(userp);
ssize_t dataWritten = target->Write(buffer, size * nmemb);
return size_t(dataWritten);
}
status_t
HTTPClient::Get(BString* url, BEntry* destination)
{
BFile target(destination, B_CREATE_FILE | B_ERASE_FILE | B_WRITE_ONLY);
curl_easy_setopt(fCurl, CURLOPT_URL, url->String());
curl_easy_setopt(fCurl, CURLOPT_WRITEDATA, &target);
CURLcode res = curl_easy_perform(fCurl);
// Check result
if (res != CURLE_OK) {
fprintf(stderr, "curl_easy_perform() failed: %s\n",
curl_easy_strerror(res));
return B_ERROR;
}
return B_OK;
}

32
src/tools/get_package_dependencies/HTTPClient.h Normal file
View File

@ -0,0 +1,32 @@
/*
* Copyright 2016, Haiku, Inc. All Rights Reserved.
* Distributed under the terms of the MIT License.
*
* Authors:
* Alexander von Gluck IV <kallisti5@unixzen.com>
*/
#ifndef HTTP_CLIENT_H
#define HTTP_CLIENT_H
#include <String.h>
#include <Entry.h>
#include <curl/curl.h>
class HTTPClient {
public:
HTTPClient();
~HTTPClient();
status_t Get(BString* url, BEntry* destination);
private:
static size_t _WriteCallback(void *buffer, size_t size, size_t nmemb,
void *userp);
CURL* fCurl;
};
#endif /* HTTP_CLIENT_H */

3
src/tools/get_package_dependencies/Jamfile
View File

@ -8,8 +8,9 @@ USES_BE_API on <build>get_package_dependencies = true ;
BuildPlatformMain <build>get_package_dependencies :
get_package_dependencies.cpp
HTTPClient.cpp
:
libpackage-add-on-libsolv_build.so
libsolvext_build.so libsolv_build.so
libpackage_build.so $(HOST_LIBBE) $(HOST_LIBSUPC++) $(HOST_LIBSTDC++)
libpackage_build.so $(HOST_LIBBE) curl $(HOST_LIBSUPC++) $(HOST_LIBSTDC++)
;

67
src/tools/get_package_dependencies/get_package_dependencies.cpp
View File

@ -1,9 +1,10 @@
/*
* Copyright 2013, Haiku, Inc. All Rights Reserved.
* Copyright 2013-2016, Haiku, Inc. All Rights Reserved.
* Distributed under the terms of the MIT License.
*
* Authors:
* Ingo Weinhold <ingo_weinhold@gmx.de>
* Alexander von Gluck IV <kallisti5@unixzen.com>
*/
@ -13,6 +14,8 @@
#include <map>
#include <FindDirectory.h>
#include <package/RepositoryConfig.h>
#include <package/RepositoryCache.h>
#include <package/manager/Exceptions.h>
#include <package/manager/RepositoryBuilder.h>
@ -23,6 +26,9 @@
#include <package/solver/SolverProblemSolution.h>
#include <package/solver/SolverRepository.h>
#include <package/solver/SolverResult.h>
#include <Path.h>
#include "HTTPClient.h"
using namespace BPackageKit;
@ -44,7 +50,7 @@ void
print_usage_and_exit(bool error)
{
fprintf(error ? stderr : stdout,
"Usage: %s <repository> ... -- <package> ...\n"
"Usage: %s <repository-config> ... -- <package> ...\n"
"Resolves the dependencies of the given packages using the given\n"
"repositories and prints the URLs of the packages that are also\n"
"needed to satisfy all requirements. Fails, if there are conflicts\n"
@ -54,6 +60,33 @@ print_usage_and_exit(bool error)
}
static status_t
get_repocache(BRepositoryConfig config, BPath* outFile)
{
BString finalURL;
finalURL.SetTo(config.BaseURL());
finalURL += "/repo";
if (find_directory(B_SYSTEM_TEMP_DIRECTORY, outFile) != B_OK)
outFile->SetTo("/tmp");
BString tempFile = config.Name();
tempFile += ".repocache";
outFile->Append(tempFile);
//printf("Downloading: %s\n", finalURL.String());
//printf("Final output will be %s\n", outFile->Path());
BEntry entry(outFile->Path());
HTTPClient* http = new HTTPClient;
status_t result = http->Get(&finalURL, &entry);
delete http;
return result;
}
int
main(int argc, const char* const* argv)
{
@ -93,17 +126,31 @@ main(int argc, const char* const* argv)
DIE(B_OK, "%s", e.Details().String());
}
// add external repositories
std::map<BSolverRepository*, BRepositoryInfo> repositoryInfos;
// add specified remote repositories
std::map<BSolverRepository*, BString> repositoryURLs;
for (int i = 0; i < repositoryCount; i++) {
BSolverRepository* repository = new BSolverRepository;
BRepositoryCache cache;
error = cache.SetTo(repositories[i]);
BRepositoryConfig config;
error = config.SetTo(repositories[i]);
if (error != B_OK)
DIE(error, "failed to read repository file '%s'", repositories[i]);
DIE(error, "failed to read repository config '%s'", repositories[i]);
// TODO: It would make sense if BRepositoryBuilder could accept a
// BRepositoryConfig directly and "get" the remote BRepositoryCache
// to properly solve dependencies. For now we curl here.
BPath output;
get_repocache(config, &output);
BRepositoryCache cache;
error = cache.SetTo(output.Path());
if (error != B_OK)
DIE(error, "failed to read repository cache '%s'", repositories[i]);
BRepositoryBuilder(*repository, cache)
.AddToSolver(solver, false);
repositoryInfos[repository] = cache.Info();
repositoryURLs[repository] = config.BaseURL();
}
// solve
@ -152,9 +199,7 @@ main(int argc, const char* const* argv)
switch (element->Type()) {
case BSolverResultElement::B_TYPE_INSTALL:
if (package->Repository() != &installedRepository) {
const BRepositoryInfo& info
= repositoryInfos[package->Repository()];
BString url = info.OriginalBaseURL();
BString url = repositoryURLs[package->Repository()];
url << "/packages/" << package->Info().CanonicalFileName();
printf("%s\n", url.String());
}