From 5ffaf72c8a74a7eb6827e4d2b1f47c9360ddaefe Mon Sep 17 00:00:00 2001 From: Alexander von Gluck IV Date: Fri, 2 Dec 2016 00:41:06 -0600 Subject: [PATCH] repo rework: Remove need for repos to be self-aware * See #12917 for details. * Squashed to one commit to make revert easy if we run into any issues. * pkgman is now pre-attached to the 'current' repo version within nightly images so they can be updated by default. * This shouldn't impact us keeping older sets of package versions by commit hash for building older hrevs. * There are XXX stubs with "Kill me". These will need to be dropped after users are given sufficent time to upgrade. We're dropping a previously required field (url) so making this a slowish roll out. * Makes the repos a lot less restrictive which should help PM package building automation be a bit easier. * Once this stuff smooths out, we'll add UUID's to the repo definitions for duplicate repo detection. --- build/jam/ImageRules | 2 +- build/jam/RepositoryRules | 9 +-- build/jam/repositories/Haiku | 6 +- build/scripts/build_haiku_image | 2 +- headers/os/package/RepositoryInfo.h | 7 +- src/bin/package_repo/command_list.cpp | 1 - src/bin/pkgman/command_list_repos.cpp | 2 - src/data/repository_infos/haiku | 1 - src/data/repository_infos/haikuports | 1 - src/kits/package/RepositoryInfo.cpp | 36 +++------- .../create_repository_config.cpp | 17 ++--- .../get_package_dependencies/HTTPClient.cpp | 61 +++++++++++++++++ .../get_package_dependencies/HTTPClient.h | 32 +++++++++ src/tools/get_package_dependencies/Jamfile | 3 +- .../get_package_dependencies.cpp | 67 ++++++++++++++++--- 15 files changed, 184 insertions(+), 63 deletions(-) create mode 100644 src/tools/get_package_dependencies/HTTPClient.cpp create mode 100644 src/tools/get_package_dependencies/HTTPClient.h diff --git a/build/jam/ImageRules b/build/jam/ImageRules index c018470320..3141d70951 100644 --- a/build/jam/ImageRules +++ b/build/jam/ImageRules @@ -901,7 +901,7 @@ rule AddPackagesAndRepositoryVariablesToContainerScript script : container local repositoryFiles ; for repository in $(HAIKU_REPOSITORIES) { repositoryFiles - += [ on $(repository) return $(HAIKU_REPOSITORY_CACHE_FILE) ] ; + += [ on $(repository) return $(HAIKU_REPOSITORY_CONFIG_FILE) ] ; } AddTargetVariableToScript $(script) : $(repositoryFiles) : repositories ; diff --git a/build/jam/RepositoryRules b/build/jam/RepositoryRules index fb91089c19..9290f2fd7a 100644 --- a/build/jam/RepositoryRules +++ b/build/jam/RepositoryRules @@ -761,10 +761,10 @@ actions BuildRemoteHaikuPortsRepository1 } -rule HaikuRepository repository : repoInfoTemplate : packages +rule HaikuRepository repository : repoInfoTemplate : packages : url { - # HaikuRepository : : - # [ : [ : ] ] ; + # HaikuRepository : : : + # [ : ] ; # Builds the Haiku repository from the given packages and repository info # template. must already be located. # @@ -773,6 +773,7 @@ rule HaikuRepository repository : repoInfoTemplate : packages # meta data files. # - The repository info template file to be used. # - The packages to be added to the repository. + # - The remote location of the initial update repository local architecture = $(HAIKU_PACKAGING_ARCH) ; local secondaryArchitecture ; @@ -791,7 +792,7 @@ rule HaikuRepository repository : repoInfoTemplate : packages # build the respository config local repositoryConfig = $(repository:G=repository-config)-config ; MakeLocate $(repositoryConfig) : $(repositoriesDir) ; - RepositoryConfig $(repositoryConfig) : $(repositoryInfo) ; + RepositoryConfig $(repositoryConfig) : $(repositoryInfo) : $(url) ; HAIKU_REPOSITORY_CONFIG_FILE on $(repository) = $(repositoryConfig) ; # setup the repository cache file diff --git a/build/jam/repositories/Haiku b/build/jam/repositories/Haiku index 14f5ebeaf7..1e2f0b12b1 100644 --- a/build/jam/repositories/Haiku +++ b/build/jam/repositories/Haiku @@ -52,4 +52,8 @@ if $(webPositiveIsAvailable) { packages += webpositive ; } -HaikuRepository $(haikuRepository) : $(repoInfo) : $(packages:S=.hpkg) ; +local version = current ; +#local version = $(HAIKU_VERSION) ; +local remoteURL = http://packages.haiku-os.org/haiku/master/$(HAIKU_PACKAGING_ARCH)/$(version) ; + +HaikuRepository $(haikuRepository) : $(repoInfo) : $(packages:S=.hpkg) : $(remoteURL) ; diff --git a/build/scripts/build_haiku_image b/build/scripts/build_haiku_image index 28ab70959e..7554e0a72d 100755 --- a/build/scripts/build_haiku_image +++ b/build/scripts/build_haiku_image @@ -7,7 +7,7 @@ set -o errexit # tmpDir # addBuildCompatibilityLibDir # systemPackages - lists of the hpkg packages copied/updated -# repositories - all repository files +# repositories - all repository configuration files # downloadDir # The following are only for image types: # installDir diff --git a/headers/os/package/RepositoryInfo.h b/headers/os/package/RepositoryInfo.h index 32aa05d6c2..513539c18b 100644 --- a/headers/os/package/RepositoryInfo.h +++ b/headers/os/package/RepositoryInfo.h @@ -33,7 +33,6 @@ public: status_t InitCheck() const; const BString& Name() const; - const BString& OriginalBaseURL() const; const BString& Vendor() const; const BString& Summary() const; uint8 Priority() const; @@ -42,7 +41,6 @@ public: const BStringList& LicenseTexts() const; void SetName(const BString& name); - void SetOriginalBaseURL(const BString& url); void SetVendor(const BString& vendor); void SetSummary(const BString& summary); void SetPriority(uint8 priority); @@ -57,8 +55,10 @@ public: static const uint8 kDefaultPriority; + // XXX: Kill me after everyone upgrades + static const char* const kURLField; + static const char* const kNameField; - static const char* const kURLField; static const char* const kVendorField; static const char* const kSummaryField; static const char* const kPriorityField; @@ -74,7 +74,6 @@ private: status_t fInitStatus; BString fName; - BString fOriginalBaseURL; BString fVendor; BString fSummary; uint8 fPriority; diff --git a/src/bin/package_repo/command_list.cpp b/src/bin/package_repo/command_list.cpp index c2dcd12203..54d223dec6 100644 --- a/src/bin/package_repo/command_list.cpp +++ b/src/bin/package_repo/command_list.cpp @@ -69,7 +69,6 @@ struct RepositoryContentListHandler : BRepositoryContentHandler { printf("repository-info:\n"); printf("\tname: %s\n", repositoryInfo.Name().String()); printf("\tsummary: %s\n", repositoryInfo.Summary().String()); - printf("\turl: %s\n", repositoryInfo.OriginalBaseURL().String()); printf("\tvendor: %s\n", repositoryInfo.Vendor().String()); printf("\tpriority: %u\n", repositoryInfo.Priority()); printf("\tarchitecture: %s\n", diff --git a/src/bin/pkgman/command_list_repos.cpp b/src/bin/pkgman/command_list_repos.cpp index 677983a8f6..c7a26cc3e7 100644 --- a/src/bin/pkgman/command_list_repos.cpp +++ b/src/bin/pkgman/command_list_repos.cpp @@ -118,8 +118,6 @@ ListReposCommand::Execute(int argc, const char* const* argv) repoCache.Info().Architecture()]); printf("\t\tpkg-count: %" B_PRIu32 "\n", repoCache.CountPackages()); - printf("\t\torig-url: %s\n", - repoCache.Info().OriginalBaseURL().String()); printf("\t\torig-prio: %u\n", repoCache.Info().Priority()); } else printf("\t\t\n"); diff --git a/src/data/repository_infos/haiku b/src/data/repository_infos/haiku index a6501ab66c..1c1a558e72 100644 --- a/src/data/repository_infos/haiku +++ b/src/data/repository_infos/haiku @@ -2,5 +2,4 @@ name Haiku vendor "Haiku Project" summary "The Haiku repository (for Haiku %HAIKU_VERSION_NO_REVISION%)" priority 1 -url http://packages.haiku-os.org/haiku/master/%HAIKU_PACKAGING_ARCH%/%HAIKU_VERSION_NO_REVISION% architecture %HAIKU_PACKAGING_ARCH% diff --git a/src/data/repository_infos/haikuports b/src/data/repository_infos/haikuports index 3c62733115..bb623210b8 100644 --- a/src/data/repository_infos/haikuports +++ b/src/data/repository_infos/haikuports @@ -2,5 +2,4 @@ name HaikuPorts vendor "Haiku Project" summary "The HaikuPorts repository (for Haiku %HAIKU_VERSION_NO_REVISION%)" priority 1 -url http://packages.haiku-os.org/haikuports/master/repo/%HAIKU_PACKAGING_ARCH%/dummy architecture %HAIKU_PACKAGING_ARCH% diff --git a/src/kits/package/RepositoryInfo.cpp b/src/kits/package/RepositoryInfo.cpp index aaff367a6e..46828540ab 100644 --- a/src/kits/package/RepositoryInfo.cpp +++ b/src/kits/package/RepositoryInfo.cpp @@ -1,9 +1,10 @@ /* - * Copyright 2011, Haiku, Inc. All Rights Reserved. + * Copyright 2011-2016, Haiku, Inc. All Rights Reserved. * Distributed under the terms of the MIT License. * * Authors: * Oliver Tappe + * Alexander von Gluck IV */ @@ -27,7 +28,6 @@ namespace BPackageKit { const uint8 BRepositoryInfo::kDefaultPriority = 50; const char* const BRepositoryInfo::kNameField = "name"; -const char* const BRepositoryInfo::kURLField = "url"; const char* const BRepositoryInfo::kVendorField = "vendor"; const char* const BRepositoryInfo::kSummaryField = "summary"; const char* const BRepositoryInfo::kPriorityField = "priority"; @@ -35,6 +35,9 @@ const char* const BRepositoryInfo::kArchitectureField = "architecture"; const char* const BRepositoryInfo::kLicenseNameField = "licenseName"; const char* const BRepositoryInfo::kLicenseTextField = "licenseText"; +// XXX: Kill me after everyone upgrades. +const char* const BRepositoryInfo::kURLField = "url"; + BRepositoryInfo::BRepositoryInfo() : @@ -82,14 +85,15 @@ BRepositoryInfo::Archive(BMessage* data, bool deep) const if (result != B_OK) return result; - if ((result = data->AddString(kNameField, fName)) != B_OK) + // XXX: Kill me after everyone upgrades + if ((result = data->AddString(kURLField, "STUB")) != B_OK) return result; - if ((result = data->AddString(kURLField, fOriginalBaseURL)) != B_OK) + + if ((result = data->AddString(kNameField, fName)) != B_OK) return result; if ((result = data->AddString(kVendorField, fVendor)) != B_OK) return result; - result = data->AddString(kSummaryField, fSummary); - if (result != B_OK) + if ((result = data->AddString(kSummaryField, fSummary)) != B_OK) return result; if ((result = data->AddUInt8(kPriorityField, fPriority)) != B_OK) return result; @@ -138,13 +142,6 @@ BRepositoryInfo::Name() const } -const BString& -BRepositoryInfo::OriginalBaseURL() const -{ - return fOriginalBaseURL; -} - - const BString& BRepositoryInfo::Vendor() const { @@ -194,13 +191,6 @@ BRepositoryInfo::SetName(const BString& name) } -void -BRepositoryInfo::SetOriginalBaseURL(const BString& url) -{ - fOriginalBaseURL = url; -} - - void BRepositoryInfo::SetVendor(const BString& vendor) { @@ -257,8 +247,6 @@ BRepositoryInfo::_SetTo(const BMessage* data) status_t result; if ((result = data->FindString(kNameField, &fName)) != B_OK) return result; - if ((result = data->FindString(kURLField, &fOriginalBaseURL)) != B_OK) - return result; if ((result = data->FindString(kVendorField, &fVendor)) != B_OK) return result; result = data->FindString(kSummaryField, &fSummary); @@ -318,7 +306,6 @@ BRepositoryInfo::_SetTo(const BEntry& entry) &unload_driver_settings); const char* name = get_driver_parameter(settingsHandle, "name", NULL, NULL); - const char* url = get_driver_parameter(settingsHandle, "url", NULL, NULL); const char* vendor = get_driver_parameter(settingsHandle, "vendor", NULL, NULL); const char* summary @@ -328,7 +315,7 @@ BRepositoryInfo::_SetTo(const BEntry& entry) const char* architectureString = get_driver_parameter(settingsHandle, "architecture", NULL, NULL); - if (name == NULL || *name == '\0' || url == NULL || *url == '\0' + if (name == NULL || *name == '\0' || vendor == NULL || *vendor == '\0' || summary == NULL || *summary == '\0' || priorityString == NULL || *priorityString == '\0' @@ -343,7 +330,6 @@ BRepositoryInfo::_SetTo(const BEntry& entry) } fName = name; - fOriginalBaseURL = url; fVendor = vendor; fSummary = summary; fPriority = atoi(priorityString); diff --git a/src/tools/create_repository_config/create_repository_config.cpp b/src/tools/create_repository_config/create_repository_config.cpp index 9837945b31..549146f725 100644 --- a/src/tools/create_repository_config/create_repository_config.cpp +++ b/src/tools/create_repository_config/create_repository_config.cpp @@ -1,9 +1,10 @@ /* - * Copyright 2013, Haiku, Inc. All Rights Reserved. + * Copyright 2013-2016, Haiku, Inc. All Rights Reserved. * Distributed under the terms of the MIT License. * * Authors: * Ingo Weinhold + * Alexander von Gluck IV */ @@ -37,11 +38,10 @@ void print_usage_and_exit(bool error) { fprintf(error ? stderr : stdout, - "Usage: %s [ ] \n" + "Usage: %s \n" "Creates a repository config file from a given repository info and\n" - "the base URL (the directory in which the \"repo\", \"repo.info\',\n" - "and \"repo.sha256 files can be found). If the URL is not specified,\n" - "the one from the repository info is used.", + "the base URL (the directory in which the 'repo', 'repo.info',\n" + "and 'repo.sha256' files can be found).\n", sProgramName); exit(error ? 1 : 0); } @@ -50,7 +50,7 @@ print_usage_and_exit(bool error) int main(int argc, const char* const* argv) { - if (argc < 3 || argc > 4) { + if (argc != 4) { if (argc == 2 && (strcmp(argv[1], "-h") == 0 || strcmp(argv[1], "--help") == 0)) { print_usage_and_exit(false); @@ -59,7 +59,7 @@ main(int argc, const char* const* argv) } int argi = 1; - const char* url = argc == 4 ? argv[argi++] : NULL; + const char* url = argv[argi++]; const char* infoPath = argv[argi++]; const char* configPath = argv[argi++]; @@ -68,9 +68,6 @@ main(int argc, const char* const* argv) DIE_ON_ERROR(repoInfo.SetTo(infoPath), "failed to read repository info file \"%s\"", infoPath); - if (url == NULL) - url = repoInfo.OriginalBaseURL(); - // init and write the config BPackageKit::BRepositoryConfig repoConfig; repoConfig.SetName(repoInfo.Name()); diff --git a/src/tools/get_package_dependencies/HTTPClient.cpp b/src/tools/get_package_dependencies/HTTPClient.cpp new file mode 100644 index 0000000000..2297d4d8b8 --- /dev/null +++ b/src/tools/get_package_dependencies/HTTPClient.cpp @@ -0,0 +1,61 @@ +/* + * Copyright 2016, Haiku, Inc. All Rights Reserved. + * Distributed under the terms of the MIT License. + * + * Authors: + * Alexander von Gluck IV + */ + + +#include "HTTPClient.h" + +#include + + +HTTPClient::HTTPClient() + : + fCurl(NULL) +{ + fCurl = curl_easy_init(); + curl_easy_setopt(fCurl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(fCurl, CURLOPT_NOSIGNAL, 1); + curl_easy_setopt(fCurl, CURLOPT_ACCEPT_ENCODING, "deflate"); + curl_easy_setopt(fCurl, CURLOPT_WRITEFUNCTION, &_WriteCallback); +} + + +HTTPClient::~HTTPClient() +{ + curl_easy_cleanup(fCurl); +} + + +size_t +HTTPClient::_WriteCallback(void *buffer, size_t size, size_t nmemb, + void *userp) +{ + BFile* target = reinterpret_cast(userp); + ssize_t dataWritten = target->Write(buffer, size * nmemb); + return size_t(dataWritten); +} + + +status_t +HTTPClient::Get(BString* url, BEntry* destination) +{ + BFile target(destination, B_CREATE_FILE | B_ERASE_FILE | B_WRITE_ONLY); + + curl_easy_setopt(fCurl, CURLOPT_URL, url->String()); + curl_easy_setopt(fCurl, CURLOPT_WRITEDATA, &target); + + CURLcode res = curl_easy_perform(fCurl); + + // Check result + if (res != CURLE_OK) { + fprintf(stderr, "curl_easy_perform() failed: %s\n", + curl_easy_strerror(res)); + return B_ERROR; + } + + return B_OK; +} diff --git a/src/tools/get_package_dependencies/HTTPClient.h b/src/tools/get_package_dependencies/HTTPClient.h new file mode 100644 index 0000000000..d178153e77 --- /dev/null +++ b/src/tools/get_package_dependencies/HTTPClient.h @@ -0,0 +1,32 @@ +/* + * Copyright 2016, Haiku, Inc. All Rights Reserved. + * Distributed under the terms of the MIT License. + * + * Authors: + * Alexander von Gluck IV + */ +#ifndef HTTP_CLIENT_H +#define HTTP_CLIENT_H + + +#include +#include +#include + + +class HTTPClient { +public: + HTTPClient(); + ~HTTPClient(); + + status_t Get(BString* url, BEntry* destination); + +private: + static size_t _WriteCallback(void *buffer, size_t size, size_t nmemb, + void *userp); + + CURL* fCurl; +}; + + +#endif /* HTTP_CLIENT_H */ diff --git a/src/tools/get_package_dependencies/Jamfile b/src/tools/get_package_dependencies/Jamfile index 0bba9b052a..452afab2b4 100644 --- a/src/tools/get_package_dependencies/Jamfile +++ b/src/tools/get_package_dependencies/Jamfile @@ -8,8 +8,9 @@ USES_BE_API on get_package_dependencies = true ; BuildPlatformMain get_package_dependencies : get_package_dependencies.cpp + HTTPClient.cpp : libpackage-add-on-libsolv_build.so libsolvext_build.so libsolv_build.so - libpackage_build.so $(HOST_LIBBE) $(HOST_LIBSUPC++) $(HOST_LIBSTDC++) + libpackage_build.so $(HOST_LIBBE) curl $(HOST_LIBSUPC++) $(HOST_LIBSTDC++) ; diff --git a/src/tools/get_package_dependencies/get_package_dependencies.cpp b/src/tools/get_package_dependencies/get_package_dependencies.cpp index 63d54398ba..09ec554a32 100644 --- a/src/tools/get_package_dependencies/get_package_dependencies.cpp +++ b/src/tools/get_package_dependencies/get_package_dependencies.cpp @@ -1,9 +1,10 @@ /* - * Copyright 2013, Haiku, Inc. All Rights Reserved. + * Copyright 2013-2016, Haiku, Inc. All Rights Reserved. * Distributed under the terms of the MIT License. * * Authors: * Ingo Weinhold + * Alexander von Gluck IV */ @@ -13,6 +14,8 @@ #include +#include +#include #include #include #include @@ -23,6 +26,9 @@ #include #include #include +#include + +#include "HTTPClient.h" using namespace BPackageKit; @@ -44,7 +50,7 @@ void print_usage_and_exit(bool error) { fprintf(error ? stderr : stdout, - "Usage: %s ... -- ...\n" + "Usage: %s ... -- ...\n" "Resolves the dependencies of the given packages using the given\n" "repositories and prints the URLs of the packages that are also\n" "needed to satisfy all requirements. Fails, if there are conflicts\n" @@ -54,6 +60,33 @@ print_usage_and_exit(bool error) } +static status_t +get_repocache(BRepositoryConfig config, BPath* outFile) +{ + BString finalURL; + finalURL.SetTo(config.BaseURL()); + finalURL += "/repo"; + + if (find_directory(B_SYSTEM_TEMP_DIRECTORY, outFile) != B_OK) + outFile->SetTo("/tmp"); + + BString tempFile = config.Name(); + tempFile += ".repocache"; + + outFile->Append(tempFile); + + //printf("Downloading: %s\n", finalURL.String()); + //printf("Final output will be %s\n", outFile->Path()); + + BEntry entry(outFile->Path()); + HTTPClient* http = new HTTPClient; + status_t result = http->Get(&finalURL, &entry); + delete http; + + return result; +} + + int main(int argc, const char* const* argv) { @@ -93,17 +126,31 @@ main(int argc, const char* const* argv) DIE(B_OK, "%s", e.Details().String()); } - // add external repositories - std::map repositoryInfos; + // add specified remote repositories + std::map repositoryURLs; + for (int i = 0; i < repositoryCount; i++) { BSolverRepository* repository = new BSolverRepository; - BRepositoryCache cache; - error = cache.SetTo(repositories[i]); + BRepositoryConfig config; + error = config.SetTo(repositories[i]); if (error != B_OK) - DIE(error, "failed to read repository file '%s'", repositories[i]); + DIE(error, "failed to read repository config '%s'", repositories[i]); + + // TODO: It would make sense if BRepositoryBuilder could accept a + // BRepositoryConfig directly and "get" the remote BRepositoryCache + // to properly solve dependencies. For now we curl here. + + BPath output; + get_repocache(config, &output); + + BRepositoryCache cache; + error = cache.SetTo(output.Path()); + if (error != B_OK) + DIE(error, "failed to read repository cache '%s'", repositories[i]); + BRepositoryBuilder(*repository, cache) .AddToSolver(solver, false); - repositoryInfos[repository] = cache.Info(); + repositoryURLs[repository] = config.BaseURL(); } // solve @@ -152,9 +199,7 @@ main(int argc, const char* const* argv) switch (element->Type()) { case BSolverResultElement::B_TYPE_INSTALL: if (package->Repository() != &installedRepository) { - const BRepositoryInfo& info - = repositoryInfos[package->Repository()]; - BString url = info.OriginalBaseURL(); + BString url = repositoryURLs[package->Repository()]; url << "/packages/" << package->Info().CanonicalFileName(); printf("%s\n", url.String()); }