repo rework: Remove need for repos to be self-aware

* See #12917 for details.
* Squashed to one commit to make revert easy if we
  run into any issues.
* pkgman is now pre-attached to the 'current' repo
  version within nightly images so they can be updated
  by default.
* This shouldn't impact us keeping older sets of package
  versions by commit hash for building older hrevs.
* There are XXX stubs with "Kill me". These will need
  to be dropped after users are given sufficent time to
  upgrade. We're dropping a previously required field (url)
  so making this a slowish roll out.
* Makes the repos a lot less restrictive which should
  help PM package building automation be a bit easier.
* Once this stuff smooths out, we'll add UUID's to the
  repo definitions for duplicate repo detection.

build/jam/ImageRules

@@ -901,7 +901,7 @@ rule AddPackagesAndRepositoryVariablesToContainerScript script : container
 	local repositoryFiles ;
 	for repository in $(HAIKU_REPOSITORIES) {
 		repositoryFiles
-			+= [ on $(repository) return $(HAIKU_REPOSITORY_CACHE_FILE) ] ;
+			+= [ on $(repository) return $(HAIKU_REPOSITORY_CONFIG_FILE) ] ;
 	}
 
 	AddTargetVariableToScript $(script) : $(repositoryFiles) : repositories ;

build/jam/RepositoryRules

@@ -761,10 +761,10 @@ actions BuildRemoteHaikuPortsRepository1
 }
 
 
-rule HaikuRepository repository : repoInfoTemplate : packages
+rule HaikuRepository repository : repoInfoTemplate : packages : url
 {
-	# HaikuRepository <repository> : <repoInfoTemplate> : <packages>
+	# HaikuRepository <repository> : <repoInfoTemplate> : <packages> : <url>
-	#	[ : <url> [ : <versionFile> ] ] ;
+	#	[ : <versionFile> ] ;
 	# Builds the Haiku repository from the given packages and repository info
 	# template. <repository> must already be located.
 	#
@@ -773,6 +773,7 @@ rule HaikuRepository repository : repoInfoTemplate : packages
 	#	meta data files.
 	# <repoInfoTemplate> - The repository info template file to be used.
 	# <packages> - The packages to be added to the repository.
+	# <url> - The remote location of the initial update repository
 
 	local architecture = $(HAIKU_PACKAGING_ARCH) ;
 	local secondaryArchitecture ;
@@ -791,7 +792,7 @@ rule HaikuRepository repository : repoInfoTemplate : packages
 	# build the respository config
 	local repositoryConfig = $(repository:G=repository-config)-config ;
 	MakeLocate $(repositoryConfig) : $(repositoriesDir) ;
-	RepositoryConfig $(repositoryConfig) : $(repositoryInfo) ;
+	RepositoryConfig $(repositoryConfig) : $(repositoryInfo) : $(url) ;
 	HAIKU_REPOSITORY_CONFIG_FILE on $(repository) = $(repositoryConfig) ;
 
 	# setup the repository cache file

build/jam/repositories/Haiku

@@ -52,4 +52,8 @@ if $(webPositiveIsAvailable) {
 	packages += webpositive ;
 }
 
-HaikuRepository $(haikuRepository) : $(repoInfo) : $(packages:S=.hpkg) ;
+local version = current ;
+#local version = $(HAIKU_VERSION) ;
+local remoteURL = http://packages.haiku-os.org/haiku/master/$(HAIKU_PACKAGING_ARCH)/$(version) ;
+
+HaikuRepository $(haikuRepository) : $(repoInfo) : $(packages:S=.hpkg) : $(remoteURL) ;

build/scripts/build_haiku_image

@@ -7,7 +7,7 @@ set -o errexit
 # tmpDir
 # addBuildCompatibilityLibDir
 # systemPackages - lists of the hpkg packages copied/updated
-# repositories - all repository files
+# repositories - all repository configuration files
 # downloadDir
 # The following are only for image types:
 # installDir

headers/os/package/RepositoryInfo.h

@@ -33,7 +33,6 @@ public:
 			status_t			InitCheck() const;
 
 			const BString&		Name() const;
-			const BString&		OriginalBaseURL() const;
 			const BString&		Vendor() const;
 			const BString&		Summary() const;
 			uint8				Priority() const;
@@ -42,7 +41,6 @@ public:
 			const BStringList&	LicenseTexts() const;
 
 			void				SetName(const BString& name);
-			void				SetOriginalBaseURL(const BString& url);
 			void				SetVendor(const BString& vendor);
 			void				SetSummary(const BString& summary);
 			void				SetPriority(uint8 priority);
@@ -57,8 +55,10 @@ public:
 
 	static	const uint8			kDefaultPriority;
 
+	// XXX: Kill me after everyone upgrades
+	static  const char* const	kURLField;
+
 	static	const char* const	kNameField;
-	static	const char* const	kURLField;
 	static	const char* const	kVendorField;
 	static	const char*	const	kSummaryField;
 	static	const char*	const	kPriorityField;
@@ -74,7 +74,6 @@ private:
 			status_t			fInitStatus;
 
 			BString				fName;
-			BString				fOriginalBaseURL;
 			BString				fVendor;
 			BString				fSummary;
 			uint8				fPriority;

src/bin/package_repo/command_list.cpp

@@ -69,7 +69,6 @@ struct RepositoryContentListHandler : BRepositoryContentHandler {
 		printf("repository-info:\n");
 		printf("\tname: %s\n", repositoryInfo.Name().String());
 		printf("\tsummary: %s\n", repositoryInfo.Summary().String());
-		printf("\turl: %s\n", repositoryInfo.OriginalBaseURL().String());
 		printf("\tvendor: %s\n", repositoryInfo.Vendor().String());
 		printf("\tpriority: %u\n", repositoryInfo.Priority());
 		printf("\tarchitecture: %s\n",

src/bin/pkgman/command_list_repos.cpp

@@ -118,8 +118,6 @@ ListReposCommand::Execute(int argc, const char* const* argv)
 						repoCache.Info().Architecture()]);
 				printf("\t\tpkg-count: %" B_PRIu32 "\n",
 					repoCache.CountPackages());
-				printf("\t\torig-url:  %s\n",
-					repoCache.Info().OriginalBaseURL().String());
 				printf("\t\torig-prio: %u\n", repoCache.Info().Priority());
 			} else
 				printf("\t\t<no repository cache found>\n");

src/data/repository_infos/haiku

@@ -2,5 +2,4 @@ name Haiku
 vendor "Haiku Project"
 summary "The Haiku repository (for Haiku %HAIKU_VERSION_NO_REVISION%)"
 priority 1
-url http://packages.haiku-os.org/haiku/master/%HAIKU_PACKAGING_ARCH%/%HAIKU_VERSION_NO_REVISION%
 architecture %HAIKU_PACKAGING_ARCH%

src/data/repository_infos/haikuports

@@ -2,5 +2,4 @@ name HaikuPorts
 vendor "Haiku Project"
 summary "The HaikuPorts repository (for Haiku %HAIKU_VERSION_NO_REVISION%)"
 priority 1
-url http://packages.haiku-os.org/haikuports/master/repo/%HAIKU_PACKAGING_ARCH%/dummy
 architecture %HAIKU_PACKAGING_ARCH%

src/kits/package/RepositoryInfo.cpp

@@ -1,9 +1,10 @@
 /*
- * Copyright 2011, Haiku, Inc. All Rights Reserved.
+ * Copyright 2011-2016, Haiku, Inc. All Rights Reserved.
  * Distributed under the terms of the MIT License.
  *
  * Authors:
  *		Oliver Tappe <zooey@hirschkaefer.de>
+ *		Alexander von Gluck IV <kallisti5@unixzen.com>
  */
 
 
@@ -27,7 +28,6 @@ namespace BPackageKit {
 const uint8 BRepositoryInfo::kDefaultPriority	= 50;
 
 const char* const BRepositoryInfo::kNameField			= "name";
-const char* const BRepositoryInfo::kURLField			= "url";
 const char* const BRepositoryInfo::kVendorField			= "vendor";
 const char* const BRepositoryInfo::kSummaryField		= "summary";
 const char* const BRepositoryInfo::kPriorityField		= "priority";
@@ -35,6 +35,9 @@ const char* const BRepositoryInfo::kArchitectureField	= "architecture";
 const char* const BRepositoryInfo::kLicenseNameField	= "licenseName";
 const char* const BRepositoryInfo::kLicenseTextField	= "licenseText";
 
+// XXX: Kill me after everyone upgrades.
+const char* const BRepositoryInfo::kURLField			= "url";
+
 
 BRepositoryInfo::BRepositoryInfo()
 	:
@@ -82,14 +85,15 @@ BRepositoryInfo::Archive(BMessage* data, bool deep) const
 	if (result != B_OK)
 		return result;
 
-	if ((result = data->AddString(kNameField, fName)) != B_OK)
+	// XXX: Kill me after everyone upgrades
+	if ((result = data->AddString(kURLField, "STUB")) != B_OK)
 		return result;
-	if ((result = data->AddString(kURLField, fOriginalBaseURL)) != B_OK)
+
+	if ((result = data->AddString(kNameField, fName)) != B_OK)
 		return result;
 	if ((result = data->AddString(kVendorField, fVendor)) != B_OK)
 		return result;
-	result = data->AddString(kSummaryField, fSummary);
+	if ((result = data->AddString(kSummaryField, fSummary)) != B_OK)
-	if (result != B_OK)
 		return result;
 	if ((result = data->AddUInt8(kPriorityField, fPriority)) != B_OK)
 		return result;
@@ -138,13 +142,6 @@ BRepositoryInfo::Name() const
 }
 
 
-const BString&
-BRepositoryInfo::OriginalBaseURL() const
-{
-	return fOriginalBaseURL;
-}
-
-
 const BString&
 BRepositoryInfo::Vendor() const
 {
@@ -194,13 +191,6 @@ BRepositoryInfo::SetName(const BString& name)
 }
 
 
-void
-BRepositoryInfo::SetOriginalBaseURL(const BString& url)
-{
-	fOriginalBaseURL = url;
-}
-
-
 void
 BRepositoryInfo::SetVendor(const BString& vendor)
 {
@@ -257,8 +247,6 @@ BRepositoryInfo::_SetTo(const BMessage* data)
 	status_t result;
 	if ((result = data->FindString(kNameField, &fName)) != B_OK)
 		return result;
-	if ((result = data->FindString(kURLField, &fOriginalBaseURL)) != B_OK)
-		return result;
 	if ((result = data->FindString(kVendorField, &fVendor)) != B_OK)
 		return result;
 	result = data->FindString(kSummaryField, &fSummary);
@@ -318,7 +306,6 @@ BRepositoryInfo::_SetTo(const BEntry& entry)
 		&unload_driver_settings);
 
 	const char* name = get_driver_parameter(settingsHandle, "name", NULL, NULL);
-	const char* url = get_driver_parameter(settingsHandle, "url", NULL, NULL);
 	const char* vendor
 		= get_driver_parameter(settingsHandle, "vendor", NULL, NULL);
 	const char* summary
@@ -328,7 +315,7 @@ BRepositoryInfo::_SetTo(const BEntry& entry)
 	const char* architectureString
 		= get_driver_parameter(settingsHandle, "architecture", NULL, NULL);
 
-	if (name == NULL || *name == '\0' || url == NULL || *url == '\0'
+	if (name == NULL || *name == '\0'
 		|| vendor == NULL || *vendor == '\0'
 		|| summary == NULL || *summary == '\0'
 		|| priorityString == NULL || *priorityString == '\0'
@@ -343,7 +330,6 @@ BRepositoryInfo::_SetTo(const BEntry& entry)
 	}
 
 	fName = name;
-	fOriginalBaseURL = url;
 	fVendor = vendor;
 	fSummary = summary;
 	fPriority = atoi(priorityString);

src/tools/create_repository_config/create_repository_config.cpp

@@ -1,9 +1,10 @@
 /*
- * Copyright 2013, Haiku, Inc. All Rights Reserved.
+ * Copyright 2013-2016, Haiku, Inc. All Rights Reserved.
  * Distributed under the terms of the MIT License.
  *
  * Authors:
  *		Ingo Weinhold <ingo_weinhold@gmx.de>
+ * 		Alexander von Gluck IV <kallisti5@unixzen.com>
  */
 
 
@@ -37,11 +38,10 @@ void
 print_usage_and_exit(bool error)
 {
 	fprintf(error ? stderr : stdout,
-		"Usage: %s [ <URL> ] <repository info> <repository config>\n"
+		"Usage: %s <URL> <repository info> <repository config>\n"
 		"Creates a repository config file from a given repository info and\n"
-		"the base URL (the directory in which the \"repo\", \"repo.info\',\n"
+		"the base URL (the directory in which the 'repo', 'repo.info',\n"
-		"and \"repo.sha256 files can be found). If the URL is not specified,\n"
+		"and 'repo.sha256' files can be found).\n",
-		"the one from the repository info is used.",
 		sProgramName);
 	exit(error ? 1 : 0);
 }
@@ -50,7 +50,7 @@ print_usage_and_exit(bool error)
 int
 main(int argc, const char* const* argv)
 {
-	if (argc < 3 || argc > 4) {
+	if (argc != 4) {
 		if (argc == 2
 			&& (strcmp(argv[1], "-h") == 0 || strcmp(argv[1], "--help") == 0)) {
 				print_usage_and_exit(false);
@@ -59,7 +59,7 @@ main(int argc, const char* const* argv)
 	}
 
 	int argi = 1;
-	const char* url = argc == 4 ? argv[argi++] : NULL;
+	const char* url = argv[argi++];
 	const char* infoPath = argv[argi++];
 	const char* configPath = argv[argi++];
 
@@ -68,9 +68,6 @@ main(int argc, const char* const* argv)
 	DIE_ON_ERROR(repoInfo.SetTo(infoPath),
 		"failed to read repository info file \"%s\"", infoPath);
 
-	if (url == NULL)
-		url = repoInfo.OriginalBaseURL();
-
 	// init and write the config
 	BPackageKit::BRepositoryConfig repoConfig;
 	repoConfig.SetName(repoInfo.Name());

src/tools/get_package_dependencies/HTTPClient.cpp

@@ -0,0 +1,61 @@
+/*
+ * Copyright 2016, Haiku, Inc. All Rights Reserved.
+ * Distributed under the terms of the MIT License.
+ *
+ * Authors:
+ *		Alexander von Gluck IV <kallisti5@unixzen.com>
+ */
+
+
+#include "HTTPClient.h"
+
+#include <File.h>
+
+
+HTTPClient::HTTPClient()
+	:
+	fCurl(NULL)
+{
+	fCurl = curl_easy_init();
+	curl_easy_setopt(fCurl, CURLOPT_FOLLOWLOCATION, 1L);
+	curl_easy_setopt(fCurl, CURLOPT_NOSIGNAL, 1);
+	curl_easy_setopt(fCurl, CURLOPT_ACCEPT_ENCODING, "deflate");
+	curl_easy_setopt(fCurl, CURLOPT_WRITEFUNCTION, &_WriteCallback);
+}
+
+
+HTTPClient::~HTTPClient()
+{
+	curl_easy_cleanup(fCurl);
+}
+
+
+size_t
+HTTPClient::_WriteCallback(void *buffer, size_t size, size_t nmemb,
+	void *userp)
+{
+	BFile* target = reinterpret_cast<BFile*>(userp);
+	ssize_t dataWritten = target->Write(buffer, size * nmemb);
+	return size_t(dataWritten);
+}
+
+
+status_t
+HTTPClient::Get(BString* url, BEntry* destination)
+{
+	BFile target(destination, B_CREATE_FILE | B_ERASE_FILE | B_WRITE_ONLY);
+
+	curl_easy_setopt(fCurl, CURLOPT_URL, url->String());
+	curl_easy_setopt(fCurl, CURLOPT_WRITEDATA, &target);
+
+    CURLcode res = curl_easy_perform(fCurl);
+
+	// Check result
+	if (res != CURLE_OK) {
+		fprintf(stderr, "curl_easy_perform() failed: %s\n",
+			curl_easy_strerror(res));
+		return B_ERROR;
+	}
+
+	return B_OK;
+}

src/tools/get_package_dependencies/HTTPClient.h

@@ -0,0 +1,32 @@
+/*
+ * Copyright 2016, Haiku, Inc. All Rights Reserved.
+ * Distributed under the terms of the MIT License.
+ *
+ * Authors:
+ *		Alexander von Gluck IV <kallisti5@unixzen.com>
+ */
+#ifndef HTTP_CLIENT_H
+#define HTTP_CLIENT_H
+
+
+#include <String.h>
+#include <Entry.h>
+#include <curl/curl.h>
+
+
+class HTTPClient {
+public:
+						HTTPClient();
+						~HTTPClient();
+
+			status_t	Get(BString* url, BEntry* destination);
+
+private:
+	static	size_t		_WriteCallback(void *buffer, size_t size, size_t nmemb,
+							void *userp);
+
+			CURL*		fCurl;
+};
+
+
+#endif /* HTTP_CLIENT_H */

src/tools/get_package_dependencies/Jamfile

@@ -8,8 +8,9 @@ USES_BE_API on <build>get_package_dependencies = true ;
 
 BuildPlatformMain <build>get_package_dependencies :
 	get_package_dependencies.cpp
+	HTTPClient.cpp
 	:
 	libpackage-add-on-libsolv_build.so
 	libsolvext_build.so libsolv_build.so
-	libpackage_build.so $(HOST_LIBBE) $(HOST_LIBSUPC++) $(HOST_LIBSTDC++)
+	libpackage_build.so $(HOST_LIBBE) curl $(HOST_LIBSUPC++) $(HOST_LIBSTDC++)
 	;

src/tools/get_package_dependencies/get_package_dependencies.cpp

@@ -1,9 +1,10 @@
 /*
- * Copyright 2013, Haiku, Inc. All Rights Reserved.
+ * Copyright 2013-2016, Haiku, Inc. All Rights Reserved.
  * Distributed under the terms of the MIT License.
  *
  * Authors:
  *		Ingo Weinhold <ingo_weinhold@gmx.de>
+ *		Alexander von Gluck IV <kallisti5@unixzen.com>
  */
 
 
@@ -13,6 +14,8 @@
 
 #include <map>
 
+#include <FindDirectory.h>
+#include <package/RepositoryConfig.h>
 #include <package/RepositoryCache.h>
 #include <package/manager/Exceptions.h>
 #include <package/manager/RepositoryBuilder.h>
@@ -23,6 +26,9 @@
 #include <package/solver/SolverProblemSolution.h>
 #include <package/solver/SolverRepository.h>
 #include <package/solver/SolverResult.h>
+#include <Path.h>
+
+#include "HTTPClient.h"
 
 
 using namespace BPackageKit;
@@ -44,7 +50,7 @@ void
 print_usage_and_exit(bool error)
 {
 	fprintf(error ? stderr : stdout,
-		"Usage: %s <repository> ... -- <package> ...\n"
+		"Usage: %s <repository-config> ... -- <package> ...\n"
 		"Resolves the dependencies of the given packages using the given\n"
 		"repositories and prints the URLs of the packages that are also\n"
 		"needed to satisfy all requirements. Fails, if there are conflicts\n"
@@ -54,6 +60,33 @@ print_usage_and_exit(bool error)
 }
 
 
+static status_t
+get_repocache(BRepositoryConfig config, BPath* outFile)
+{
+	BString finalURL;
+	finalURL.SetTo(config.BaseURL());
+	finalURL += "/repo";
+
+	if (find_directory(B_SYSTEM_TEMP_DIRECTORY, outFile) != B_OK)
+		outFile->SetTo("/tmp");
+
+	BString tempFile = config.Name();
+	tempFile += ".repocache";
+
+	outFile->Append(tempFile);
+
+	//printf("Downloading: %s\n", finalURL.String());
+	//printf("Final output will be %s\n", outFile->Path());
+
+	BEntry entry(outFile->Path());
+	HTTPClient* http = new HTTPClient;
+	status_t result = http->Get(&finalURL, &entry);
+	delete http;
+
+	return result;
+}
+
+
 int
 main(int argc, const char* const* argv)
 {
@@ -93,17 +126,31 @@ main(int argc, const char* const* argv)
 		DIE(B_OK, "%s", e.Details().String());
 	}
 
-	// add external repositories
+	// add specified remote repositories
-	std::map<BSolverRepository*, BRepositoryInfo> repositoryInfos;
+	std::map<BSolverRepository*, BString> repositoryURLs;
+
 	for (int i = 0; i < repositoryCount; i++) {
 		BSolverRepository* repository = new BSolverRepository;
-		BRepositoryCache cache;
+		BRepositoryConfig config;
-		error = cache.SetTo(repositories[i]);
+		error = config.SetTo(repositories[i]);
 		if (error != B_OK)
-			DIE(error, "failed to read repository file '%s'", repositories[i]);
+			DIE(error, "failed to read repository config '%s'", repositories[i]);
+
+		// TODO: It would make sense if BRepositoryBuilder could accept a
+		// BRepositoryConfig directly and "get" the remote BRepositoryCache
+		// to properly solve dependencies. For now we curl here.
+
+		BPath output;
+		get_repocache(config, &output);
+
+		BRepositoryCache cache;
+		error = cache.SetTo(output.Path());
+		if (error != B_OK)
+			DIE(error, "failed to read repository cache '%s'", repositories[i]);
+
 		BRepositoryBuilder(*repository, cache)
 			.AddToSolver(solver, false);
-		repositoryInfos[repository] = cache.Info();
+		repositoryURLs[repository] = config.BaseURL();
 	}
 
 	// solve
@@ -152,9 +199,7 @@ main(int argc, const char* const* argv)
 		switch (element->Type()) {
 			case BSolverResultElement::B_TYPE_INSTALL:
 				if (package->Repository() != &installedRepository) {
-					const BRepositoryInfo& info
+					BString url = repositoryURLs[package->Repository()];
-						= repositoryInfos[package->Repository()];
-					BString url = info.OriginalBaseURL();
 					url << "/packages/" << package->Info().CanonicalFileName();
 					printf("%s\n", url.String());
 				}