[project @ 2004-10-01 21:31:55 by jmb]

A somewhat better implementation of referrers which no longer sends the referer if the URL schemes don't match.

Things to do:
1) Preservation of referer across redirects (see comment in browser.c:284)
2) GUI templates/code for configuration of referer sending (simple on/off toggle only)
3) Make referer sending when fetching objects/stylesheets for a page pay attention to option_send_referer?
4) Handle the case where the referer is in the form of http://moo:foo@mysite.com/ (ie the login details embedded in the referer - not good).

svn path=/import/netsurf/; revision=1297

content/fetch.c

@@ -228,6 +228,7 @@ struct fetch * fetch_start(char *url, char *referer,
 	CURLMcode codem;
 	struct curl_slist *slist;
 	url_func_result res;
+	char *ref1 = 0, *ref2 = 0;
 
 	fetch = malloc(sizeof (*fetch));
 	if (!fetch)
@@ -238,6 +239,18 @@ struct fetch * fetch_start(char *url, char *referer,
 	if (res == URL_FUNC_NOMEM)
 		goto failed;
 
+	res = url_scheme(url, &ref1);
+	/* we only fail memory exhaustion */
+	if (res == URL_FUNC_NOMEM)
+		goto failed;
+
+	if (referer) {
+		res = url_scheme(referer, &ref2);
+		/* we only fail memory exhaustion */
+		if (res == URL_FUNC_NOMEM)
+			goto failed;
+	}
+
 	LOG(("fetch %p, url '%s'", fetch, url));
 
 	/* construct a new fetch structure */
@@ -250,8 +263,11 @@ struct fetch * fetch_start(char *url, char *referer,
 	fetch->cookies = cookies;
 	fetch->url = strdup(url);
 	fetch->referer = 0;
-	if (referer)
+	/* only send the referer if the schemes match */
-		fetch->referer = strdup(referer);
+	if (referer) {
+		if (ref1 && ref2 && strcasecmp(ref1, ref2) == 0)
+			fetch->referer = strdup(referer);
+	}
 	fetch->p = p;
 	fetch->headers = 0;
 	fetch->host = host;
@@ -269,11 +285,23 @@ struct fetch * fetch_start(char *url, char *referer,
 	fetch->prev = 0;
 	fetch->next = 0;
 
-	if (!fetch->url || (referer && !fetch->referer) ||
+	if (!fetch->url || (referer &&
+			(ref1 && ref2 && strcasecmp(ref1, ref2) == 0) &&
+			!fetch->referer) ||
 			(post_urlenc && !fetch->post_urlenc) ||
 			(post_multipart && !fetch->post_multipart))
 		goto failed;
 
+	/* these aren't needed past here */
+	if (ref1) {
+		free(ref1);
+		ref1 = 0;
+	}
+	if (ref2) {
+		free(ref2);
+		ref2 = 0;
+	}
+
 #define APPEND(list, value) \
 	slist = curl_slist_append(list, value);		\
 	if (!slist)					\
@@ -336,6 +364,10 @@ struct fetch * fetch_start(char *url, char *referer,
 
 failed:
 	free(host);
+	if (ref1)
+		free(ref1);
+	if (ref2)
+		free(ref2);
 	free(fetch->url);
 	free(fetch->referer);
 	free(fetch->post_urlenc);

desktop/browser.c

@@ -29,6 +29,7 @@
 #include "netsurf/desktop/browser.h"
 #include "netsurf/desktop/gui.h"
 #include "netsurf/desktop/imagemap.h"
+#include "netsurf/desktop/options.h"
 #include "netsurf/render/box.h"
 #include "netsurf/render/font.h"
 #include "netsurf/render/form.h"
@@ -82,11 +83,13 @@ static void browser_form_submit(struct browser_window *bw, struct form *form,
 /**
  * Create and open a new browser window with the given page.
  *
- * \param  url   URL to start fetching in the new window (copied)
+ * \param  url     URL to start fetching in the new window (copied)
- * \param  clone The browser window to clone
+ * \param  clone   The browser window to clone
+ * \param  referer The referring uri
  */
 
-void browser_window_create(const char *url, struct browser_window *clone)
+void browser_window_create(const char *url, struct browser_window *clone,
+		char *referer)
 {
 	struct browser_window *bw;
 
@@ -106,7 +109,7 @@ void browser_window_create(const char *url, struct browser_window *clone)
 		free(bw);
 		return;
 	}
-	browser_window_go(bw, url, false);
+	browser_window_go(bw, url, referer);
 }
 
 
@@ -115,13 +118,13 @@ void browser_window_create(const char *url, struct browser_window *clone)
  *
  * \param  bw      browser window
  * \param  url     URL to start fetching (copied)
- * \param  referer whether to send the referer header
+ * \param  referer the referring uri
  *
  * Any existing fetches in the window are aborted.
  */
 
 void browser_window_go(struct browser_window *bw, const char *url,
-		bool referer)
+		char* referer)
 {
 	browser_window_go_post(bw, url, 0, 0, true, referer);
 }
@@ -135,6 +138,7 @@ void browser_window_go(struct browser_window *bw, const char *url,
  * \param  post_urlenc     url encoded post data, or 0 if none
  * \param  post_multipart  multipart post data, or 0 if none
  * \param  history_add     add to window history
+ * \param  referer         the referring uri
  *
  * Any existing fetches in the window are aborted.
  *
@@ -147,7 +151,7 @@ void browser_window_go(struct browser_window *bw, const char *url,
 void browser_window_go_post(struct browser_window *bw, const char *url,
 		char *post_urlenc,
 		struct form_successful_control *post_multipart,
-		bool history_add, bool referer)
+		bool history_add, char *referer)
 {
 	struct content *c;
 	char *url2;
@@ -191,7 +195,7 @@ void browser_window_go_post(struct browser_window *bw, const char *url,
 	bw->loading_content = c;
 	browser_window_start_throbber(bw);
 
-	fetchcache_go(c, referer ? gui_window_get_url(bw->window) : 0,
+	fetchcache_go(c, option_send_referer ? referer : 0,
 			browser_window_callback, bw, 0,
 			post_urlenc, post_multipart, true);
 }
@@ -277,8 +281,27 @@ void browser_window_callback(content_msg msg, struct content *c,
 			bw->loading_content = 0;
 			browser_window_set_status(bw,
 					messages_get("Redirecting"));
-			/* hmm, should we do send the referrer here? */
+			/** \todo Send referer on redirect.
-			browser_window_go(bw, data.redirect, false);
+			 * We can't simply grab bw->current_content->url
+			 * because:
+			 *	a) This would leak data if the referer
+			 *	   wasn't sent by the initial call
+			 *	b) We may have started this fetch via
+			 *	   browser_window_create. Therefore,
+			 *	   there'd be no current_content to
+			 *	   read the referer from.
+			 *
+			 * Therefore, we either need a way of extracting
+			 * the referer information from the fetch struct
+			 * or we store the information at a higher level
+			 * (such as in the browser_window struct). I'm
+			 * not entirely sure of the best solution here,
+			 * so I've left it for now.
+			 */
+			/* the spec says nothing about referrers and
+			 * redirects => follow Mozilla and preserve the
+			 * referrer across the redirect */
+			browser_window_go(bw, data.redirect, 0);
 			break;
 
 		case CONTENT_MSG_REFORMAT:
@@ -474,7 +497,7 @@ void browser_window_reload(struct browser_window *bw, bool all)
 		}
 	}
 	bw->current_content->fresh = false;
-	browser_window_go_post(bw, bw->current_content->url, 0, 0, false, false);
+	browser_window_go_post(bw, bw->current_content->url, 0, 0, false, 0);
 }
 
 
@@ -788,9 +811,9 @@ void browser_window_mouse_click_html(struct browser_window *bw,
 				click == BROWSER_MOUSE_CLICK_2) {
 			if (fetch_can_fetch(url)) {
 				if (click == BROWSER_MOUSE_CLICK_1)
-					browser_window_go(bw, url, true);
+					browser_window_go(bw, url, c->url);
 				else
-					browser_window_create(url, bw);
+					browser_window_create(url, bw, c->url);
 			} else {
 				gui_launch_url(url);
 			}
@@ -1726,7 +1749,7 @@ void browser_form_submit(struct browser_window *bw, struct form *form,
 			res = url_join(url, base, &url1);
 			if (res != URL_FUNC_OK)
 				break;
-			browser_window_go(bw, url1, true);
+			browser_window_go(bw, url1, bw->current_content->url);
 			break;
 
 		case method_POST_URLENC:
@@ -1739,14 +1762,16 @@ void browser_form_submit(struct browser_window *bw, struct form *form,
 			res = url_join(form->action, base, &url);
 			if (res != URL_FUNC_OK)
 				break;
-			browser_window_go_post(bw, url, data, 0, true, true);
+			browser_window_go_post(bw, url, data, 0, true,
+					bw->current_content->url);
 			break;
 
 		case method_POST_MULTIPART:
 			res = url_join(form->action, base, &url);
 			if (res != URL_FUNC_OK)
 				break;
-			browser_window_go_post(bw, url, 0, success, true, true);
+			browser_window_go_post(bw, url, 0, success, true,
+					bw->current_content->url);
 			break;
 
 		default:

desktop/browser.h

@@ -70,13 +70,14 @@ typedef enum {
 } browser_mouse_click;
 
 
-void browser_window_create(const char *url, struct browser_window *clone);
+void browser_window_create(const char *url, struct browser_window *clone,
+		char *referer);
 void browser_window_go(struct browser_window *bw, const char *url,
-		bool referer);
+		char *referer);
 void browser_window_go_post(struct browser_window *bw, const char *url,
 		char *post_urlenc,
 		struct form_successful_control *post_multipart,
-		bool history_add, bool referer);
+		bool history_add, char *referer);
 void browser_window_stop(struct browser_window *bw);
 void browser_window_reload(struct browser_window *bw, bool all);
 void browser_window_destroy(struct browser_window *bw);

desktop/options.c

@@ -53,6 +53,8 @@ bool option_ssl_verify_certificates = true;
 int option_memory_cache_size = 2 * 1024 * 1024;
 /** Whether to block advertisements */
 bool option_block_ads = false;
+/** Whether to send the referer HTTP header */
+bool option_send_referer = true;
 
 EXTRA_OPTION_DEFINE
 
@@ -74,6 +76,7 @@ struct {
 	{ "ssl_verify_certificates", OPTION_BOOL, &option_ssl_verify_certificates },
 	{ "memory_cache_size", OPTION_INTEGER, &option_memory_cache_size },
 	{ "block_advertisements", OPTION_BOOL, &option_block_ads },
+	{ "send_referer",    OPTION_BOOL,    &option_send_referer },
 	EXTRA_OPTION_TABLE
 };
 

desktop/options.h

@@ -39,6 +39,7 @@ extern char *option_accept_language;
 extern bool option_ssl_verify_certificates;
 extern int option_memory_cache_size;
 extern bool option_block_ads;
+extern bool option_send_referer;
 
 void options_read(const char *path);
 void options_write(const char *path);

gtk/gtk_gui.c

@@ -34,7 +34,7 @@ void gui_init(int argc, char** argv)
 
 void gui_init2(int argc, char** argv)
 {
-	browser_window_create("http://netsurf.sourceforge.net/", 0);
+	browser_window_create("http://netsurf.sourceforge.net/", 0, 0);
 }
 
 

gtk/gtk_window.c

@@ -184,11 +184,16 @@ gboolean gui_window_url_key_press_event(GtkWidget *widget,
 		GdkEventKey *event, gpointer data)
 {
 	struct gui_window *g = data;
+	char *referer = 0;
 
 	if (event->keyval != GDK_Return)
 		return FALSE;
 
-	browser_window_go(g->bw, gtk_entry_get_text(GTK_ENTRY(g->url_bar)), false);
+	if (g->bw->current_content && g->bw->current_content->url)
+		referer = g->bw->current_content->url;
+
+	browser_window_go(g->bw, gtk_entry_get_text(GTK_ENTRY(g->url_bar)),
+				referer);
 
 	return TRUE;
 }

riscos/401login.c

@@ -99,7 +99,7 @@ bool ro_gui_401login_keypress(wimp_key *key)
 		case wimp_KEY_RETURN:
 			get_unamepwd();
 			ro_gui_dialog_close(dialog_401li);
-			browser_window_go(bwin, url, false);
+			browser_window_go(bwin, url, 0);
 			return true;
 	}
 
@@ -118,7 +118,7 @@ void ro_gui_401login_click(wimp_pointer *pointer)
 		case ICON_401LOGIN_LOGIN:
 			get_unamepwd();
 			ro_gui_dialog_close(dialog_401li);
-			browser_window_go(bwin, url, false);
+			browser_window_go(bwin, url, 0);
 			break;
 		case ICON_401LOGIN_CANCEL:
 			ro_gui_dialog_close(dialog_401li);

riscos/dialog.c

@@ -854,7 +854,7 @@ void ro_gui_dialog_click_config_th(wimp_pointer *pointer)
 		case ICON_CONFIG_TH_GET:
 			browser_window_create(
 					"http://netsurf.sourceforge.net/themes/",
-					NULL);
+					NULL, 0);
 			break;
 	}
 }

riscos/history.c

@@ -663,7 +663,7 @@ void history_go(struct browser_window *bw, struct history_entry *entry)
 	else
 		url = entry->url;
 
-	browser_window_go_post(bw, url, 0, 0, false, false);
+	browser_window_go_post(bw, url, 0, 0, false, 0);
 
 	if (entry->frag_id)
 		free(url);

riscos/hotlist.c

@@ -1612,7 +1612,7 @@ void ro_gui_hotlist_click(wimp_pointer *pointer) {
 			*/
 			if (((buttons == wimp_DOUBLE_SELECT) || (buttons == wimp_DOUBLE_ADJUST)) &&
 					(entry->children == -1)) {
-				browser_window_create(entry->url, NULL);
+				browser_window_create(entry->url, NULL, 0);
 				if (buttons == wimp_DOUBLE_SELECT) {
 					ro_gui_hotlist_selection_state(root.child_entry,
 							false, true);
@@ -1872,7 +1872,7 @@ int ro_gui_hotlist_selection_count(struct hotlist_entry *entry, bool folders) {
 void ro_gui_hotlist_launch_selection(struct hotlist_entry *entry) {
 	if (!entry) return;
 	while (entry) {
-		if ((entry->selected) && (entry->url)) browser_window_create(entry->url, NULL);
+		if ((entry->selected) && (entry->url)) browser_window_create(entry->url, NULL, 0);
 		if (entry->child_entry) ro_gui_hotlist_launch_selection(entry->child_entry);
 		entry = entry->next_entry;
 	}

riscos/menus.c

@@ -770,7 +770,7 @@ void ro_gui_menu_selection(wimp_selection *selection)
 					case 5: /* Print */
 						break;
 					case 6: /* New window */
-						browser_window_create(current_gui->bw->current_content->url, current_gui->bw);
+						browser_window_create(current_gui->bw->current_content->url, current_gui->bw, 0);
 						break;
 					case 7: /* Page source */
 						ro_gui_view_source(c);
@@ -817,12 +817,12 @@ void ro_gui_menu_selection(wimp_selection *selection)
 				switch (selection->items[1]) {
 					case 0: /* Home */
 						if (option_homepage_url && option_homepage_url[0]) {
-							browser_window_go_post(current_gui->bw, option_homepage_url, 0, 0, true, false);
+							browser_window_go_post(current_gui->bw, option_homepage_url, 0, 0, true, 0);
 						} else {
 							snprintf(url, sizeof url,
 									"file:/<NetSurf$Dir>/Docs/intro_%s",
 									option_language);
-							browser_window_go_post(current_gui->bw, url, 0, 0, true, false);
+							browser_window_go_post(current_gui->bw, url, 0, 0, true, 0);
 						}
 						break;
 					case 1: /* Back */
@@ -967,7 +967,7 @@ void ro_gui_menu_selection(wimp_selection *selection)
 						break;
 					case 3: /* About NetSurf */
 						browser_window_create("file:/"
-						"<NetSurf$Dir>/Docs/about", 0);
+						"<NetSurf$Dir>/Docs/about", 0, 0);
 						break;
 					case 4: /* Interactive help */
 						xos_cli("Filer_Run Resources:$.Apps.!Help");

riscos/plugin.c

@@ -730,10 +730,10 @@ void plugin_url_access(wimp_message *message)
 			    strcasecmp(window, "_parent") == 0 ||
 			    strcasecmp(window, "_top") == 0 ||
 			    strcasecmp(window, "") == 0) {
-				browser_window_go(c->data.plugin.bw, url, false);
+				browser_window_go(c->data.plugin.bw, url, 0);
 			}
 			else if (strcasecmp(window, "_blank") == 0) {
-				browser_window_create(url, NULL);
+				browser_window_create(url, NULL, 0);
 			}
 		}
 		else { /* POST request */

riscos/uri.c

@@ -55,7 +55,7 @@ void ro_uri_message_received(uri_full_message_process* uri_message)
 
   xuri_request_uri(0, uri_requested, uri_length, uri_handle, NULL);
 
-  browser_window_create(uri_requested, NULL);
+  browser_window_create(uri_requested, NULL, 0);
 
   xfree(uri_requested);
 }

riscos/url_protocol.c

@@ -103,7 +103,7 @@ void ro_url_message_received(wimp_message *message)
 	}
 
 	/* create new browser window */
-	browser_window_create(url, 0);
+	browser_window_create(url, 0, 0);
 
 	free(url);
 }

riscos/window.c

@@ -1099,18 +1099,18 @@ void ro_gui_toolbar_click(struct gui_window *g, wimp_pointer *pointer)
 			if (option_homepage_url && option_homepage_url[0]) {
 				if (pointer->buttons == wimp_CLICK_SELECT) {
 					browser_window_go_post(g->bw, option_homepage_url,
-							0, 0, true, false);
+							0, 0, true, 0);
 				} else {
-					browser_window_create(option_homepage_url, NULL);
+					browser_window_create(option_homepage_url, NULL, 0);
 				}
 			} else {
 				snprintf(url, sizeof url,
 						"file:/<NetSurf$Dir>/Docs/intro_%s",
 						option_language);
 				if (pointer->buttons == wimp_CLICK_SELECT) {
-					browser_window_go_post(g->bw, url, 0, 0, true, false);
+					browser_window_go_post(g->bw, url, 0, 0, true, 0);
 				} else {
-					browser_window_create(url, NULL);
+					browser_window_create(url, NULL, 0);
 				}
 			}
 			break;
@@ -1461,7 +1461,7 @@ bool ro_gui_window_keypress(struct gui_window *g, int key, bool toolbar)
 			res = url_normalize(toolbar_url, &url);
 			if (res == URL_FUNC_OK) {
 				gui_window_set_url(g, url);
-				browser_window_go(g->bw, url, false);
+				browser_window_go(g->bw, url, 0);
 				free(url);
 			}
 			return true;
@@ -1473,7 +1473,7 @@ bool ro_gui_window_keypress(struct gui_window *g, int key, bool toolbar)
 		case 14:	/* CTRL+N */
 			current_gui = g;
 			browser_window_create(current_gui->bw->current_content->url,
-					current_gui->bw);
+					current_gui->bw, 0);
 			return true;
 		case 18:	/* CTRL+R */
 			browser_window_reload(g->bw, false);

utils/url.c

@@ -411,6 +411,39 @@ url_func_result url_host(const char *url, char **result)
 	return URL_FUNC_OK;
 }
 
+/**
+ * Return the scheme name from an URL
+ *
+ * \param url     an absolute URL
+ * \param result  pointer to pointer to buffer to hold scheme name
+ * \return URL_FUNC_OK on success
+ */
+url_func_result url_scheme(const char *url, char **result)
+{
+	int m;
+	regmatch_t match[10];
+
+	(*result) = 0;
+
+	m = regexec(&url_re, url, 10, match, 0);
+	if (m) {
+		LOG(("url '%s' failed to match regex", url));
+		return URL_FUNC_FAILED;
+	}
+	if (match[2].rm_so == -1)
+		return URL_FUNC_FAILED;
+
+	(*result) = malloc(match[2].rm_eo - match[2].rm_so + 1);
+	if (!(*result)) {
+		LOG(("malloc failed"));
+		return URL_FUNC_NOMEM;
+	}
+
+	strncpy((*result), url + match[2].rm_so, match[2].rm_eo - match[2].rm_so);
+	(*result)[match[2].rm_eo - match[2].rm_so] = 0;
+
+	return URL_FUNC_OK;
+}
 
 /**
  * Attempt to find a nice filename for a URL.
@@ -514,8 +547,8 @@ int main(int argc, char *argv[])
 	url_func_result res;
 	char *s;
 	url_init();
-/*	for (i = 1; i != argc; i++) {
+	for (i = 1; i != argc; i++) {
-		printf("==> '%s'\n", argv[i]);
+/*		printf("==> '%s'\n", argv[i]);
 		res = url_normalize(argv[i], &s);
 		if (res == URL_FUNC_OK) {
 			printf("<== '%s'\n", s);
@@ -527,19 +560,19 @@ int main(int argc, char *argv[])
 			printf("<== '%s'\n", s);
 			free(s);
 		}*/
-/*		if (1 != i) {
+		if (1 != i) {
 			res = url_join(argv[i], argv[1], &s);
 			if (res == URL_FUNC_OK) {
 				printf("'%s' + '%s' \t= '%s'\n", argv[1],
 						argv[i], s);
 				free(s);
 			}
-		}*/
+		}
-		res = url_nice(argv[i], &s);
+/*		res = url_nice(argv[i], &s);
 		if (res == URL_FUNC_OK) {
 			printf("'%s'\n", s);
 			free(s);
-		}
+		}*/
 	}
 	return 0;
 }

utils/url.h

@@ -22,6 +22,7 @@ void url_init(void);
 url_func_result url_normalize(const char *url, char **result);
 url_func_result url_join(const char *rel, const char *base, char **result);
 url_func_result url_host(const char *url, char **result);
+url_func_result url_scheme(const char *url, char **result);
 url_func_result url_nice(const char *url, char **result);
 
 #endif