tty: Implement exclusive mode

Implemented exclusive mode on Haiku and added the related `ioctl`
operations (`TIOCEXCL` and `TIOCNXCL`).

Change-Id: Iaa201ea20eec0e45d02dd5db9ba6aa35fd27dfb2
Reviewed-on: https://review.haiku-os.org/c/haiku/+/6387
Tested-by: Commit checker robot <no-reply+buildbot@haiku-os.org>
Reviewed-by: Jessica Hamilton <jessica.l.hamilton@gmail.com>

headers/os/drivers/tty/tty_module.h

@@ -46,12 +46,12 @@ typedef struct tty_module_info tty_module_info;
 struct tty_module_info {
 	module_info	mi;
 
-	struct tty *(*tty_create)(tty_service_func serviceFunction, struct tty* master);
+	status_t	(*tty_create)(tty_service_func serviceFunction, struct tty *master,
+					struct tty **tty);
 	void		(*tty_destroy)(struct tty *tty);
 
-	struct tty_cookie *
+	status_t	(*tty_create_cookie)(struct tty *masterTTY, struct tty *slaveTTY,
-				(*tty_create_cookie)(struct tty *masterTTY, struct tty *slaveTTY,
+					uint32 openMode, struct tty_cookie **cookie);
-					uint32 openMode);
 	void		(*tty_close_cookie)(struct tty_cookie *cookie);
 	void		(*tty_destroy_cookie)(struct tty_cookie *cookie);
 

headers/posix/termios.h

@@ -189,6 +189,8 @@ struct termios {
 #define TIOCMBIC			(TCGETA + 23)	/* clear bits in line state */
 #define	TIOCGSID			(TCGETA + 24)	/* get session leader process group ID */
 #define TIOCOUTQ			(TCGETA + 25)	/* get output queue size */
+#define TIOCEXCL			(TCGETA + 26)	/* set exclusive use of tty */
+#define TIOCNXCL			(TCGETA + 27)	/* clear exclusive use of tty */
 
 /* Event codes.  Returned from TCWAITEVENT */
 #define EV_RING			0x0001

src/add-ons/kernel/drivers/ports/pc_serial/SerialDevice.cpp

@@ -545,36 +545,34 @@ SerialDevice::Open(uint32 flags)
 	if (fDeviceRemoved)
 		return B_DEV_NOT_READY;
 
-	fMasterTTY = gTTYModule->tty_create(pc_serial_service, NULL);
+	status = gTTYModule->tty_create(pc_serial_service, NULL, &fMasterTTY);
-	if (fMasterTTY == NULL) {
+	if (status != B_OK) {
 		TRACE_ALWAYS("open: failed to init master tty\n");
-		return B_NO_MEMORY;
+		return status;
 	}
 
-	fSlaveTTY = gTTYModule->tty_create(pc_serial_service, fMasterTTY);
+	status = gTTYModule->tty_create(pc_serial_service, fMasterTTY, &fSlaveTTY);
-	if (fSlaveTTY == NULL) {
+	if (status != B_OK) {
 		TRACE_ALWAYS("open: failed to init slave tty\n");
 		gTTYModule->tty_destroy(fMasterTTY);
-		return B_NO_MEMORY;
+		return status;
 	}
 
-	fSystemTTYCookie = gTTYModule->tty_create_cookie(fMasterTTY, fSlaveTTY,
+	status = gTTYModule->tty_create_cookie(fMasterTTY, fSlaveTTY, O_RDWR, &fSystemTTYCookie);
-		O_RDWR);
+	if (status != B_OK) {
-	if (fSystemTTYCookie == NULL) {
 		TRACE_ALWAYS("open: failed to init system tty cookie\n");
 		gTTYModule->tty_destroy(fMasterTTY);
 		gTTYModule->tty_destroy(fSlaveTTY);
-		return B_NO_MEMORY;
+		return status;
 	}
 
-	fDeviceTTYCookie = gTTYModule->tty_create_cookie(fSlaveTTY, fMasterTTY,
+	status = gTTYModule->tty_create_cookie(fSlaveTTY, fMasterTTY, O_RDWR, &fDeviceTTYCookie);
-		O_RDWR);
+	if (status != B_OK) {
-	if (fDeviceTTYCookie == NULL) {
 		TRACE_ALWAYS("open: failed to init device tty cookie\n");
 		gTTYModule->tty_destroy_cookie(fSystemTTYCookie);
 		gTTYModule->tty_destroy(fMasterTTY);
 		gTTYModule->tty_destroy(fSlaveTTY);
-		return B_NO_MEMORY;
+		return status;
 	}
 
 	ResetDevice();

src/add-ons/kernel/drivers/ports/usb_serial/SerialDevice.cpp

@@ -287,42 +287,42 @@ SerialDevice::Service(struct tty *tty, uint32 op, void *buffer, size_t length)
 status_t
 SerialDevice::Open(uint32 flags)
 {
+	status_t status = B_OK;
+
 	if (fDeviceOpen)
 		return B_BUSY;
 
 	if (fDeviceRemoved)
 		return B_DEV_NOT_READY;
 
-	fMasterTTY = gTTYModule->tty_create(usb_serial_service, NULL);
+	status = gTTYModule->tty_create(usb_serial_service, NULL, &fMasterTTY);
-	if (fMasterTTY == NULL) {
+	if (status != B_OK) {
 		TRACE_ALWAYS("open: failed to init master tty\n");
-		return B_NO_MEMORY;
+		return status;
 	}
 
-	fSlaveTTY = gTTYModule->tty_create(usb_serial_service, fMasterTTY);
+	status = gTTYModule->tty_create(usb_serial_service, fMasterTTY, &fSlaveTTY);
-	if (fSlaveTTY == NULL) {
+	if (status != B_OK) {
 		TRACE_ALWAYS("open: failed to init slave tty\n");
 		gTTYModule->tty_destroy(fMasterTTY);
-		return B_NO_MEMORY;
+		return status;
 	}
 
-	fSystemTTYCookie = gTTYModule->tty_create_cookie(fMasterTTY, fSlaveTTY,
+	status = gTTYModule->tty_create_cookie(fMasterTTY, fSlaveTTY, O_RDWR, &fSystemTTYCookie);
-		O_RDWR);
+	if (status != B_OK) {
-	if (fSystemTTYCookie == NULL) {
 		TRACE_ALWAYS("open: failed to init system tty cookie\n");
 		gTTYModule->tty_destroy(fMasterTTY);
 		gTTYModule->tty_destroy(fSlaveTTY);
-		return B_NO_MEMORY;
+		return status;
 	}
 
-	fDeviceTTYCookie = gTTYModule->tty_create_cookie(fSlaveTTY, fMasterTTY,
+	status = gTTYModule->tty_create_cookie(fSlaveTTY, fMasterTTY, O_RDWR, &fDeviceTTYCookie);
-		O_RDWR);
+	if (status != B_OK) {
-	if (fDeviceTTYCookie == NULL) {
 		TRACE_ALWAYS("open: failed to init device tty cookie\n");
 		gTTYModule->tty_destroy_cookie(fSystemTTYCookie);
 		gTTYModule->tty_destroy(fMasterTTY);
 		gTTYModule->tty_destroy(fSlaveTTY);
-		return B_NO_MEMORY;
+		return status;
 	}
 
 	ResetDevice();
@@ -342,9 +342,8 @@ SerialDevice::Open(uint32 flags)
 		| USB_CDC_CONTROL_SIGNAL_STATE_RTS;
 	SetControlLineState(fControlOut);
 
-	status_t status = gUSBModule->queue_interrupt(fControlPipe,
+	status = gUSBModule->queue_interrupt(fControlPipe, fInterruptBuffer, fInterruptBufferSize,
-		fInterruptBuffer, fInterruptBufferSize, _InterruptCallbackFunction,
+		_InterruptCallbackFunction, this);
-		this);
 	if (status < B_OK)
 		TRACE_ALWAYS("failed to queue initial interrupt\n");
 

src/add-ons/kernel/drivers/pty/driver.cpp

@@ -222,21 +222,23 @@ master_open(const char *name, uint32 flags, void **_cookie)
 		return B_BUSY;
 	}
 
+	status_t status = B_OK;
+
 	if (gMasterTTYs[index] == NULL) {
-		gMasterTTYs[index] = gTTYModule->tty_create(master_service, NULL);
+		status = gTTYModule->tty_create(master_service, NULL, &gMasterTTYs[index]);
-		if (gMasterTTYs[index] == NULL)
+		if (status != B_OK)
-			return B_NO_MEMORY;
+			return status;
 	}
 	if (gSlaveTTYs[index] == NULL) {
-		gSlaveTTYs[index] = gTTYModule->tty_create(slave_service, gMasterTTYs[index]);
+		status = gTTYModule->tty_create(slave_service, gMasterTTYs[index], &gSlaveTTYs[index]);
-		if (gSlaveTTYs[index] == NULL)
+		if (status != B_OK)
-			return B_NO_MEMORY;
+			return status;
 	}
 
-	tty_cookie *cookie = gTTYModule->tty_create_cookie(gMasterTTYs[index],
+	tty_cookie *cookie;
-		gSlaveTTYs[index], flags);
+	status = gTTYModule->tty_create_cookie(gMasterTTYs[index], gSlaveTTYs[index], flags, &cookie);
-	if (cookie == NULL)
+	if (status != B_OK)
-		return B_NO_MEMORY;
+		return status;
 
 	*_cookie = cookie;
 	return B_OK;
@@ -301,12 +303,11 @@ slave_open(const char *name, uint32 flags, void **_cookie)
 		gSlaveTTYs[index]->settings->pgrp_id = -1;
 	}
 
-	tty_cookie *cookie = gTTYModule->tty_create_cookie(gSlaveTTYs[index],
+	tty_cookie *cookie;
-		gMasterTTYs[index], flags);
+	status_t status = gTTYModule->tty_create_cookie(gSlaveTTYs[index], gMasterTTYs[index], flags,
-	if (cookie == NULL) {
+		&cookie);
-		gSlaveTTYs[index] = NULL;
+	if (status != B_OK)
-		return B_NO_MEMORY;
+		return status;
-	}
 
 	if (makeControllingTTY) {
 		gSlaveTTYs[index]->settings->session_id = sessionID;

src/add-ons/kernel/generic/tty/tty.cpp

@@ -1324,12 +1324,12 @@ tty_write_to_tty_slave(tty_cookie* sourceCookie, const void* _buffer,
 // #pragma mark - public API
 
 
-struct tty*
+status_t
-tty_create(tty_service_func func, struct tty* master)
+tty_create(tty_service_func func, struct tty* master, struct tty** _tty)
 {
 	struct tty* tty = new(std::nothrow) (struct tty);
 	if (tty == NULL)
-		return NULL;
+		return B_NO_MEMORY;
 
 	if (master == NULL) {
 		tty->is_master = true;
@@ -1339,7 +1339,7 @@ tty_create(tty_service_func func, struct tty* master)
 			delete tty->lock;
 			delete tty->settings;
 			delete tty;
-			return NULL;
+			return B_NO_MEMORY;
 		}
 
 		recursive_lock_init(tty->lock, "tty lock");
@@ -1356,19 +1356,24 @@ tty_create(tty_service_func func, struct tty* master)
 	tty->select_pool = NULL;
 	tty->pending_eof = 0;
 	tty->hardware_bits = 0;
+	tty->is_exclusive = false;
 
-	if (init_line_buffer(tty->input_buffer, TTY_BUFFER_SIZE) < B_OK) {
+	status_t status = init_line_buffer(tty->input_buffer, TTY_BUFFER_SIZE);
+
+	if (status < B_OK) {
 		if (tty->is_master) {
 			recursive_lock_destroy(tty->lock);
 			delete tty->lock;
 		}
 		delete tty;
-		return NULL;
+		return status;
 	}
 
 	tty->service_func = func;
 
-	return tty;
+	*_tty = tty;
+
+	return B_OK;
 }
 
 
@@ -1388,17 +1393,18 @@ tty_destroy(struct tty* tty)
 }
 
 
-tty_cookie*
+status_t
-tty_create_cookie(struct tty* tty, struct tty* otherTTY, uint32 openMode)
+tty_create_cookie(struct tty* tty, struct tty* otherTTY, uint32 openMode, tty_cookie** _cookie)
 {
 	tty_cookie* cookie = new(std::nothrow) tty_cookie;
 	if (cookie == NULL)
-		return NULL;
+		return B_NO_MEMORY;
 
 	cookie->blocking_semaphore = create_sem(0, "wait for tty close");
 	if (cookie->blocking_semaphore < 0) {
+		status_t status = cookie->blocking_semaphore;
 		delete cookie;
-		return NULL;
+		return status;
 	}
 
 	cookie->tty = tty;
@@ -1410,13 +1416,21 @@ tty_create_cookie(struct tty* tty, struct tty* otherTTY, uint32 openMode)
 
 	RecursiveLocker locker(cookie->tty->lock);
 
+	if (tty->is_exclusive && geteuid() != 0) {
+		delete_sem(cookie->blocking_semaphore);
+		delete cookie;
+		return B_BUSY;
+	}
+
 	// add to the TTY's cookie list
 	tty->cookies.Add(cookie);
 	tty->open_count++;
 	tty->ref_count++;
 	tty->opened_count++;
 
-	return cookie;
+	*_cookie = cookie;
+
+	return B_OK;
 }
 
 
@@ -1513,6 +1527,8 @@ tty_close_cookie(tty_cookie* cookie)
 		// notify a select write event on the other tty, if we've closed this tty
 		if (cookie->other_tty->open_count > 0)
 			tty_notify_select_event(cookie->other_tty, B_SELECT_WRITE);
+
+		cookie->tty->is_exclusive = false;
 	}
 }
 
@@ -1944,6 +1960,18 @@ tty_control(tty_cookie* cookie, uint32 op, void* buffer, size_t length)
 
 			return B_ERROR;
 		}
+
+		case TIOCEXCL:
+		{
+			tty->is_exclusive = true;
+			return B_OK;
+		}
+
+		case TIOCNXCL:
+		{
+			tty->is_exclusive = false;
+			return B_OK;
+		}
 	}
 
 	TRACE(("tty: unsupported opcode %" B_PRIu32 "\n", op));

src/add-ons/kernel/generic/tty/tty_private.h

@@ -140,17 +140,19 @@ struct tty {
 	recursive_lock*		lock;
 	tty_settings*		settings;
 	uint8				hardware_bits;
+	bool				is_exclusive;
 };
 
 
 extern struct mutex gTTYCookieLock;
 extern struct recursive_lock gTTYRequestLock;
 
-extern struct tty *tty_create(tty_service_func func, struct tty* masterTTY);
+extern status_t tty_create(tty_service_func func, struct tty *masterTTY,
+	struct tty **tty);
 extern void tty_destroy(struct tty *tty);
 
-extern tty_cookie *tty_create_cookie(struct tty *tty, struct tty *otherTTY,
+extern status_t tty_create_cookie(struct tty *tty, struct tty *otherTTY,
-	uint32 openMode);
+	uint32 openMode, struct tty_cookie **cookie);
 extern void tty_destroy_cookie(tty_cookie *cookie);
 extern void tty_close_cookie(tty_cookie *cookie);
 

src/bin/debug/strace/ioctl.cpp

@@ -137,6 +137,8 @@ static const ioctl_info kIOCtls[] = {
 	IOCTL_INFO_ENTRY(TIOCMBIS),
 	IOCTL_INFO_ENTRY(TIOCMBIC),
 	IOCTL_INFO_ENTRY(TIOCOUTQ),
+	IOCTL_INFO_ENTRY(TIOCEXCL),
+	IOCTL_INFO_ENTRY(TIOCNXCL),
 	// private termios
 	IOCTL_INFO_ENTRY(B_IOCTL_GET_TTY_INDEX),
 	IOCTL_INFO_ENTRY(B_IOCTL_GRANT_TTY),