runtime_loader: Detect and trigger on PE binaries

* Previously PE binaries would trigger the "incorrectly
  executable" dialog. Now we get a special message for
  B_LEGACY_EXECUTABLE and B_UNKNOWN_EXECUTABLE
* Legacy at the moment is a R3 x86 PE binary. This could
  be extended to gcc2 binaries someday far, far, down the
  road though
* The check for legacy is based on a PE flag I see
  set on every R3 binary (that isn't set on dos ones)
* Unknown is something we know *is* an executable, but
  can't do anything with (such as an MSDOS or Windows
  application)
* No performance drops as we do the PE scan last
* Tested on x86 and x86_gcc2

headers/build/os/support/Errors.h

@@ -82,6 +82,8 @@ enum {
 	B_NOT_AN_EXECUTABLE,
 	B_MISSING_LIBRARY,
 	B_MISSING_SYMBOL,
+	B_UNKNOWN_EXECUTABLE,
+	B_LEGACY_EXECUTABLE,
 
 	B_DEBUGGER_ALREADY_INSTALLED = B_OS_ERROR_BASE + 0x400
 };

headers/os/support/Errors.h

@@ -71,6 +71,8 @@
 #define B_NOT_AN_EXECUTABLE			(B_OS_ERROR_BASE + 0x302)
 #define B_MISSING_LIBRARY			(B_OS_ERROR_BASE + 0x303)
 #define B_MISSING_SYMBOL			(B_OS_ERROR_BASE + 0x304)
+#define B_UNKNOWN_EXECUTABLE		(B_OS_ERROR_BASE + 0x305)
+#define B_LEGACY_EXECUTABLE			(B_OS_ERROR_BASE + 0x306)
 
 #define B_DEBUGGER_ALREADY_INSTALLED	(B_OS_ERROR_BASE + 0x400)
 

headers/private/system/pe_common.h

@@ -0,0 +1,89 @@
+/*
+ * Copyright 2013-2014, Haiku, Inc.
+ * Distributed under the terms of the MIT License.
+ *
+ * Authors:
+ *	Alexander von Gluck IV, <kallisti5@unixzen.com>
+ */
+#ifndef _PE_COMMON_H
+#define _PE_COMMON_H
+
+
+#include <SupportDefs.h>
+#include <ByteOrder.h>
+
+
+// Magic strings
+#define MZ_MAGIC "MZ"
+#define PE_MAGIC "PE"
+#define PE_OPTIONAL_MAGIC_PE32 0x010b
+#define PE_OPTIONAL_MAGIC_PE32P 0x020b
+
+
+typedef struct {
+	uint16 magic; /* == MZ_MAGIC */
+	uint16 bytesInLastBlock;
+	uint16 blocksInFile;
+	uint16 numRelocations;
+	uint16 headerParagraphs;
+	uint16 minExtraParagraphs;
+	uint16 maxExtraParagraphs;
+	uint16 ss;
+	uint16 sp;
+	uint16 checksum;
+	uint16 ip;
+	uint16 cs;
+	uint16 relocationTableOffset;
+	uint16 overlayNumber;
+	uint16 reserved[4];
+	uint16 oemID;
+	uint16 oemInfo;
+	uint16 reserved2[10];
+	uint32 lfaNew;  // PE Header start addr
+} MzHeader;
+
+typedef struct {
+	uint32 magic; // == PE_MAGIC */
+	uint16 machine;
+	uint16 numberOfSections;
+	uint32 timeDateStamp;
+	uint32 pointerToSymbolTable;
+	uint32 numberOfSymbols;
+	uint16 sizeOfOptionalHeader;
+	uint16 characteristics;
+} PeHeader;
+
+typedef struct {
+	uint16 magic; // == 0x010b - PE32, 0x020b - PE32+ (64 bit)
+	uint8  majorLinkerVersion;
+	uint8  minorLinkerVersion;
+	uint32 sizeOfCode;
+	uint32 sizeOfInitializedData;
+	uint32 sizeOfUninitializedData;
+	uint32 addressOfEntryPoint;
+	uint32 baseOfCode;
+	uint32 baseOfData;
+	uint32 imageBase;
+	uint32 sectionAlignment;
+	uint32 fileAlignment;
+	uint16 majorOperatingSystemVersion;
+	uint16 minorOperatingSystemVersion;
+	uint16 majorImageVersion;
+	uint16 minorImageVersion;
+	uint16 majorSubsystemVersion;
+	uint16 minorSubsystemVersion;
+	uint32 win32VersionValue;
+	uint32 sizeOfImage;
+	uint32 sizeOfHeaders;
+	uint32 checksum;
+	uint16 subsystem;
+	uint16 llCharacteristics;
+	uint32 sizeOfStackReserve;
+	uint32 sizeOfStackCommit;
+	uint32 sizeOfHeapReserve;
+	uint32 sizeOfHeapCommit;
+	uint32 loaderFlags;
+	uint32 numberOfRvaAndSizes;
+} Pe32OptionalHeader;
+
+#endif /* _PE_COMMON_H */

src/kits/tracker/FSUtils.cpp

@@ -3514,8 +3514,19 @@ _TrackerLaunchDocuments(const entry_ref* /*doNotUse*/, const BMessage* refs,
 			openWithOK = false;
 			openedDocuments = false;
 		}
-
-		if (error == B_LAUNCH_FAILED_EXECUTABLE && !refsToPass) {
+		if (error == B_UNKNOWN_EXECUTABLE && !refsToPass) {
+			// We know it's an executable, but something unsupported
+			alertString.SetTo(B_TRANSLATE("\"%name\" is an unsupported "
+				"executable."));
+			alertString.ReplaceFirst("%name", app.name);
+		} else if (error == B_LEGACY_EXECUTABLE && !refsToPass) {
+			// For the moment, this marks an old R3 binary, we may want to
+			// extend it to gcc2 binaries someday post R1
+			alertString.SetTo(B_TRANSLATE("\"%name\" is a legacy executable. "
+				"Please obtain an updated version or recompile "
+				"the application."));
+			alertString.ReplaceFirst("%name", app.name);
+		} else if (error == B_LAUNCH_FAILED_EXECUTABLE && !refsToPass) {
 			alertString.SetTo(B_TRANSLATE("Could not open \"%name\". "
 				"The file is mistakenly marked as executable. "));
 			alertString.ReplaceFirst("%name", app.name);

src/system/runtime_loader/Jamfile

@@ -76,6 +76,7 @@ local sources =
 	elf_load_image.cpp
 	elf_symbol_lookup.cpp
 	elf_versioning.cpp
+	pe.cpp
 	errors.cpp
 	export.cpp
 	heap.cpp

src/system/runtime_loader/pe.cpp

@@ -0,0 +1,70 @@
+/*
+ * Copyright 2013-2014, Haiku, Inc.
+ * Distributed under the terms of the MIT License.
+ *
+ * Authors:
+ *	Alexander von Gluck IV, <kallisti5@unixzen.com>
+ */
+
+
+#include "pe.h"
+
+#include <ctype.h>
+#include <dlfcn.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+
+static status_t
+parse_mz_header(MzHeader* mzHeader, off_t* peOffset)
+{
+	if (memcmp(&mzHeader->magic, MZ_MAGIC, 2) != 0)
+		return B_NOT_AN_EXECUTABLE;
+
+	*peOffset = (off_t)mzHeader->lfaNew;
+	return B_OK;
+}
+
+
+static status_t
+parse_pe_header(PeHeader* peHeader)
+{
+	if (memcmp(&peHeader->magic, PE_MAGIC, 2) != 0)
+		return B_NOT_AN_EXECUTABLE;
+
+	// Looks like an old BeOS R3 x86 program
+	if (peHeader->characteristics == 0x10E)
+		return B_LEGACY_EXECUTABLE;
+
+	return B_OK;
+}
+
+
+/*! Read and verify the PE header */
+status_t
+pe_verify_header(void *header, size_t length)
+{
+	if (length < sizeof(MzHeader))
+		return B_NOT_AN_EXECUTABLE;
+
+	// Verify MZ header, pull PE header offset
+	off_t peOffset = 0;
+	if (parse_mz_header((MzHeader*)header, &peOffset) != B_OK)
+		return B_NOT_AN_EXECUTABLE;
+
+	// MS-DOS program
+	if (peOffset == 0)
+		return B_UNKNOWN_EXECUTABLE;
+
+	// Something is wrong with the binary
+	if (peOffset + sizeof(PeHeader) > length)
+		return B_UNKNOWN_EXECUTABLE;
+
+	// Find the PE header based on MZ provided offset
+	uint8* pePtr = (uint8*)header;
+	pePtr += peOffset;
+
+	// Win32 program or old BeOS R3 x86 program
+	return parse_pe_header((PeHeader*)pePtr);
+}

src/system/runtime_loader/pe.h

@@ -0,0 +1,23 @@
+/*
+ * Copyright 2013-2014, Haiku, Inc.
+ * Distributed under the terms of the MIT License.
+ *
+ * Authors:
+ *  Alexander von Gluck IV, <kallisti5@unixzen.com>
+ */
+#ifndef PE_H
+#define PE_H
+
+
+#include <OS.h>
+
+#include <syscalls.h>
+#include <util/kernel_cpp.h>
+
+#include "pe_common.h"
+
+
+status_t pe_verify_header(void *header, size_t length);
+
+
+#endif /* PE_H */

src/system/runtime_loader/runtime_loader.cpp

@@ -24,6 +24,7 @@
 #include <vm_defs.h>
 
 #include "elf_symbol_lookup.h"
+#include "pe.h"
 
 
 struct user_space_program_args *gProgramArgs;
@@ -396,8 +397,8 @@ test_executable(const char *name, char *invoker)
 
 	status = elf_verify_header(buffer, length);
 	if (status == B_NOT_AN_EXECUTABLE) {
-		// test for shell scripts
 		if (!strncmp(buffer, "#!", 2)) {
+			// test for shell scripts
 			char *end;
 			buffer[min_c((size_t)length, sizeof(buffer) - 1)] = '\0';
 
@@ -414,6 +415,14 @@ test_executable(const char *name, char *invoker)
 			}
 
 			status = B_OK;
+		} else {
+			// Something odd like a PE?
+			status = pe_verify_header(buffer, length);
+
+			// It is a PE, throw B_UNKNOWN_EXECUTABLE
+			// likely win32 at this point
+			if (status == B_OK)
+				status = B_UNKNOWN_EXECUTABLE;
 		}
 	} else if (status == B_OK) {
 		elf_ehdr *elfHeader = (elf_ehdr *)buffer;