should have commited it working inside qemu, oh well

go.sh

@@ -1,8 +1,8 @@
 #!/bin/sh
 set -e
 
-#BIN=../tests/ctf/ezhp
-BIN=../tests/ctf/hudak
+BIN=../tests/ctf/ezhp
+#BIN=../tests/ctf/hudak
 #BIN=../tests/ctf/simple
 #SRC=../tests/hello.c
 #SRC=../tests/algo.c
@@ -18,14 +18,13 @@ fi
 cd scripts
 #echo "hello" | ./run_qemu.sh $BIN
 #echo "4t_l34st_it_was_1mperat1v3..." | ./run_qemu.sh $BIN
-echo "i wish i were a valid key bob" | ./run_qemu.sh $BIN
-#./run_qemu.sh $BIN
-#./run_qemu.sh $BIN
+#echo "i wish i were a valid key bob" | ./run_qemu.sh $BIN
+./run_qemu.sh $BIN
 
 echo "*** build the Program database"
 time python db_commit_asm.py $BIN $SRC
-echo "*** filter the Change database"
-time python db_filter_log.py
+#echo "*** filter the Change database"
+#time python db_filter_log.py
 echo "*** build the Change database"
 time python db_commit_log.py
 echo "*** build the memory json"

qemu_mods/tci.c

@@ -426,7 +426,6 @@ static bool tci_compare64(uint64_t u0, uint64_t u1, TCGCond condition)
 #include <fcntl.h>
 #include <sys/mman.h>
 
-
 #define QIRA_DEBUG(...) {}
 //#define QIRA_DEBUG qemu_log
 
@@ -437,6 +436,8 @@ void track_load(target_ulong a, uint64_t data, int size);
 void track_store(target_ulong a, uint64_t data, int size);
 void track_read(target_ulong base, target_ulong offset, target_ulong data, int size);
 void track_write(target_ulong base, target_ulong offset, target_ulong data, int size);
+void add_pending_change(target_ulong addr, uint64_t data, uint32_t flags);
+void commit_pending_changes(void);
 
 // struct storing change data
 struct change {
@@ -452,13 +453,19 @@ struct change *GLOBAL_change_buffer;
 uint32_t GLOBAL_changelist_number = 0;
 
 uint32_t GLOBAL_qira_log_fd;
-uint32_t GLOBAL_change_count = 0;
+uint32_t *GLOBAL_change_count;
 uint32_t GLOBAL_change_size;
+uint32_t GLOBAL_is_filtered = 0;
+
+// should be 0ed on startup
+#define PENDING_CHANGES_MAX_ADDR 0x100
+struct change GLOBAL_pending_changes[PENDING_CHANGES_MAX_ADDR/4];
 
 #define IS_VALID      0x80000000
 #define IS_WRITE      0x40000000
 #define IS_MEM        0x20000000
 #define IS_START      0x10000000
+#define SIZE_MASK 0xFF
 
 void init_QIRA(CPUArchState *env) {
   QIRA_DEBUG("init QIRA called\n");
@@ -466,7 +473,7 @@ void init_QIRA(CPUArchState *env) {
   GLOBAL_qira_log_fd = open("/tmp/qira_log", O_RDWR | O_CREAT, 0644);
   GLOBAL_change_size = 1;
   GLOBAL_QIRA_did_init = 1;
-  GLOBAL_change_count = 0;
+  memset(GLOBAL_pending_changes, 0, (PENDING_CHANGES_MAX_ADDR/4) * sizeof(struct change));
 
   if(ftruncate(GLOBAL_qira_log_fd, GLOBAL_change_size * sizeof(struct change))) {
     perror("ftruncate");
@@ -475,10 +482,13 @@ void init_QIRA(CPUArchState *env) {
     mmap(NULL, GLOBAL_change_size * sizeof(struct change),
          PROT_READ | PROT_WRITE, MAP_SHARED, GLOBAL_qira_log_fd, 0);
   if (GLOBAL_change_buffer == NULL) QIRA_DEBUG("MMAP FAILED!\n");
+  GLOBAL_change_count = (uint32_t*)GLOBAL_change_buffer;
+  // first change is fake invalid shit
+  *GLOBAL_change_count = 1;
 }
 
 void add_change(target_ulong addr, uint64_t data, uint32_t flags) {
-  if (GLOBAL_change_count == GLOBAL_change_size) {
+  if (*GLOBAL_change_count == GLOBAL_change_size) {
     // double the buffer size
     QIRA_DEBUG("doubling buffer with size %d\n", GLOBAL_change_size);
     if(ftruncate(GLOBAL_qira_log_fd, GLOBAL_change_size * sizeof(struct change) * 2)) {
@@ -496,7 +506,25 @@ void add_change(target_ulong addr, uint64_t data, uint32_t flags) {
   GLOBAL_change_buffer->changelist_number = GLOBAL_changelist_number;
   GLOBAL_change_buffer->flags = IS_VALID | flags;
   ++GLOBAL_change_buffer;
-  ++GLOBAL_change_count;
+  // must inc this afterward
+  ++(*GLOBAL_change_count);
+}
+
+void add_pending_change(target_ulong addr, uint64_t data, uint32_t flags) {
+  if (addr < PENDING_CHANGES_MAX_ADDR) {
+    GLOBAL_pending_changes[addr/4].address = (uint64_t)addr;
+    GLOBAL_pending_changes[addr/4].data = data;
+    GLOBAL_pending_changes[addr/4].flags = IS_VALID | flags;
+  }
+}
+
+void commit_pending_changes(void) {
+  int i;
+  for (i = 0; i < PENDING_CHANGES_MAX_ADDR/4; i++) {
+    struct change *c = &GLOBAL_pending_changes[i];
+    if (c->flags & IS_VALID) add_change(c->address, c->data, c->flags);
+  }
+  memset(GLOBAL_pending_changes, 0, (PENDING_CHANGES_MAX_ADDR/4) * sizeof(struct change));
 }
 
 void track_load(target_ulong addr, uint64_t data, int size) {
@@ -512,13 +540,14 @@ void track_store(target_ulong addr, uint64_t data, int size) {
 void track_read(target_ulong base, target_ulong offset, target_ulong data, int size) {
   if ((int)offset < 0) return;
   QIRA_DEBUG("read: %x+%x:%d = %x\n", base, offset, size, data);
-  add_change(offset, data, size);
+  if (GLOBAL_is_filtered == 0) add_change(offset, data, size);
 }
 
 void track_write(target_ulong base, target_ulong offset, target_ulong data, int size) {
   if ((int)offset < 0) return;
   QIRA_DEBUG("write: %x+%x:%d = %x\n", base, offset, size, data);
-  add_change(offset, data, IS_WRITE | size);
+  if (GLOBAL_is_filtered == 0) add_change(offset, data, IS_WRITE | size);
+  else add_pending_change(offset, data, IS_WRITE | size);
 }
 
 // careful, this does it twice, MMIO?
@@ -585,15 +614,24 @@ void track_write(target_ulong base, target_ulong offset, target_ulong data, int
 /* Interpret pseudo code in tb. */
 uintptr_t tcg_qemu_tb_exec(CPUArchState *env, uint8_t *tb_ptr)
 {
-
 #ifdef QIRA_TRACKING
     if (unlikely(GLOBAL_QIRA_did_init == 0)) init_QIRA(env);
-    GLOBAL_changelist_number++;
     CPUState *cpu = ENV_GET_CPU(env);
     TranslationBlock *tb = cpu->current_tb;
 
-    qemu_log("set changelist %d at %x(%d)\n", GLOBAL_changelist_number, tb->pc, tb->size);
-    add_change(tb->pc, tb->size, IS_START);
+    // hacky check
+    if (tb->pc > 0x40000000) {
+      GLOBAL_is_filtered = 1;
+    } else {
+      if (GLOBAL_is_filtered == 1) {
+        commit_pending_changes();
+        GLOBAL_is_filtered = 0;
+      }
+      GLOBAL_changelist_number++;
+      add_change(tb->pc, tb->size, IS_START);
+    }
+
+    QIRA_DEBUG("set changelist %d at %x(%d)\n", GLOBAL_changelist_number, tb->pc, tb->size);
 #endif
 
     long tcg_temps[CPU_TEMP_BUF_NLONGS];

qiradb/qiradb.cc

@@ -0,0 +1,97 @@
+#include <stdio.h>
+#include <pthread.h>
+#include <mongoc.h>
+#include <bson.h>
+
+#define MONGO_DEBUG printf
+//#define MONGO_DEBUG(...) {}
+
+// use like an include maybe?
+// struct storing change data
+struct change {
+  uint64_t address;
+  uint64_t data;
+  uint32_t changelist_number;
+  uint32_t flags;
+};
+
+int main(int argc, char* argv[]) {
+  bool ret;
+
+  mongoc_init();
+  mongoc_client_t *client;
+  mongoc_collection_t *collection;
+  client = mongoc_client_new("mongodb://localhost:3001");
+  collection = mongoc_client_get_collection(client, "meteor", "change");
+  ret = mongoc_collection_drop(collection, NULL);
+  if (!ret) MONGO_DEBUG("drop failed\n");
+
+  uint32_t mongo_qira_log_fd = open("/tmp/qira_log", O_RDONLY);
+  uint32_t mongo_change_count = 0;
+
+  GLOBAL_change_buffer =
+    mmap(NULL, GLOBAL_change_size * sizeof(struct change),
+         PROT_READ | PROT_WRITE, MAP_SHARED, GLOBAL_qira_log_fd, 0);
+
+  // begin thread run loop
+  while (1) {
+    usleep(10*1000);  // commit every 10ms
+
+    mongoc_bulk_operation_t *bulk;
+    bson_t reply;
+    bson_error_t error;
+    bson_t *doc;
+
+    // set up bulk operation
+    bulk = mongoc_collection_create_bulk_operation(collection, true, NULL);
+
+    // add new changes
+    int lcount = 0;
+    while (mongo_change_count < GLOBAL_change_count) {
+      struct change tmp;
+      int a = read(mongo_qira_log_fd, &tmp, sizeof(struct change));
+      if (a != sizeof(struct change)) {
+        qemu_log("READ ERROR");
+        break;
+      }
+
+      char typ[2]; typ[1] = '\0';
+      uint32_t flags = tmp.flags;
+      if (flags & IS_START) typ[0] = 'I';
+      else if ((flags & IS_WRITE) && (flags & IS_MEM)) typ[0] = 'S';
+      else if (!(flags & IS_WRITE) && (flags & IS_MEM)) typ[0] = 'L';
+      else if ((flags & IS_WRITE) && !(flags & IS_MEM)) typ[0] = 'W';
+      else if (!(flags & IS_WRITE) && !(flags & IS_MEM)) typ[0] = 'R';
+
+      doc = bson_new();
+      BSON_APPEND_INT32(doc, "address", tmp.address);
+      BSON_APPEND_UTF8(doc, "type", typ);
+      BSON_APPEND_INT32(doc, "size", tmp.flags & SIZE_MASK);
+      BSON_APPEND_INT32(doc, "clnum", tmp.changelist_number);
+      BSON_APPEND_INT32(doc, "data", tmp.data);
+      mongoc_bulk_operation_insert(bulk, doc);
+      bson_destroy(doc);
+
+      mongo_change_count++;
+      lcount++;
+    }
+
+    if (lcount > 0) {
+      MONGO_DEBUG("commit %d\n", mongo_change_count);
+
+      // do bulk operation
+      ret = mongoc_bulk_operation_execute(bulk, &reply, &error);
+      if (!ret) MONGO_DEBUG("mongo error: %s\n", error.message);
+
+      // did bulk operation
+      bson_destroy(&reply);
+    }
+    mongoc_bulk_operation_destroy(bulk);
+  }
+
+  // thread exit
+  mongoc_collection_destroy(collection);
+  mongoc_client_destroy(client);
+  mongoc_cleanup();
+  return NULL;
+}

scripts/db_commit_log.py

@@ -4,8 +4,8 @@ from pymongo import MongoClient
 db = MongoClient('localhost', 3001).meteor
 
 print "reading log"
-#dat = read_log("/tmp/qira_log")
-dat = read_log("/tmp/qira_log_filtered")
+dat = read_log("/tmp/qira_log")
+#dat = read_log("/tmp/qira_log_filtered")
 
 print "building database data"
 
@@ -32,11 +32,12 @@ coll = db.change
 print "doing db insert of",len(ds),"changes"
 coll.drop()
 coll.insert(ds)
-print "db insert done, building indexes"
-coll.ensure_index("data")
-coll.ensure_index([("data", 1), ("address", 1)])
-coll.ensure_index("address")
-coll.ensure_index("clnum")
-coll.ensure_index([("address", 1), ("type", 1)])
-print "indexes built"
+#print "db insert done, building indexes"
+#coll.ensure_index("data")
+#coll.ensure_index([("data", 1), ("address", 1)])
+#coll.ensure_index("address")
+#coll.ensure_index("clnum")
+#coll.ensure_index([("address", 1), ("type", 1)])
+#print "indexes built"
+print "db insert done"
 

scripts/debug_qemu.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+set -e
+
+pushd .
+cd ~/build/qemu
+make -j32
+popd
+
+#rm -rf /tmp/qira*
+~/build/qemu/i386-linux-user/qemu-i386 -singlestep $@
+ls -l /tmp/qira*
+

scripts/run_qemu.sh

@@ -6,7 +6,7 @@ cd ~/build/qemu
 make -j32
 popd
 
-rm -rf /tmp/qira*
+#rm -rf /tmp/qira*
 ~/build/qemu/i386-linux-user/qemu-i386 -singlestep -d in_asm $@ 2> /tmp/qira_disasm
 ls -l /tmp/qira*
 

web/client/idump.js

@@ -38,6 +38,9 @@ Template.idump.hexaddress = function() {
 Template.idump.events({
   'click .change': function() {
     Session.set('clnum', this.clnum);
+  },
+  'click .datainstruction': function() {
+    Session.set('iaddr', this.address);
   }
 });
 

web/client/qira.js

@@ -34,10 +34,14 @@ function check_hash() {
 window.onload = check_hash;
 window.onhashchange = check_hash;
 
-Meteor.subscribe('max_clnum', {onReady: function() {
-  post = Change.findOne({}, {sort: {clnum: -1}});
+Meteor.subscribe('max_clnum');
 
-  Session.setDefault("clnum", post.clnum);
-  Session.set("max_clnum", post.clnum);
-}});
+Deps.autorun(function() {
+  var post = Change.findOne({type: "I"}, {sort: {clnum: -1}, limit: 1});
+
+  if (post !== undefined) {
+    Session.setDefault("clnum", post.clnum);
+    Session.set("max_clnum", post.clnum);
+  }
+});
 

web/client/vtimeline.js

@@ -1,7 +1,4 @@
 Meteor.startup(function() {
-  Deps.autorun(function() {
-    zoom_out_max(true);
-  });
 
   /*$("#vtimeline").click(function(e) {
     if (e.target !== $("#vtimeline")[0]) return;
@@ -57,7 +54,7 @@ function register_drag_zoom() {
   $("#vtimeline").mouseup(function(e) {
     if (e.button != 0) return;
     var up = get_clnum(e);
-    if (up == undefined) return;
+    if (up === undefined) return;
     if (down != -1) {
       p("drag "+down+"-"+up);
       if (down == up) {
@@ -76,7 +73,7 @@ var flags = {};
 
 function get_cscale() {
   var cview = Session.get("cview");
-  if (cview == undefined) return;
+  if (cview === undefined) return;
   var range = cview[1] - cview[0];
   var box = $("#vtimelinebox");
   if (box.length == 0) return undefined;
@@ -135,11 +132,19 @@ function add_flag(type, clnum) {
 function remove_flags(type) {
   for (clnum in flags) {
     var index = flags[clnum].indexOf(type);
-    if (index != -1) flags[clnum].splice(index, 1);
+    while (index != -1) {
+      flags[clnum].splice(index, 1)
+      index = flags[clnum].indexOf(type);
+    }
     if (flags[clnum].length == 0) delete flags[clnum];
   }
 }
 
+Deps.autorun(function() {
+  // false here forces update on max_clnum update
+  zoom_out_max(false);
+});
+
 Deps.autorun(function() {
   var clnum = Session.get("clnum");
   remove_flags("change");

web/qira.html

@@ -1,14 +1,8 @@
 <head>
   <title>qira</title>
-  <!--<link rel="stylesheet" href="http://code.jquery.com/mobile/1.4.2/jquery.mobile-1.4.2.min.css">-->
 </head>
 
 <body>
-<!--<div id="timeline">
-<div id="cl_selector_box">
-<input type="range" id="cl_selector" value="1" min="1" max="1">
-</div>
-</div>-->
 <div id="vtimelinebox">
 <div id="vtimeline">
 </div>
@@ -42,9 +36,9 @@
 {{#each instructions}}
   <div class="instruction">
   <div class="change {{ischange}}">{{clnum}}</div>
+  <span class="datainstruction {{isiaddr}}">{{hexaddress}}</span>
   {{#with program_instruction}}
       <span class="name">{{name}}</span>
-      <span class="datainstruction {{isiaddr}}">{{hexaddress}}</span>
       {{instruction}}
       <span class="comment">{{comment}}</span>
   {{/with}}

web/server/qira.js

@@ -4,7 +4,7 @@ Meteor.startup(function () {
 
 Meteor.publish('max_clnum', function() {
   // extract 'clnum' from this to get the max clnum
-  return Change.find({}, {sort: {clnum: -1}, limit: 1});
+  return Change.find({type: "I"}, {sort: {clnum: -1}, limit: 1});
 });
 
 Meteor.publish('pmaps', function() {
@@ -35,7 +35,7 @@ Meteor.publish('dat_daddr', function(daddr) {
   // fetch the dynamic info about the instruction range
   //return Change.find({address : {$gt: daddr-0x100, $lt: daddr+0x300}});
   if (daddr >= 0x1000) {
-    return Change.find({address:daddr, $or: [{type: "L"}, {type: "S"}]}, {sort: {clnum: 1}, limit:50});
+    return Change.find({address:daddr, $or: [{type: "L"}, {type: "S"}]}, {sort: {clnum: 1}, limit:30});
   } else {
     return false;
   }

web/server/regmem.js

@@ -65,7 +65,7 @@ Meteor.startup(function () {
       clearTimeout(tmout);
       tmout = undefined;
     }
-    tmout = setTimeout(read_memdb, 200);
+    tmout = setTimeout(read_memdb, 500);
   });
 });