moved cda into qira, started vimplugin, new configuration file

cda/cachegen.py

@@ -10,10 +10,14 @@ ci.Config.set_library_file(basedir+"/clang/build/Release+Asserts/lib/libclang.so
 import pickle
 from clang.cindex import CursorKind
 
+import json
+from hashlib import sha1
+
 # debug
 DEBUG = 0
 
 # cache generated
+file_cache = {}
 object_cache = {}
 xref_cache = {}
 
@@ -115,3 +119,10 @@ def parse_file(filename, args=[]):
 
   return (care, rdat)
 
+def parse_files(files, args=[]):
+  for fn in files:
+    print "CDA: caching",fn
+    file_cache[fn] = parse_file(fn)
+  dat = (object_cache, file_cache, xref_cache)
+  return dat
+

cda/cacheserver.py

@@ -2,25 +2,16 @@
 import os
 import sys
 import cgi
-from flask import Flask,redirect, request
+from flask import Flask,redirect,request,Blueprint
 from html import XHTML
-import pickle
-app = Flask(__name__, static_folder='static', static_url_path='/static')
+
+app = Blueprint('cda',__name__)
 
 # escape on the real
 def escape(s, crap=False):
   return s.replace("<", "&lt;").replace(">", "&gt;").replace(" ", "&nbsp;").replace("\n", "<br/>").replace("\t", "&nbsp;"*4).replace("\x00", " ")
 cgi.escape = escape
 
-@app.route("/")
-def index():
-  h = XHTML().html
-  h.head.link(rel="stylesheet", href="/static/cda.css")
-  h.head.style('body{margin:0;padding:0;}')
-  h.body.iframe(src='/list', id="topframe")
-  h.body.iframe(src='/x/=', id="bottomframe")
-  return str(h)
-
 @app.route("/list")
 def home():
   # add files
@@ -34,7 +25,7 @@ def home():
 
   # generate html
   h = XHTML().html
-  h.head.link(rel="stylesheet", href="/static/cda.css")
+  h.head.link(rel="stylesheet", href="/cdastatic/cda.css")
   body = h.body
   objs = list(set(objs))
   objs.sort()
@@ -42,16 +33,17 @@ def home():
     body.div.a(obj[1], href=obj[0], klass=obj[2])
   return str(h)
 
-@app.route("/x/<xref>")
-def display_xref(xref):
-  xref = xref.decode("base64")
+@app.route("/x/<b64xref>")
+def display_xref(b64xref):
+  xref = b64xref.decode("base64")
   h = XHTML().html
-  h.head.link(rel="stylesheet", href="/static/cda.css")
+  h.head.link(rel="stylesheet", href="/cdastatic/cda.css")
   body = h.body(klass="xref")
   body.div.div(xref, klass="xrefstitle")
   if xref in xref_cache:
     for obj in xref_cache[xref]:
-      body.div.a(obj, onclick="parent.frames[0].location = '/f?"+obj+"';", klass="filelink")
+      linkobj = obj+","+b64xref
+      body.div.a(obj, onclick="parent.location = '/f?"+linkobj+"';", klass="filelink")
   return str(h)
 
 @app.route("/f")
@@ -61,26 +53,28 @@ def display_file():
     return "file "+str(path)+" not found"
   # generate the HTML
   h = XHTML().html
-  h.head.link(rel="stylesheet", href="/static/cda.css")
-  h.head.script(src="/static/socket.io.min.js")
-  h.head.script(src="/static/jquery-2.1.0.js")
-  h.head.script(src="/static/jquery.scrollTo-1.4.3.1.js")
-  h.head.script(src="/static/cda.js?"+os.urandom(16).encode("hex"))
+  h.head.link(rel="stylesheet", href="/cdastatic/cda.css")
+  h.head.script(src="/cdastatic/socket.io.min.js")
+  h.head.script(src="/cdastatic/jquery-2.1.0.js")
+  h.head.script(src="/cdastatic/jquery.scrollTo-1.4.3.1.js")
+  h.head.script(src="/cdastatic/cda.js?"+os.urandom(16).encode("hex"))
   body = h.body
-  body.div(path, id="filename")
+  body.div(path, id='filename')
+  prog = body.div(id="program")
+  body.iframe(id='bottomframe')
 
   # get parsed file
   (care, rdat) = file_cache[path]
 
   # add line numbers
   lc = len(rdat.split("\n"))
-  ln = body.div(id="ln")
+  ln = prog.div(id="ln")
   for linenum in range(lc):
-    ln.span("%5d \n" % (linenum+1), id="l"+str(linenum+1), onclick='location.hash='+str(linenum+1))
+    ln.span("%5d \n" % (linenum+1), id="l"+str(linenum+1), onclick='go_to_line('+str(linenum+1)+')')
 
   # add the code
   #print object_cache
-  p = body.div(id="code")
+  p = prog.div(id="code")
   last = 0
   for (start, end, klass, usr) in care:
     if last > start:
@@ -104,12 +98,8 @@ def display_file():
 
   return str(h)
 
-def start(cache):
+def set_cache(cache):
   global object_cache, file_cache, xref_cache
-
   (object_cache, file_cache, xref_cache) = cache
   print "read",len(file_cache),"files",len(object_cache),"objects",len(xref_cache),"xrefs"
 
-  #app.run(host='127.0.0.1', debug=True, port=5000)
-  app.run(host='127.0.0.1', port=5000)
-

cda/cda

@@ -1,43 +0,0 @@
-#!/usr/bin/env python2
-import os
-import sys
-import json
-from hashlib import sha1
-
-import cachegen
-import cacheserver
-
-file_cache = {}
-
-if __name__ == "__main__":
-  try:
-    os.mkdir("/tmp/cdacaches")
-  except:
-    pass
-    
-  files = []
-  for fn in sys.argv[1:]:
-    fn = os.path.realpath(sys.argv[1])
-    files.append(fn)
-
-  cachename = "/tmp/cdacaches/"+sha1(''.join(files)).hexdigest()
-
-  if os.path.isfile(cachename):
-    dat = json.load(open(cachename))
-    print "read cache",cachename
-  else:
-    for fn in files:
-      print "caching",fn
-      file_cache[fn] = cachegen.parse_file(fn)
-    dat = (cachegen.object_cache, file_cache, cachegen.xref_cache)
-    f = open(cachename, "wb")
-    json.dump(dat, f)
-    f.close()
-    print "wrote cache",cachename
-
-  cacheserver.start(dat)
-
-
-
-
-

cda/static/cda.css

@@ -111,20 +111,26 @@ iframe {
   border: 0 none;
 }
 
-#topframe {
-  width:100%;
-  height: 70%;
+body {
+  margin: 0;
+  padding: 0;
 }
 
 #bottomframe {
-  width:100%;
-  height:30%;
+  width: 100%;
+  height: 200px;
   border-top: 1px solid black;
+  position: absolute;
+  bottom: 0;
+  left: 0;
+  background-color: #EEEEEE;
+}
+
+#program {
+  padding: 5px;
 }
 
 #filename {
-  padding-bottom: 10px;
-  font-size: 14px;
+  display: none;
 }
 
-

cda/static/cda.js

@@ -8,38 +8,49 @@ function p(s) {
 var highlighted = $();
 
 $(window).on('hashchange', function() {
-  if (window.location.hash == "") return;
-  var ln = window.location.hash.substr(1);
+  if (location.hash == "") location.replace("#0");
+  var ln = location.hash.substr(1).split(",")[0];
+  var b64xref = location.hash.split(",")[1];
   highlighted.removeClass("line_highlighted")
   highlighted = $("#l" + ln)
-  highlighted.addClass("line_highlighted");
-  $(window).scrollTo(highlighted, {offset: -150})
-  stream.emit('navigateline', $('#filename')[0].innerHTML, parseInt(ln))
+  if (highlighted.length > 0) {
+    highlighted.addClass("line_highlighted");
+    $(window).scrollTo(highlighted, {offset: -150})
+    stream.emit('navigateline', $('#filename')[0].innerHTML, parseInt(ln))
+  }
+  if (b64xref !== undefined) {
+    selected.removeClass('highlighted');
+    selected = $(document.getElementsByName(atob(b64xref)));
+    selected.addClass('highlighted');
+    if (frames[0].location.pathname != '/x/'+b64xref) {
+      frames[0].location.replace('/x/'+b64xref);
+    }
+  }
 });
 
 var selected = $();
 
 function link_click_handler(e) {
-  //p(e.target.getAttribute('name'));
-  selected.removeClass('highlighted');
-  selected = $(document.getElementsByName(e.target.getAttribute('name')));
-  selected.addClass('highlighted');
+  var usr = e.target.getAttribute('name');
+  location.replace(location.hash.split(",")[0]+","+btoa(usr));
 }
 
 function link_dblclick_handler(e) {
   var targets = e.target.getAttribute('targets').split(" ");
   p(targets);
-  parent.frames[0].location = "/f?"+targets[0];
-}
-
-function xref_handler(e) {
   var usr = e.target.getAttribute('name');
-  p("xref "+usr);
-  parent.frames[1].location = '/x/'+btoa(usr)
-  return false;
+  location = "/f?"+targets[0]+","+btoa(usr);
+}
+
+function go_to_line(line) {
+  var b64xref = location.hash.split(",")[1];
+  var newl = "#"+line;
+  if (b64xref !== undefined) {
+    newl += ","+b64xref;
+  }
+  location.replace(newl);
 }
 
-// no selection
 window.onmousedown = function() { return false; };
 
 // when the page loads we need to check the hash
@@ -48,8 +59,6 @@ window.onload = function() {
 
   $('.link').bind('click', link_click_handler); 
   $('.link').bind('dblclick', link_dblclick_handler); 
-  $('.link').bind('contextmenu', xref_handler); 
 };
 
 
-

install.sh

@@ -15,7 +15,7 @@ elif [ $(which pacman) ]; then
 fi
 
 echo "installing pip packages"
-sudo $PIP install flask-socketio pillow pyelftools ./qiradb
+sudo $PIP install html flask-socketio pillow pyelftools ./qiradb
 
 echo "making symlink"
 sudo ln -sf $(pwd)/qira /usr/local/bin/qira

middleware/qira.py

@@ -5,24 +5,33 @@ import socket
 import threading
 import time
 
+import qira_config
 import qira_socat
 import qira_program
 import qira_webserver
 
 if __name__ == '__main__':
+  # define arguments
   parser = argparse.ArgumentParser(description = 'Analyze binary. Like "qira /bin/ls /"')
   parser.add_argument('-s', "--server", help="bind on port 4000. like socat", action="store_true")
   parser.add_argument('-t', "--tracelibraries", help="trace into all libraries", action="store_true")
   parser.add_argument('binary', help="path to the binary")
   parser.add_argument('args', nargs='*', help="arguments to the binary")
+  parser.add_argument("--dwarf", help="parse program dwarf data", action="store_true")
+  parser.add_argument("--cda", help="use CDA to view source", action="store_true")
+
+  # parse arguments
   args = parser.parse_args()
+  if args.tracelibraries:
+    qira_config.TRACE_LIBRARIES = True
+  if args.dwarf:
+    qira_config.WITH_DWARF = True
+  if args.cda:
+    qira_config.WITH_CDA = True
 
   # creates the file symlink, program is constant through server run
   program = qira_program.Program(args.binary, args.args)
 
-  if args.tracelibraries:
-    program.defaultargs.append("-tracelibraries")
-
   is_qira_running = 1
   try:
     socket.create_connection(('127.0.0.1', qira_webserver.QIRA_WEB_PORT))

middleware/qira_analysis.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+import qira_config
 import qira_program
 import time
 

middleware/qira_config.py

@@ -0,0 +1,4 @@
+WITH_CDA = False
+WITH_DWARF = False
+TRACE_LIBRARIES = False
+

middleware/qira_program.py

@@ -1,7 +1,12 @@
+import qira_config
 import os
 import sys
-import struct
+from hashlib import sha1
+basedir = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(basedir+"/../cda")
 
+import json
+import struct
 import qiradb
 
 PPCREGS = ([], 4, True)
@@ -12,6 +17,25 @@ ARMREGS = (['R0','R1','R2','R3','R4','R5','R6','R7','R8','R9','R10','R11','R12',
 X86REGS = (['EAX', 'ECX', 'EDX', 'EBX', 'ESP', 'EBP', 'ESI', 'EDI', 'EIP'], 4, False)
 X64REGS = (['RAX', 'RCX', 'RDX', 'RBX', 'RSP', 'RBP', 'RSI', 'RDI', 'RIP'], 8, False)
 
+def cachewrap(cachedir, cachename, cachegen):
+  try:
+    os.mkdir(cachedir)
+  except:
+    pass
+  cachename = cachedir + "/" + cachename
+  if os.path.isfile(cachename):
+    dat = json.load(open(cachename))
+    print "read cache",cachename
+  else:
+    print "cache",cachename,"not found, generating"
+    dat = cachegen()
+    if dat == None:
+      return None
+    f = open(cachename, "wb")
+    json.dump(dat, f)
+    f.close()
+    print "wrote cache",cachename
+  return dat
 
 def which(prog):
   import subprocess
@@ -37,6 +61,8 @@ class Program:
     # call which to match the behavior of strace and gdb
     self.program = which(prog)
     self.args = args
+    self.proghash = sha1(open(prog).read()).hexdigest()
+    print "*** program is",self.program,"with hash",self.proghash
 
     # bring this back
     if self.program != "/tmp/qira_binary":
@@ -48,6 +74,8 @@ class Program:
 
     # defaultargs for qira binary
     self.defaultargs = ["-strace", "-D", "/dev/null", "-d", "in_asm", "-singlestep"]
+    if qira_config.TRACE_LIBRARIES:
+      program.defaultargs.append("-tracelibraries")
 
     # pmaps is global, but updated by the traces
     self.instructions = {}
@@ -158,39 +186,68 @@ class Program:
     os.execvp(self.qirabinary, eargs)
 
   def getdwarf(self):
-    self.dwarves = {}
-    self.rdwarves = {}
-    from elftools.elf.elffile import ELFFile
-    elf = ELFFile(open(self.program))
-    if not elf.has_dwarf_info():
+    (self.dwarves, self.rdwarves) = ({}, {})
+
+    if not qira_config.WITH_DWARF:
       return
 
     # DWARF IS STUPIDLY COMPLICATED
-    di = elf.get_dwarf_info()
-    for cu in di.iter_CUs():
-      basedir = ''
-      # get the base directory
-      for die in cu.iter_DIEs():
-        if die.tag == "DW_TAG_compile_unit":
-          basedir = die.attributes['DW_AT_comp_dir'].value
-      # get the line program?
-      lp = di.line_program_for_CU(cu)
-      dir_index = lp['file_entry'][0].dir_index
-      if dir_index > 0:
-        basedir = lp['include_directory'][dir_index-1]
-      # now we have the filename
-      filename = basedir + "/" + lp['file_entry'][0].name
+    def parse_dwarf():
+      dwarves = {}
+      rdwarves = {}
+
+      from elftools.elf.elffile import ELFFile
+      elf = ELFFile(open(self.program))
+      if not elf.has_dwarf_info():
+        return (dwarves, rdwarves)
+      files = []
+      filename = None
+      di = elf.get_dwarf_info()
+      for cu in di.iter_CUs():
+        try:
+          basedir = None
+          # get the base directory
+          for die in cu.iter_DIEs():
+            if die.tag == "DW_TAG_compile_unit":
+              basedir = die.attributes['DW_AT_comp_dir'].value + "/"
+          if basedir == None:
+            continue
+          # get the line program?
+          lp = di.line_program_for_CU(cu)
+          dir_index = lp['file_entry'][0].dir_index
+          if dir_index > 0:
+            basedir += lp['include_directory'][dir_index-1]+"/"
+          # now we have the filename
+          filename = basedir + lp['file_entry'][0].name
+          files.append(filename)
+          lines = open(filename).read().split("\n")
+          print "DWARF: parsing",filename
+          for entry in lp.get_entries():
+            s = entry.state
+            if s != None:
+              #print filename, s.line, len(lines)
+              dwarves[s.address] = (s.line, lines[s.line-1])
+              rdwarves[filename+"#"+str(s.line)] = s.address
+        except Exception as e:
+          print "DWARF: error on",filename,"got",e
+      return (dwarves, rdwarves)
+
+    (self.dwarves, self.rdwarves) = cachewrap("/tmp/dwarfcaches", self.proghash, parse_dwarf)
+
+    # cda
+    if not qira_config.WITH_CDA:
+      return
+
+    def parse_cda():
       try:
-        lines = open(filename).read().split("\n")
-      except:
-        print "*** couldn't find %s for DWARF", filename
-        continue
-      for entry in lp.get_entries():
-        #print entry
-        s = entry.state
-        if s != None:
-          self.dwarves[s.address] = (s.line, lines[s.line-1])
-          self.rdwarves[(filename, s.line)] = s.address
+        import cachegen
+        return cachegen.parse_files(files)
+      except Exception as e:
+        print "CDA: cachegen failed with",e
+        return None
+
+    self.cda = cachewrap("/tmp/cdacaches", self.proghash, parse_cda)
+
 
 class Trace:
   def __init__(self, fn, forknum, r1, r2, r3):

middleware/qira_webserver.py

@@ -1,4 +1,9 @@
+import qira_config
 import os
+import sys
+basedir = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(basedir+"/../cda")
+
 import qira_socat
 import time
 
@@ -23,8 +28,17 @@ gevent.monkey.patch_all()
 # done with that
 
 app = Flask(__name__)
+#app.config['DEBUG'] = True
 socketio = SocketIO(app)
 
+# add cda server paths here
+if qira_config.WITH_CDA:
+  try:
+    import cacheserver
+    app.register_blueprint(cacheserver.app)
+  except Exception as e:
+    print "CDA: load cacheserver failed with",e
+
 def ghex(a):
   if a == None:
     return None
@@ -64,7 +78,7 @@ def mwpoller():
 @socketio.on('navigateline', namespace='/qira')
 def navigateline(fn, ln):
   try:
-    iaddr = program.rdwarves[(fn,ln)]
+    iaddr = program.rdwarves[fn+"#"+str(ln)]
   except:
     return
   print 'navigateline',fn,ln,iaddr
@@ -279,7 +293,6 @@ def get_strace(forknum):
 
 # ***** generic webserver stuff *****
   
-
 @app.route('/', defaults={'path': 'index.html'})
 @app.route('/<path:path>')
 def serve(path):
@@ -290,11 +303,11 @@ def serve(path):
 
   ext = path.split(".")[-1]
 
-  if ext == 'css':
-    path = "qira.css"
-
-  dat = open(webstatic+path).read()
-  if ext == 'js' and not path.startswith('client/compatibility/') and not path.startswith('packages/'):
+  try:
+    dat = open(webstatic+path).read()
+  except:
+    return ""
+  if ext == 'js' and not path.startswith('client/compatibility/') and path.startswith('client/'):
     dat = "(function(){"+dat+"})();"
 
   if ext == 'js':
@@ -309,6 +322,10 @@ def run_server(largs, lprogram):
   global program
   args = largs
   program = lprogram
+  try:
+    cacheserver.set_cache(program.cda)
+  except:
+    pass
   print "starting socketio server..."
   threading.Thread(target=mwpoller).start()
   socketio.run(app, port=QIRA_WEB_PORT)

tests/vimplugin/main.c

@@ -1,3 +1,6 @@
+void swag();
+void swag2();
+
 int main() {
   int i;
   int j = 1;
@@ -8,6 +11,8 @@ int main() {
     k += 1;
     l += i;
   }
+  swag();
+  swag2();
   printf("%d %d %d\n", j, k, l);
 }
 

tests/vimplugin/swag.c

@@ -0,0 +1,8 @@
+void swag() {
+  printf("SWAG!\n");
+}
+
+void swag2() {
+  printf("SWAG2!\n");
+}
+

vim/qira.vim

@@ -0,0 +1,7 @@
+if !has('python')
+  echo "vim must be compiled with +python"
+  finish
+endif
+
+function! 
+

webstatic/cdastatic

@@ -0,0 +1 @@
+../cda/static/