kuroko/tools/gendoc.krk

#!/usr/bin/env kuroko
'''
@brief Tool for dynamically generating documentation files through introspection.

Imports modules and scans through members to generate Markdown files to feed into
Doxygen. Uses dynamic introspection to obtain member lists and look up function
arguments, docstrings, etc.

Markdown output is aided by a set of macros defined in the Kuroko API Documentation
Doxyfile. The output should be suitable for use with a normal Doxygen build, but some
additional customization is available.
'''
import fileio
import kuroko
import syntax.highlighter

let realPrint = print

let blacklistedMethods = [
    '__func__',
    '__repr__',
    '__str__',
]

let specialMethods = {
    '__contains__': lambda cls, args: f'{args or "<i>needle</i>"} <b>in</b> {cls}',
    '__init__': lambda cls, args: f'let <i>x</i> = <b>{cls}</b>({args})',
    '__getitem__': lambda cls, args: f'{cls}[{args or "<i>key</i>"}]',
    '__delitem__': lambda cls, args: f'<b>del</b> {cls}[{args or "<i>key</i>"}]',
    '__setitem__': lambda cls, args: f'{cls}[{args or "<i>key</i>"}] = <i>value</i>',
    '__len__': lambda cls, args: f'<b>len</b>({cls})',
    '__call__': lambda cls, args: f'{cls}({args})',
    '__iter__': lambda cls, args: f'<b>for</b> <i>x</i> <b>in</b> {cls}:',
}

let tripletops = (
    ('add',     '+'),
    ('sub',     '-'),
    ('mul',     '*'),
    ('or',      '|'),
    ('xor',     '^'),
    ('and',     '&amp;'),
    ('lshift',  '&lt;&lt;'),
    ('rshift',  '&gt;&gt;'),
    ('mod',     '%'),
    ('truediv', '/'),
    ('floordiv','//'),
    ('pow',     '**'),
)

for operation in tripletops:
    let name, op = operation # potentially surprising closure behavior
    specialMethods[f'__{name}__'] = lambda cls, args: f'{cls} {op} {args or "<i>other</i>"}'
    specialMethods[f'__r{name}__'] = lambda cls, args: f'{args or "<i>other</i>"} {op} {cls}'
    specialMethods[f'__i{name}__'] = lambda cls, args: f'{cls} {op}= {args or "<i>other</i>"}'

let basicops = (
    ('eq', '=='),
    ('lt', '&lt;'),
    ('gt', '&gt;'),
    ('ge', '&gt;='),
    ('le', '&lt;='),
)

for operation in basicops:
    let name, op = operation
    specialMethods[f'__{name}__'] = lambda cls, args: f'{cls} {op} {args or "<i>other</i>"}'

let prefixops = (
    ('invert', '~'),
    ('neg', '-'),
)

for operation in prefixops:
    let name, op = operation
    specialMethods[f'__{name}__'] = lambda cls, args: f'{op}{cls}'

class Pair():
    '''Makes a silly sortable pair that can be expanded like a two-tuple.'''
    def __init__(left,right):
        self.left = left
        self.right = right
    def __eq__(other):
        if not isinstance(other,Pair): return False
        return self.left == other.left
    def __lt__(other):
        if self.left == '__init__' and other.left != '__init__': return True
        if other.left == '__init__': return False
        return self.left < other.left
    def __gt__(other):
        if self.left == '__init__': return False
        if other.left == '__init__': return True
        return self.left > other.left
    def __repr__():
        return f'Pair({self.left!r},{self.right!r})'
    def __iter__():
        yield self.left
        yield self.right

let modules = [
    # Integrated stuff
    'builtins',
    'kuroko',
    'threading',

    # C modules
    'os',
    'math',
    'gc',
    'dis',
    'fileio',
    'time',
    'socket',
    'stat',
    'timeit',
    '_pheap',
    'pheap',
    'random',
    'wcwidth',

    # Stuff from modules/
    'json',
    'collections',
    'string',
    'callgrind',

    # Other stuff
    'tools.gendoc',

    # Codecs module
    'codecs',
    'codecs.bespokecodecs',
    'codecs.binascii',
    'codecs.dbdata',
    'codecs.dbextra',
    'codecs.dbextra_data_7bit',
    'codecs.dbextra_data_8bit',
    'codecs.infrastructure',
    'codecs.isweblabel',
    'codecs.pifonts',
    'codecs.sbencs',
    'codecs.sbextra',
]

let docString = {}

def fixup(mod):
    '''Escapes some characters in module names to make better page IDs.'''
    if mod.startswith('__'): return '\\' + mod
    return mod

def truncateString(s):
    '''If @p s is longer than 100 characters, truncate it with an ellipsis.'''
    s = s.strip()
    if '\n' in s:
        s = s.split('\n')[0]
    let short = s[:100]
    if len(short) < len(s):
        return short + '...'
    else:
        return short

def fixupDoc(doc):
    '''Cleans up docstring contents for display in the module list.'''
    for line in doc.split('\n'):
        if line.strip().startswith('@brief '):
            doc = line.strip().replace('@brief ', '', 1).strip()
            break
    return doc.replace(',','\\,').replace('<','&lt;').replace('>','&gt;')

def isExceptionType(cls):
    '''Determines if @p cls is an @c Exception type by examining its superclass chain.'''
    return issubclass(cls,BaseException)

def typeName(t: type) -> str:
    '''Get a clean type name from @p t '''
    if t is type: return 'type'
    if isinstance(t,type):
        return t.__qualname__ if hasattr(t,'__qualname__') else t.__name__
    return str(t)

def getArgs(func):
    '''Extract the arguments of either a managed (@c func.__args__) or native (from an `@arguments` docstring) function.'''
    if func.__file__ == '<builtin>':
        if '__doc__' in dir(func) and func.__doc__ and '@arguments ' in func.__doc__:
            let before, after = func.__doc__.split('@arguments ',1)
            let line, rest = after.split('\n',1) if '\n' in after else (after,None)
            return line.strip().replace(',','\\,')
        return ''
    let argNames = [x for x in func.__args__ if x != 'self']
    if hasattr(func,'__annotations__') and func.__annotations__:
        for i in range(len(argNames)):
            if argNames[i] in func.__annotations__:
                argNames[i] += '<span class="type-hint">' + typeName(func.__annotations__[argNames[i]]) + '</span>'
    return '\\,'.join(argNames)

def functionDoc(func):
    '''Extracts the docstring from a function and removes markup that Doxygen will choke on.'''
    let doc = func.__doc__ if ('__doc__' in dir(func) and func.__doc__) else ''
    if '@arguments ' in doc:
        doc = '\n'.join([x for x in doc.split('\n') if '@arguments' not in x])
    return "<p>" + doc + "</p>"

def processModules(modules):

    let globalClassList = []
    let globalFunctionList = []

    for modulepath in modules:
        # Import the module.
        let module = kuroko.importmodule(modulepath)

        let output = fileio.open(f'docs/mod.{modulepath}.md','w')

        realPrint(f"Processing {modulepath}")

        def print(*args):
            output.write(' '.join(args))
            output.write('\n')

        print('## ' + fixup(modulepath) + ' {#mod_' + modulepath.replace('.','_') + '}')
        let rsplit = lambda s,d,l: reversed("".join(reversed(i)) for i in "".join(reversed(s)).split(d, l))
        if "." in modulepath:
            let parent = rsplit(modulepath, ".", 1)[0]
            if parent in modules:
                let parentpath = fixup(parent).replace('.','_')
                print(f"\n<a href='mod_{parentpath}.html'>← {parent}</a>\n")

        if '__doc__' in dir(module) and module.__doc__:
            print(module.__doc__.strip())
            docString[modulepath] = truncateString(module.__doc__)
        else:
            docString[modulepath] = 'TODO'

        def outputFunction(name, func, prefix=None):
            # Build the function signature
            let _args = getArgs(func)
            let args = ('<i>' + _args + '</i>') if _args else ''
            let body = functionDoc(func)
            let maybePrefix = prefix + '.' if prefix else ''
            let formatted = maybePrefix + '<b>' + name + '</b>' + f'({args})'
            let maybeDetail = ''
            if prefix and name in specialMethods:
                maybeDetail = ',' + formatted
                formatted = specialMethods[name](prefix,args)
            if hasattr(func,'__annotations__') and func.__annotations__ and 'return' in func.__annotations__:
                formatted += '<span class="type-hint-return">' + typeName(func.__annotations__['return']) + '</span>'
            print('\\methodstart{' + (maybePrefix + name).replace('.','_') + ',' + formatted + ',' + ('h4,methodDef' if prefix else 'h3,functionDef') + maybeDetail + '}')
            if body: print(body)
            print('\n\\methodend')
            print('')

        def outputProperty(name, func, prefix=None):
            let body = functionDoc(func)
            let maybePrefix = prefix + '.' if prefix else ''
            let formatted = '<i>property</i> ' + maybePrefix + '<b>' + name + '</b>'
            print('\\methodstart{' + (maybePrefix + name).replace('.','_') + ',' + formatted + ',' + ('h4,methodDef' if prefix else 'h3,functionDef') + '}')
            if body: print(body)
            print('\n\\methodend')
            print('')

        def outputConstant(name, val):
            print(f'<h3 class=\"letDef\"><i>let</i> <b>{name}</b> = <code>\htmlonly {val!r} \\endhtmlonly</code></h3>\n')

        def outputOther(name, val):
            print(f'<h3 class=\"letDef\"><i>let</i> <b>{name}</b> = <i>{type(val).__name__}</i></h3>\n')

        def outputClass(name, cls):
            let classType = 'exceptionDef' if isExceptionType(cls) else 'classDef'
            let superclass = cls.__base__.__name__ if cls.__base__ else ''
            let formatted = f'<i>class</i> <b>{name}</b>({superclass})'
            print('\\methodstart{' + name + ',' + formatted + ',h3,' + classType + '}')
            if '__doc__' in dir(cls) and isinstance(cls.__doc__,str) and (cls is object or cls is cls.__base__ or cls.__doc__ is not cls.__base__.__doc__):
                print('<p>' + cls.__doc__ + '</p>')
            let seen = []
            let methods = []
            let properties = []
            for member in dir(cls):
                if member in blacklistedMethods: continue
                realPrint(cls,member)
                let obj
                try:
                   obj = getattr(cls,member)
                    if cls.__base__ and member in dir(cls.__base__) and getattr(cls.__base__,member) == obj: continue
                except:
                    continue
                if isinstance(obj, function) and member not in seen:
                    seen.append(member)
                    methods.append(Pair(member,obj))
                else if isinstance(obj, property) and member not in seen:
                    seen.append(member)
                    properties.append(Pair(member,obj.fget))
            if methods:
                methods.sort()
                for methodName, method in methods:
                    outputFunction(methodName, method, prefix=name)
            if properties:
                properties.sort()
                for propName, method in properties:
                    outputProperty(propName, method, prefix=name)
            print('\\methodend')

        def dumpModule(name, thing):
            let functions = []
            let classes = []
            let constants = []
            let exceptions = []
            let other = []

            for member in dir(thing):
                if not member.startswith('_'):
                    let obj = getattr(thing,member)
                    if isinstance(obj, function):
                        if hasattr(obj,'__file__') and thing.__file__ and obj.__file__ != thing.__file__:
                            if not (thing.__file__.endswith('.so') and obj.__file__ == '<builtin>'):
                                continue
                        functions.append(Pair(member,obj))
                    else if isinstance(obj, type):
                        if hasattr(obj,'__module__') and obj.__module__ and obj.__module__ != name:
                            continue
                        if isExceptionType(obj):
                            exceptions.append(Pair(member,obj))
                        else:
                            classes.append(Pair(member,obj))
                    else if isinstance(obj, (int,str,float,bool,type(None))):
                        constants.append(Pair(member,obj))
                    else if isinstance(obj, kuroko.module):
                        continue # Skip top-level modules as these are almost definitely imports
                    else:
                        other.append(Pair(member,obj))

            if hasattr(module, "__ispackage__") and module.__ispackage__:
                print("\n### Package contents\n")
                print('\htmlonly<div class="krk-class-index"><ul>\n')
                for i in modules:
                    if not i.startswith(name + "."):
                        continue
                    let uscored = fixup(i).replace('.','_')
                    let relative = i[len(name) + 1:]
                    print(f'<li><a class="el" href="mod_{uscored}.html">{relative}</a></li>\n')
                print('</ul></div>\endhtmlonly\n')

            if classes:
                print('\n### Classes\n')
                classes.sort()
                for clsName, cls in classes:
                    outputClass(clsName, cls)
                    globalClassList.append((name.replace('.','_'), clsName))

            if functions:
                print('\n### Functions\n')
                functions.sort()
                for funcName, func in functions:
                    outputFunction(funcName, func)
                    globalFunctionList.append((name.replace('.','_'), funcName))

            if exceptions:
                print('\n### Exceptions\n')
                exceptions.sort()
                for name, cls in exceptions:
                    outputClass(name, cls)

            if constants:
                print('\n### Constants\n')
                constants.sort()
                for name, val in constants:
                    outputConstant(name,val)

            if other:
                print('\n### Other Members\n')
                other.sort()
                for name, val in other:
                    outputOther(name, val)

        dumpModule(modulepath,module)

        output.close()

    with fileio.open('docs/modulelist.md','w') as output:
        output.write(
            '# Module List {#modulelist}\n'
            '\n'
            'Here is a list of documented modules:\n'
            '\n'
            '\\modulelist{\n'
        )

        modules.sort()

        for module in modules:
            output.write('  \\krkmodule{mod_' + fixup(module).replace('.','_') + ',' + fixupDoc(docString[module]) + '}\n')

        output.write('}\n')

    with fileio.open('docs/classindex.md','w') as output:
        output.write(
            '# Class Index {#classindex}\n'
            '\n'
            'Here are all of the available classes:\n'
            '\n'
            '\htmlonly<div class="krk-class-index"><ul>\n')

        for moduleName, className in globalClassList:
            output.write(f'<li><a class="el" href="mod_{moduleName}.html#{className}">{className}</a> <i>from <b>{moduleName}</b></i></li>\n')

        output.write('</ul></div>\endhtmlonly\n')

    with fileio.open('docs/functionindex.md','w') as output:
        output.write(
            '# Function Index {#functionindex}\n'
            '\n'
            'Here are all of the available functions:\n'
            '\n'
            '\htmlonly<div class="krk-class-index"><ul>\n')

        for moduleName, funcName in globalFunctionList:
            output.write(f'<li><a class="el" href="mod_{moduleName}.html#{funcName}">{funcName}</a> <i>from <b>{moduleName}</b></i></li>\n')

        output.write('</ul></div>\endhtmlonly\n')

if __name__ == '__main__':
    processModules(modules)