happybot/graph.py

#!/usr/bin/env python3

from graphviz import Digraph
from re import compile as regex
from collections import defaultdict
from subprocess import check_output
from os import listdir
from itertools import combinations_with_replacement as comb
from colorsys import hsv_to_rgb
from math import inf, log2

# The magic offtopia-specific variables.
eval(compile(check_output("""cat /home/zgrep/irctimes/munge.py | sed -n '/^rename\s*=\s*/,/)$/p'""", shell=True), __file__, 'exec'))
bots = 'MartesZibellina program minicat Eldis4 happybot bslsk05 WOPO oonbotti2 j-bot PeNGu1N_oF_d00m'.split()
common = [''.join(chr(int(n)) for n in x.split('_')).lower() for x in listdir('/home/zgrep/irctimes/numpydata')]

colors = ['#f0a3ff', '#0075dc', '#993f00', '#4c005c', '#191919', '#005c31', '#2bce48',
          '#ffcc99', '#808080', '#94ffb5', '#8f7c00', '#9dcc00', '#c20088', '#003380',
          '#ffa405', '#ffa8bb', '#426600', '#ff0010', '#5ef1f2', '#00998f', '#e0ff66',
          '#740aff', '#990000', '#ffff80', '#ffff00', '#ff5005']

talkto = regex(r'[0-9-]+ [0-9:]+ <([^>]+)> (?:(\S+?):\s+|\s*<([^>]+)>)?')

def readlogs():
    with open('/home/zgrep/offtopiabday/irc.freenode.net/#offtopia/out', 'rb') as fh:
        prev, replyto = None, None
        for line in fh:
            line = line.decode('utf-8', errors='ignore')
            m = talkto.match(line)
            if m:
                nick, to, to2 = m.groups()
                nick = rename.get(nick, nick)
                if to2: to = to2
                to = rename.get(to, to)
                if to and to not in ('nolog', 'D', ':D'):
                    replyto = to
                elif nick != prev:
                    replyto = prev
                if replyto\
                        and replyto not in bots\
                        and replyto.lower() in common\
                        and nick not in bots\
                        and nick.lower() in common:
                    yield nick, replyto
                prev = nick

def accum(it):
    numlines = defaultdict(int)
    graph = defaultdict(int)
    for nick, to in it:
        numlines[nick] += 1
        graph[(nick, to)] += 1
    return numlines, graph

def top(numlines, n=10):
    global colors
    assert isinstance(n, int) and 0 < n <= len(colors)
    people = list(sorted(numlines, key=numlines.get, reverse=True))
    return people[:n]

def minmax(people, data):
    minimum, maximum = inf, 0
    for a in people:
        for b in people:
            d = data[(a, b)]
            if d < minimum:
                minimum = d
            if d > maximum:
                maximum = d
    return minimum, maximum

def matrix(people, data):
    matrix = [ [0] * (1 + len(people)) for _ in range(1 + len(people)) ]
    matrix[0][0] = None
    for i, a in enumerate(people):
        matrix[0][1+i] = a
        matrix[1+i][0] = a
        for j, b in enumerate(people):
            matrix[1+i][1+j] = data[(a, b)]
    return '# Row ---sends-to---> Column\n\n' + repr(matrix) + '\n'

outdir = '/home/zgrep/offtopiabday/cloud/output/graph/'

def main():
    global colors
    print('Reading logs...')
    numlines, data = accum(readlogs())

    print('Creating matrix...')
    with open(outdir + 'matrix.py', 'w') as fh:
        fh.write(matrix(list(numlines), data))

    for a in numlines:
        print('Crunching', a, 'data...')
        graph = Digraph(format='png', engine='circo')
        graph.attr('node', shape='plaintext', fontsize='20', margin='0.2')

        minimum, maximum = inf, 0
        for b in numlines:
            for d in (data[(a, b)], data[(b, a)]):
                if d < minimum:
                    minimum = d
                if d > maximum:
                    maximum = d

        for b in numlines:
            fn, rn = data[(a, b)], data[(b, a)]
            f = 10 * (fn - minimum) / max(1, maximum - minimum)
            r = 10 * (rn - minimum) / max(1, maximum - minimum)
            w = f + r
            if w < 0.05:
                continue
            i = log2(1+log2(1+abs(f - r)/max(f, r)))
            if r > f:
                color = hsv_to_rgb(0, 1, i)
            else:
                color = hsv_to_rgb(1/3, 1, i)
            color = '#' + ''.join(hex(int(255*c))[2:].zfill(2) for c in color)
            if f < r:
                direction='back'
            elif f > r:
                direction = ''
            else:
                direction = 'both'
            graph.edge(a, b, dir=direction, penwidth=str(w), color=color)

        print('Drawing', a, 'graph...')
        graph.render(outdir + a.replace('/', '*'))

    print('Crunching collective data...')
    people = top(numlines, len(colors))
    minimum, maximum = minmax(people, data)
    colormap = { p: c for p, c in zip(sorted(people), colors) }

    graph = Digraph(format='png', engine='circo')
    graph.attr('node', shape='plaintext', fontsize='25', margin='0.2')
    #graph.attr('graph', bgcolor='#000000')

    for p in people:
        graph.node(p, fontcolor=colormap[p])

    for a, b in comb(people, 2):
        f, r = data[(a, b)], data[(b, a)]
        f = 15 * (f - minimum) / max(1, maximum - minimum)
        r = 15 * (r - minimum) / max(1, maximum - minimum)
        w = f + r
        if w < 0.05:
            continue
        graph.edge(a, b, dir='both', color=colormap[b] + ';0.5:' + colormap[a], penwidth=str(w))

    print('Drawing graph...')
    graph.render(outdir + 'c/offtopia')

    print('Done.')

    return list(numlines)

if __name__ == '__main__':
    people = main()
    people.sort()
    print('Making index.')
    with open(outdir + 'index.html', 'w') as fh:
        fh.write('<!DOCTYPE html><title>#offtopia graphs</title><meta charset="utf-8"/><ul>')
        fh.write('<li><a href="c/offtopia.png">#offtopia</a> (<a href="c/offtopia">dot</a>)</li>')
        for dotname in people:
            fname = dotname + '.png'
            fh.write('<li><a href="' + fname + '">' + dotname + '</a> (<a href="' + dotname + '">dot</a>)')
        fh.write('<li><a href="matrix.py">python matrix</a></li>')
        fh.write('</ul>\n')