#!/usr/bin/python # -*t coding: utf-8 -*- #copyright Ben Lipkowitz 2010 #you may redistribute this under the GNU General Public License version 2 or later from __future__ import division from collections import defaultdict import re import sys import time #import cairo import math #import yaml import os import psyco '''Graphical chart of my sleep habits, activities, food, etc. TODO: support for vertical graphs of arbitrary values temperature? rainfall? moon phase plot dynamic CSS with mouse-over hilighting click to hilight category java search/hilight box? expand abbrev's and food constituents integrate totals over an interval keep track of sleep onset/waking delta-T find average delta-T and standard deviation (daily, monthly) do statistical correlation dT*{food, act, onset time, ??} cron job''' psyco.full() #useful parameters newest_first = True #order reverse chronologically root = 'sleep' #timestamp timestamp tag stuff #denormalized = open("denormalized.txt", 'w') def get_scale(cr): '''for one pixel lines etc''' tmp = cr.user_to_device(1,1) scale = (tmp[0] + tmp[1]) / 2. return scale category_colors = {"net":"red", "sleep":"blue", "food":"green", "chat":"yellow", "writing":"yellow", "art":"goldenrod", "nothing":"magenta", "domestics":"darkgreen", "project":"lightgreen", "practice":"lightgreen", "fix":"lightgreen", "act":"darkorange", "watch":"cyan", "listen":"cyan", "work":"gray", "order":"gray", "help":"gray", "search":"black", "stupid":"black", "fight":"black", #those below i'm not so sure about "setup":"darkcyan", "code":"goldenrod", "garden":"darkgreen", "play":"orange", "clean":"darkcyan", "assimilate":"darkcyan", "driving":"darkred", "riding": "darkred", "walking": "darkred", "flying": "darkred", "train": "darkred", "shoppin":"darkorange", "science":"darkblue", "read":"darkmagenta" } rgb_map = {"red": (1, 0, 0), "green": (0, 1, 0), "blue": (0, 0, 1), "yellow": (1, 1, 0), "magenta": (1, 0, 1), "cyan": (0, 1, 1), "orange": (1, 0.5, 0), "darkgreen":(0, 0.5, 0), "darkblue": (0, 0, 0.5), "goldenrod":(0.5, 0.5, 0), "darkcyan": (0, 0.5, 0.5), "darkorange":(0.5, 0.25, 0), "darkred": (0.5, 0, 0), "darkmagenta":(0.5, 0, 0.5) } stylesheet = '''
''' class Chart: def __init__(self): global root self.dirname = root + '/' if not os.path.isdir(self.dirname): os.mkdir(self.dirname) # self.row_width, self.row_height = 613, 20 def get_num_rows(self): try: num_rows = len(self.days) except AttributeError: num_rows = 1 return num_rows def resize(self): self.width, self.height = self.row_width, self.row_height*self.get_num_rows() def init_cairo(self, filename=None): if filename == None: filename = self.filename self.resize() self.surface = cairo.SVGSurface(filename + '.svg', self.width, self.height) cr = self.cr = cairo.Context(self.surface) cr.translate(0.5, 0.5) cr.set_source_rgba(1,1,1,1) cr.rectangle(0,0,self.width, self.height) cr.fill() def write_png(self, filename=None): if filename == None: filename = self.filename self.surface.write_to_png(filename + '.png') self.surface.finish() class Month(Chart): '''date is a human readable filename string, start is unix timestamp at midnight''' def __init__(self, date, start): Chart.__init__(self) self.filename = self.dirname + date self.date, self.start = date, start self.days = [] def draw(self, cr): self.resize() cr.save() for day in self.days: day.row_width, day.row_height = self.row_width, self.row_height day.draw(cr) cr.translate(0, self.row_height) cr.restore() def css(self, context): for day in sorted(self.days, key=lambda(x): x.start, reverse=newest_first): day.css(context) class Day(Chart): '''date is a human readable filename string, start is unix timestamp at midnight''' def __init__(self, date, start): Chart.__init__(self) self.dirname = self.dirname + yyyy_mm(start) + '/' if not os.path.isdir(self.dirname): os.mkdir(self.dirname) self.filename = self.dirname + date self.date, self.start = date, start self.intervals = [] self.sleep_time, self.awake_time = 0,0 self.orange = False self.melatonin = False def draw(self, cr): self.resize() width, height = self.width, self.height cr.save() cr.scale(width, height) aspect = width/height cr.set_line_width(1./get_scale(cr)) cr.set_source_rgba(1, 1, 1, 0.1) #cr.fill() cr.set_source_rgba(0, 0, 0, 0.5) #underscore cr.move_to(0, 0.98) cr.line_to(1, 0.98) cr.stroke() #24 ruler for n in range(0, 24): x, y = (n/24, 0.98) ##tic marks #cr.move_to(x, y) #cr.line_to(x, 0.7) #cr.stroke() label = str(n) cr.move_to(x, y) cr.save() cr.identity_matrix() cr.set_font_size(width/(24*2)) cr.show_text(label) cr.fill() cr.restore() for i in self.intervals: i.draw(cr, width, height) cr.restore() def css(self, context): context.write('
'+self.date) if self.orange: orange = 'orange' else: orange = '' if self.melatonin: m = 'melatonin' else: m = '' context.write('''
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23''' % (m, orange)) for i in self.intervals: #we always want to read left to right, right? i.css(context) context.write('
') class Interval: def __init__(self, start, end, location=None): self.start, self.end = start, end self.actions = [] start_date = yyyy_mm_dd(self.start) end_date = yyyy_mm_dd(self.end) if start_date != end_date: print "interval %d %d crosses date boundary!" % (start, end) self.date = yyyy_mm_dd(self.start) def draw(self, cr, width, height): '''draws a gantt-charty-thing to a cairo context, with overlapping colored rectangles''' aspect = width/height day_secs = 24*60*60 day_start = days[self.date].start for action in self.actions: try: color = rgb_map[category_colors[action.category]] cr.set_source_rgba(color[0], color[1], color[2], 0.65) break except KeyError: cr.set_source_rgba(0, 0, 0, 0.2) cr.rectangle((self.start-day_start)/day_secs, 0, ((self.end - self.start )/day_secs), 0.95) cr.fill_preserve() cr.set_source_rgba(0, 0, 0, 1) cr.save() cr.set_line_width(1) cr.identity_matrix() cr.stroke() cr.restore() def css(self, context): day_secs = 24*60*60 day_start = days[self.date].start current_category = None; for action in self.actions: current_category = action.category try: color = category_colors[action.category] break except KeyError: color = 'white' context.write('''
%s
 ''' % (100*(self.start-day_start)/day_secs, 100*((self.end - self.start )/day_secs), color, action)) def yyyy_mm_dd(timestamp): '''don't forget to set your timezone properly''' return "%04d_%02d_%02d" % (time.localtime(timestamp)[0:3]) def yyyy_mm(timestamp): '''don't forget to set your timezone properly''' return "%04d_%02d" % (time.localtime(timestamp)[0:2]) class Action: def __init__(self, interval, category, string=None, comment=None): self.interval, self.category, self.string, self.comment = \ interval, category, string, comment def __str__(self): return "%s %s" %(str(self.category), str(self.string)) days = {} months = {} categories_time = defaultdict(int) categories_freq = defaultdict(int) net_topics_freq = defaultdict(int) net_topics_time = defaultdict(int) net_topics_ngrams_freq = defaultdict(int) net_topics_ngrams_time = defaultdict(int) wdata_epoch = [] def parse(file): '''each line looks like this: 1341 1514 work do-this do-that do-the-other, food carrot 2pc pie''' line = "hi!" global days linenum = 0 category, string, comment = None, None, None year, month, day = 0,0,0 last_month = 0 last_sleep_epoch = 1115932860 starthour, startmin, endhour, endmin, actions = 0,0,0,0,0 startepoch, laststart, endepoch, lastend = 0,0,0,0 while line: linenum += 1 line = file.readline() d_match = re.match('^date (\d{4}) (\d{2}) (\d{2})', line) if d_match: #a date year, month, day = (int(i) for i in d_match.groups()) start = time.mktime((year, month, day, 0, 0, 0, 0, 0, -1)) date = yyyy_mm_dd(start) days[date] = Day(date, start) #the filename yyyy_mm_dd (date) is the dict key monte = yyyy_mm(start) #like date for months instead of days if month != last_month: months[monte] = Month(monte, start) last_month = month months[monte].days.append(days[date]) i_match = re.match('^(\d{2})(\d{2}) (\d{2})(\d{2})(.*)', line) if i_match: #an interval starthour, startmin, endhour, endmin = (int(i) for i in i_match.group(1,2,3,4)) actions = i_match.groups()[-1] #print "%d %02d %02d %d%d %d%d %s" % (year, month, day, starthour, startmin, endhour, endmin, actions) laststart, lastend = startepoch, endepoch startepoch = time.mktime((year, month, day, starthour, startmin, 0, 0, 0, -1)) endepoch = time.mktime((year, month, day, endhour, endmin, 0, 0, 0, -1)) #print "start: %s end: %s actions: %s" %(startepoch, endepoch, actions) elapsed = endepoch - startepoch if (startepoch > endepoch) or (lastend > startepoch): print "time discrepancy!", date, line elapsed = 0 action_pattern = '\s(\w+)\s?([^,]*)' a_match = re.match('\s(\w+)\s?([^,]*)', actions) if a_match: category, string = a_match.groups() #denormalized.write('%d %d %s "%s"\n' %(startepoch, endepoch, category, string)) categories_freq[category] += 1 categories_time[category] += elapsed if category == 'net': for ngram in string.split(" "): net_topics_ngrams_freq[ngram] += 1 #track how often we see this phrase time_split = elapsed/len(string.split(" ")) net_topics_ngrams_time[ngram] += time_split for gram in ngram.split("-"): net_topics_freq[gram] += 1 net_topics_time[gram] += time_split #sums to more than 100% time spent on net if category == 'sleep': days[date].awake_time += startepoch - last_sleep_epoch days[date].sleep_time += endepoch - startepoch last_sleep_epoch = endepoch if category == 'herbal' and (string.find('melatonin') != -1): days[date].melatonin = True if category == 'orange': days[date].orange = True if category == 'data': print "data!" wdata_epoch.append([string, startepoch]) c_match = re.match('\(.*\)', actions) if c_match: comment = a_match.groups() #pack it into our object interval = Interval(startepoch, endepoch) #TODO one per action, not just a string... interval.actions.append(Action(interval, category, string=string, comment = comment)) interval.line = line.strip('\n') if interval.date != date: print "date discrepancy! %s %s" % (date, interval.date) days[date].intervals.append(interval) def draw_svg(months): import cairo #so it doesn't screw up people just trying to draw css #maybe i should explicitly create the dir here? keys = months.keys() #what? keys.sort() for monte in keys: html_m = open(root + month.filename + ".html", "w") html_m.write('\n') month = months[monte] month.row_width, month.row_height = 200, 5 month.init_cairo() month.draw(month.cr) print month.filename month.write_png() relative_d = '/'.join(month.filename.split('/')[1:]) html_m.write('\n') html_m.write('
') html_m.write('\n') html_d = open(month.filename + '.html', 'w') html_d.write('\n') for day in month.days: day.row_width, day.row_height = 613, 20 day.init_cairo() day.draw(day.cr) day.write_png() relative_d = '/'.join(day.filename.split('/')[1:]) html_d.write('
\n') html_d.write('') html_d.close() #print yaml.dump(day) def sortdict(mydict, reverse=False, sortby=0): '''sort by value''' if isinstance(mydict, list): if len(mydict) == 0: return [] if isinstance(mydict[0], list): return sorted(mydict, key=lambda(x): x[sortby], reverse=reverse) #sort by first element else: return sorted(mydict, reverse=reverse) #just return the list, sorted return sorted(mydict.iteritems(), key=lambda (k,v): v, reverse=reverse) #sort by value def write_stats(data, name, reverse=True, sortby=0): '''dump an incidence count dict into a file''' try: os.mkdir('stats') except OSError: pass f = open('stats/'+name+'.txt','w') for x in sortdict(data, reverse=reverse, sortby=sortby): f.write("%d %s\n"%(x[1], x[0])) f.close() if __name__ == '__main__': parse(sys.stdin) #parse(open("/home/fenn/rl/pda_db.txt")) #parse(open("2011.txt")) #parse(open("melatonin.txt")) #parse(open("test.txt")) #parse(open("10000.txt")) keys = sorted(months.keys(), reverse=newest_first) #new css-based version html_m = open(root + "/css.html", "w") html_m.write(stylesheet) for monte in keys: print monte month = months[monte] month.css(html_m) html_m.write('') html_m.close() write_stats(net_topics_ngrams_freq, 'net_topics_ngrams_freq') write_stats(net_topics_ngrams_time, 'net_topics_ngrams_time') write_stats(net_topics_freq, 'net_topics_freq') write_stats(net_topics_time, 'net_topics_time') write_stats(categories_freq, 'categories_freq') write_stats(categories_time, 'categories_time') print "wdata_epoch:", len(wdata_epoch) write_stats(wdata_epoch, 'wdata_epoch', sortby=1, reverse=False) f = open('stats/sleep_stats.txt','w') for day in days.values(): #f.write('%s: %d,\n' %(day.date, day.sleep_time)) f.write('%d %d\n' %(day.start, day.sleep_time)) f.close() #for x in sortdict(net_topics_ngrams_time): print x[1], x[0] #print "net_topics:" + '\n', sortdict(net_topics, reverse=True) #print "Categories:" + '\n', sortdict(categories, reverse=True) #denormalized.close()