bootchart: merge bootchart

Bootchart is renamed to 'systemd-bootchart' and installed as
/usr/lib/systemd/systemd-bootchart. The configuration file
will reside in /etc/systemd/bootchart.conf.

src/bootchart/README

@@ -0,0 +1,83 @@
+
+Bootchart - a 'startup' graphing tool
+
+--
+
+Bootchart is a tool, usually run at system startup, that collects and graphs
+the CPU and disk load of the system as it works. The output of bootchart is
+an SVG graph. Normally, bootchart is invoked as `bootchartd` by the kernel
+by passing "init=/sbin/bootchartd" to the kernel. Bootchart will then fork
+init off to resume normal system startup, while monitoring and logging
+startup information in the background.
+
+After collecting a certain amount of data (usually 15-30 seconds) the logging
+stops and a graph is generated from the logged information. This graph
+contains vital clues to which resources are being used, in which order, and
+where possible problems exist in the startup sequence of the system.
+
+Of course, bootchart can also be used at any moment in time to collect and
+graph some data for an amount of time. Bootchart does not even require root
+privileges to do so, and will happily run as a normal user. Bootchart graphs
+are by default written time-stamped in /var/log.
+
+--
+
+This version of bootchart was implemented from scratch and inspired by former
+incantations of bootchart:
+
+- The original bash/shell code implemented bootchart. This version logged all
+data into a compressed tarball for later processing, and did not create a graph
+on it's own.
+
+- The C-code implementation found in Ubuntu. This version replaced above shell
+code version with a faster and efficient data logger, but still did not graph
+code itself.
+
+- the original Java-based bootchart, the original graphing program that created
+a bootchart graph from logged data.
+
+- the pybootchartgui.py program, which created a graph based on the data logged
+by either standalone data logger.
+
+The version you are looking at combines these 2 parts into a single program,
+which makes running it and creating graphs a bit more efficient and simple.
+You can now run a single program at startup instead of 2. There are no timing
+problems (the graphing stage will never run if the logging stage didn't
+finish). The logged data isn't being written to disc first, then read again.
+Also, the data kept in memory is reduced to the absolute minimum needed to
+keep memory use low.
+
+--
+
+Requirements: glibc. Your kernel must have procfs support and several
+proc output options enabled:
+  CONFIG_PROC_FS
+  CONFIG_SCHEDSTATS
+  CONFIG_SCHED_DEBUG
+at a minimum. bootchartd itself does not require any graphics library
+to generate the SVG output file.
+
+--
+
+Configuration: please see bootchartd --help, as well as /etc/bootchartd.conf
+and/or /usr/share/doc/bootchart/bootchartd.conf.example for a list of
+configurable options.
+
+--
+
+Many thanks to those who contributed ideas and code:
+ - Ziga Mahkovec - Original bootchart author
+ - Anders Norgaard - PyBootchartgui
+ - Michael Meeks - bootchart2
+ - Scott James Remnant - Ubuntu C-based logger
+ - Arjan van der Ven - for the idea to merge bootgraph.pl functionality
+
+--
+
+For bugs, please contact the author or current maintainer:
+Auke Kok <auke-jan.h.kok@intel.com>
+
+--
+
+Download bootchart releases here: http://foo-projects.org/~sofar/bootchart/
+Source code is hosted here: git://github.com/sofar/bootchart

src/bootchart/bootchart.c

@@ -0,0 +1,352 @@
+/*
+ * bootchart.c
+ *
+ * Copyright (C) 2009-2012 Intel Coproration
+ *
+ * Authors:
+ *   Auke Kok <auke-jan.h.kok@intel.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; version 2
+ * of the License.
+ */
+
+
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/resource.h>
+#include <stdio.h>
+#include <signal.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <time.h>
+#include <getopt.h>
+#include <limits.h>
+#include <errno.h>
+
+
+#include "bootchart.h"
+
+double graph_start;
+double log_start;
+double sampletime[MAXSAMPLES];
+struct ps_struct *ps_first;
+struct block_stat_struct blockstat[MAXSAMPLES];
+int entropy_avail[MAXSAMPLES];
+struct cpu_stat_struct cpustat[MAXCPUS];
+int pscount;
+int cpus;
+double interval;
+FILE *of;
+int overrun = 0;
+static int exiting = 0;
+
+/* graph defaults */
+int entropy = 0;
+int initcall = 1;
+int relative;
+int filter = 1;
+int pss = 0;
+int samples;
+int len = 500; /* we record len+1 (1 start sample) */
+double hz = 25.0;   /* 20 seconds log time */
+double scale_x = 100.0; /* 100px = 1sec */
+double scale_y = 20.0;  /* 16px = 1 process bar */
+
+char init_path[PATH_MAX] = "/sbin/init";
+char output_path[PATH_MAX] = "/var/log";
+
+static struct rlimit rlim;
+
+static void signal_handler(int sig)
+{
+	if (sig++)
+		sig--;
+	exiting = 1;
+}
+
+
+int main(int argc, char *argv[])
+{
+	struct sigaction sig;
+	struct ps_struct *ps;
+	char output_file[PATH_MAX];
+	char datestr[200];
+	time_t t;
+	FILE *f;
+	int gind;
+	int i;
+
+	memset(&t, 0, sizeof(time_t));
+
+	rlim.rlim_cur = 4096;
+	rlim.rlim_max = 4096;
+	(void) setrlimit(RLIMIT_NOFILE, &rlim);
+
+	f = fopen("/etc/systemd/bootchart.conf", "r");
+	if (f) {
+		char buf[256];
+		char *key;
+		char *val;
+
+		while (fgets(buf, 80, f) != NULL) {
+			char *c;
+
+			c = strchr(buf, '\n');
+			if (c) *c = 0; /* remove trailing \n */
+
+			if (buf[0] == '#')
+				continue; /* comment line */
+
+			key = strtok(buf, "=");
+			if (!key)
+				continue;
+			val = strtok(NULL, "=");
+			if (!val)
+				continue;
+
+			// todo: filter leading/trailing whitespace
+
+			if (!strcmp(key, "samples"))
+				len = atoi(val);
+			if (!strcmp(key, "freq"))
+				hz = atof(val);
+			if (!strcmp(key, "rel"))
+				relative = atoi(val);
+			if (!strcmp(key, "filter"))
+				filter = atoi(val);
+			if (!strcmp(key, "pss"))
+				pss = atoi(val);
+			if (!strcmp(key, "output"))
+				strncpy(output_path, val, PATH_MAX - 1);
+			if (!strcmp(key, "init"))
+				strncpy(init_path, val, PATH_MAX - 1);
+			if (!strcmp(key, "scale_x"))
+				scale_x = atof(val);
+			if (!strcmp(key, "scale_y"))
+				scale_y = atof(val);
+			if (!strcmp(key, "entropy"))
+				entropy = atoi(val);
+		}
+		fclose(f);
+	}
+
+	while (1) {
+		static struct option opts[] = {
+			{"rel", 0, NULL, 'r'},
+			{"freq", 1, NULL, 'f'},
+			{"samples", 1, NULL, 'n'},
+			{"pss", 0, NULL, 'p'},
+			{"output", 1, NULL, 'o'},
+			{"init", 1, NULL, 'i'},
+			{"filter", 0, NULL, 'F'},
+			{"help", 0, NULL, 'h'},
+			{"scale-x", 1, NULL, 'x'},
+			{"scale-y", 1, NULL, 'y'},
+			{"entropy", 0, NULL, 'e'},
+			{NULL, 0, NULL, 0}
+		};
+
+		gind = 0;
+
+		i = getopt_long(argc, argv, "erpf:n:o:i:Fhx:y:", opts, &gind);
+		if (i == -1)
+			break;
+		switch (i) {
+		case 'r':
+			relative = 1;
+			break;
+		case 'f':
+			hz = atof(optarg);
+			break;
+		case 'F':
+			filter = 0;
+			break;
+		case 'n':
+			len = atoi(optarg);
+			break;
+		case 'o':
+			strncpy(output_path, optarg, PATH_MAX - 1);
+			break;
+		case 'i':
+			strncpy(init_path, optarg, PATH_MAX - 1);
+			break;
+		case 'p':
+			pss = 1;
+			break;
+		case 'x':
+			scale_x = atof(optarg);
+			break;
+		case 'y':
+			scale_y = atof(optarg);
+			break;
+		case 'e':
+			entropy = 1;
+			break;
+		case 'h':
+			fprintf(stderr, "Usage: %s [OPTIONS]\n", argv[0]);
+			fprintf(stderr, " --rel,     -r            Record time relative to recording\n");
+			fprintf(stderr, " --freq,    -f N          Sample frequency [%f]\n", hz);
+			fprintf(stderr, " --samples, -n N          Stop sampling at [%d] samples\n", len);
+			fprintf(stderr, " --scale-x, -x N          Scale the graph horizontally [%f] \n", scale_x);
+			fprintf(stderr, " --scale-y, -y N          Scale the graph vertically [%f] \n", scale_y);
+			fprintf(stderr, " --pss,     -p            Enable PSS graph (CPU intensive)\n");
+			fprintf(stderr, " --entropy, -e            Enable the entropy_avail graph\n");
+			fprintf(stderr, " --output,  -o [PATH]     Path to output files [%s]\n", output_path);
+			fprintf(stderr, " --init,    -i [PATH]     Path to init executable [%s]\n", init_path);
+			fprintf(stderr, " --filter,  -F            Disable filtering of processes from the graph\n");
+			fprintf(stderr, "                          that are of less importance or short-lived\n");
+			fprintf(stderr, " --help,    -h            Display this message\n");
+			fprintf(stderr, "See the installed README and bootchartd.conf.example for more information.\n");
+			exit (EXIT_SUCCESS);
+			break;
+		default:
+			break;
+		}
+	}
+
+	if (len > MAXSAMPLES) {
+		fprintf(stderr, "Error: samples exceeds maximum\n");
+		exit(EXIT_FAILURE);
+	}
+
+	if (hz <= 0.0) {
+		fprintf(stderr, "Error: Frequency needs to be > 0\n");
+		exit(EXIT_FAILURE);
+	}
+
+	/*
+	 * If the kernel executed us through init=/sbin/bootchartd, then
+	 * fork:
+	 * - parent execs executable specified via init_path[] (/sbin/init by default) as pid=1
+	 * - child logs data
+	 */
+	if (getpid() == 1) {
+		if (fork()) {
+			/* parent */
+			execl(init_path, init_path, NULL);
+		}
+	}
+
+	/* start with empty ps LL */
+	ps_first = malloc(sizeof(struct ps_struct));
+	if (!ps_first) {
+		perror("malloc(ps_struct)");
+		exit(EXIT_FAILURE);
+	}
+	memset(ps_first, 0, sizeof(struct ps_struct));
+
+	/* handle TERM/INT nicely */
+	memset(&sig, 0, sizeof(struct sigaction));
+	sig.sa_handler = signal_handler;
+	sigaction(SIGHUP, &sig, NULL);
+
+	interval = (1.0 / hz) * 1000000000.0;
+
+	log_uptime();
+
+	/* main program loop */
+	while (!exiting) {
+		int res;
+		double sample_stop;
+		struct timespec req;
+		time_t newint_s;
+		long newint_ns;
+		double elapsed;
+		double timeleft;
+
+		sampletime[samples] = gettime_ns();
+
+		/* wait for /proc to become available, discarding samples */
+		if (!(graph_start > 0.0))
+			log_uptime();
+		else
+			log_sample(samples);
+
+		sample_stop = gettime_ns();
+
+		elapsed = (sample_stop - sampletime[samples]) * 1000000000.0;
+		timeleft = interval - elapsed;
+
+		newint_s = (time_t)(timeleft / 1000000000.0);
+		newint_ns = (long)(timeleft - (newint_s * 1000000000.0));
+
+		/*
+		 * check if we have not consumed our entire timeslice. If we
+		 * do, don't sleep and take a new sample right away.
+		 * we'll lose all the missed samples and overrun our total
+		 * time
+		 */
+		if ((newint_ns > 0) || (newint_s > 0)) {
+			req.tv_sec = newint_s;
+			req.tv_nsec = newint_ns;
+
+			res = nanosleep(&req, NULL);
+			if (res) {
+				if (errno == EINTR) {
+					/* caught signal, probably HUP! */
+					break;
+				}
+				perror("nanosleep()");
+				exit (EXIT_FAILURE);
+			}
+		} else {
+			overrun++;
+			/* calculate how many samples we lost and scrap them */
+			len = len + ((int)(newint_ns / interval));
+		}
+
+		samples++;
+
+		if (samples > len)
+			break;
+
+	}
+
+	/* do some cleanup, close fd's */
+	ps = ps_first;
+	while (ps->next_ps) {
+		ps = ps->next_ps;
+		if (ps->schedstat)
+			close(ps->schedstat);
+		if (ps->sched)
+			close(ps->sched);
+		if (ps->smaps)
+			fclose(ps->smaps);
+	}
+	closedir(proc);
+
+	t = time(NULL);
+	strftime(datestr, sizeof(datestr), "%Y%m%d-%H%M", localtime(&t));
+	snprintf(output_file, PATH_MAX, "%s/bootchart-%s.svg", output_path, datestr);
+
+	of = fopen(output_file, "w");
+	if (!of) {
+		perror("open output_file");
+		exit (EXIT_FAILURE);
+	}
+
+	svg_do();
+
+	fprintf(stderr, "bootchartd: Wrote %s\n", output_file);
+	fclose(of);
+
+	/* nitpic cleanups */
+	ps = ps_first;
+	while (ps->next_ps) {
+		struct ps_struct *old = ps;
+		ps = ps->next_ps;
+		free(old->sample);
+		free(old);
+	}
+	free(ps->sample);
+	free(ps);
+
+	/* don't complain when overrun once, happens most commonly on 1st sample */
+	if (overrun > 1)
+		fprintf(stderr, "bootchartd: Warning: sample time overrun %i times\n", overrun);
+
+	return 0;
+}

src/bootchart/bootchart.conf

@@ -0,0 +1,20 @@
+
+#  This file is part of systemd.
+#
+#  systemd is free software; you can redistribute it and/or modify it
+#  under the terms of the GNU Lesser General Public License as published by
+#  the Free Software Foundation; either version 2.1 of the License, or
+#  (at your option) any later version.
+#
+# See systemd-bootchart.conf(5) for details
+
+#samples=500
+#freq=25
+#rel=0
+#filter=1
+#output=<folder name, defaults to /var/log>
+#init=/path/to/init-binary
+#pss=0
+#entropy=0
+#scale_x=100
+#scale_y=20

src/bootchart/bootchart.h

@@ -0,0 +1,117 @@
+/*
+ * bootchart.h
+ *
+ * Copyright (C) 2009-2012 Intel Coproration
+ *
+ * Authors:
+ *   Auke Kok <auke-jan.h.kok@intel.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; version 2
+ * of the License.
+ */
+
+#include <dirent.h>
+
+#define MAXCPUS        16
+#define MAXPIDS     65535
+#define MAXSAMPLES   8192
+
+
+struct block_stat_struct {
+	/* /proc/vmstat pgpgin & pgpgout */
+	int bi;
+	int bo;
+};
+
+struct cpu_stat_sample_struct {
+	/* /proc/schedstat fields 10 & 11 (after name) */
+	double runtime;
+	double waittime;
+};
+
+struct cpu_stat_struct {
+	/* per cpu array */
+	struct cpu_stat_sample_struct sample[MAXSAMPLES];
+};
+
+/* per process, per sample data we will log */
+struct ps_sched_struct {
+	/* /proc/<n>/schedstat fields 1 & 2 */
+	double runtime;
+	double waittime;
+	int pss;
+};
+
+/* process info */
+struct ps_struct {
+	struct ps_struct *next_ps;    /* SLL pointer */
+	struct ps_struct *parent;     /* ppid ref */
+	struct ps_struct *children;   /* children */
+	struct ps_struct *next;       /* siblings */
+
+	/* must match - otherwise it's a new process with same PID */
+	char name[16];
+	int pid;
+	int ppid;
+
+	/* cache fd's */
+	int sched;
+	int schedstat;
+	FILE *smaps;
+
+	/* index to first/last seen timestamps */
+	int first;
+	int last;
+
+	/* records actual start time, may be way before bootchart runs */
+	double starttime;
+
+	/* record human readable total cpu time */
+	double total;
+
+	/* largest PSS size found */
+	int pss_max;
+
+	/* for drawing connection lines later */
+	double pos_x;
+	double pos_y;
+
+	struct ps_sched_struct *sample;
+};
+
+extern int entropy_avail[];
+
+extern double graph_start;
+extern double log_start;
+extern double sampletime[];
+extern struct ps_struct *ps_first;
+extern struct block_stat_struct blockstat[];
+extern struct cpu_stat_struct cpustat[];
+extern int pscount;
+extern int relative;
+extern int filter;
+extern int pss;
+extern int entropy;
+extern int initcall;
+extern int samples;
+extern int cpus;
+extern int len;
+extern double hz;
+extern double scale_x;
+extern double scale_y;
+extern int overrun;
+extern double interval;
+
+extern char output_path[PATH_MAX];
+extern char init_path[PATH_MAX];
+
+extern FILE *of;
+extern DIR *proc;
+
+extern double gettime_ns(void);
+extern void log_uptime(void);
+extern void log_sample(int sample);
+
+extern void svg_do(void);

src/bootchart/log.c

@@ -0,0 +1,420 @@
+/*
+ * log.c
+ *
+ * Copyright (C) 2009-2012 Intel Coproration
+ *
+ * Authors:
+ *   Auke Kok <auke-jan.h.kok@intel.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; version 2
+ * of the License.
+ */
+
+#define _GNU_SOURCE 1
+#include <unistd.h>
+#include <stdlib.h>
+#include <limits.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <stdio.h>
+#include <string.h>
+#include <dirent.h>
+#include <fcntl.h>
+#include <time.h>
+
+
+#include "bootchart.h"
+
+/*
+ * Alloc a static 4k buffer for stdio - primarily used to increase
+ * PSS buffering from the default 1k stdin buffer to reduce
+ * read() overhead.
+ */
+static char smaps_buf[4096];
+DIR *proc;
+
+
+double gettime_ns(void)
+{
+	struct timespec now;
+
+	clock_gettime(CLOCK_MONOTONIC, &now);
+
+	return (now.tv_sec + (now.tv_nsec / 1000000000.0));
+}
+
+
+void log_uptime(void)
+{
+	FILE *f;
+	char str[32];
+	double uptime;
+
+	f = fopen("/proc/uptime", "r");
+	if (!f)
+		return;
+	if (!fscanf(f, "%s %*s", str)) {
+		fclose(f);
+		return;
+	}
+	fclose(f);
+	uptime = strtod(str, NULL);
+
+	log_start = gettime_ns();
+
+	/* start graph at kernel boot time */
+	if (relative)
+		graph_start = log_start;
+	else
+		graph_start = log_start - uptime;
+}
+
+
+static char *bufgetline(char *buf)
+{
+	char *c;
+
+	if (!buf)
+		return NULL;
+
+	c = strchr(buf, '\n');
+	if (c)
+		c++;
+	return c;
+}
+
+
+void log_sample(int sample)
+{
+	static int vmstat;
+	static int schedstat;
+	FILE *st;
+	char buf[4095];
+	char key[256];
+	char val[256];
+	char rt[256];
+	char wt[256];
+	char *m;
+	int c;
+	int p;
+	int mod;
+	static int e_fd;
+	ssize_t s;
+	ssize_t n;
+	struct dirent *ent;
+
+	if (!vmstat) {
+		/* block stuff */
+		vmstat = open("/proc/vmstat", O_RDONLY);
+		if (vmstat == -1) {
+			perror("open /proc/vmstat");
+			exit (EXIT_FAILURE);
+		}
+	}
+
+	n = pread(vmstat, buf, sizeof(buf) - 1, 0);
+	if (n <= 0) {
+		close(vmstat);
+		return;
+	}
+	buf[n] = '\0';
+
+	m = buf;
+	while (m) {
+		if (sscanf(m, "%s %s", key, val) < 2)
+			goto vmstat_next;
+		if (!strcmp(key, "pgpgin"))
+			blockstat[sample].bi = atoi(val);
+		if (!strcmp(key, "pgpgout")) {
+			blockstat[sample].bo = atoi(val);
+			break;
+		}
+vmstat_next:
+		m = bufgetline(m);
+		if (!m)
+			break;
+	}
+
+	if (!schedstat) {
+		/* overall CPU utilization */
+		schedstat = open("/proc/schedstat", O_RDONLY);
+		if (schedstat == -1) {
+			perror("open /proc/schedstat");
+			exit (EXIT_FAILURE);
+		}
+	}
+
+	n = pread(schedstat, buf, sizeof(buf) - 1, 0);
+	if (n <= 0) {
+		close(schedstat);
+		return;
+	}
+	buf[n] = '\0';
+
+	m = buf;
+	while (m) {
+		if (sscanf(m, "%s %*s %*s %*s %*s %*s %*s %s %s", key, rt, wt) < 3)
+			goto schedstat_next;
+
+		if (strstr(key, "cpu")) {
+			c = atoi((const char*)(key+3));
+			if (c > MAXCPUS)
+				/* Oops, we only have room for MAXCPUS data */
+				break;
+			cpustat[c].sample[sample].runtime = atoll(rt);
+			cpustat[c].sample[sample].waittime = atoll(wt);
+
+			if (c == cpus)
+				cpus = c + 1;
+		}
+schedstat_next:
+		m = bufgetline(m);
+		if (!m)
+			break;
+	}
+
+	if (entropy) {
+		if (!e_fd) {
+			e_fd = open("/proc/sys/kernel/random/entropy_avail", O_RDONLY);
+		}
+
+		if (e_fd) {
+			n = pread(e_fd, buf, sizeof(buf) - 1, 0);
+			if (n > 0)
+				entropy_avail[sample] = atoi(buf);
+		}
+	}
+
+	/* all the per-process stuff goes here */
+	if (!proc) {
+		/* find all processes */
+		proc = opendir("/proc");
+		if (!proc)
+			return;
+	} else {
+		rewinddir(proc);
+	}
+
+	while ((ent = readdir(proc)) != NULL) {
+		char filename[PATH_MAX];
+		int pid;
+		struct ps_struct *ps;
+
+		if ((ent->d_name[0] < '0') || (ent->d_name[0] > '9'))
+			continue;
+
+		pid = atoi(ent->d_name);
+
+		if (pid >= MAXPIDS)
+			continue;
+
+		ps = ps_first;
+		while (ps->next_ps) {
+			ps = ps->next_ps;
+			if (ps->pid == pid)
+				break;
+		}
+
+		/* end of our LL? then append a new record */
+		if (ps->pid != pid) {
+			char t[32];
+			struct ps_struct *parent;
+
+			ps->next_ps = malloc(sizeof(struct ps_struct));
+			if (!ps->next_ps) {
+				perror("malloc(ps_struct)");
+				exit (EXIT_FAILURE);
+			}
+			memset(ps->next_ps, 0, sizeof(struct ps_struct));
+			ps = ps->next_ps;
+			ps->pid = pid;
+
+			ps->sample = malloc(sizeof(struct ps_sched_struct) * (len + 1));
+			if (!ps->sample) {
+				perror("malloc(ps_struct)");
+				exit (EXIT_FAILURE);
+			}
+			memset(ps->sample, 0, sizeof(struct ps_sched_struct) * (len + 1));
+
+			pscount++;
+
+			/* mark our first sample */
+			ps->first = sample;
+
+			/* get name, start time */
+			if (!ps->sched) {
+				sprintf(filename, "/proc/%d/sched", pid);
+				ps->sched = open(filename, O_RDONLY);
+				if (ps->sched == -1)
+					continue;
+			}
+
+			s = pread(ps->sched, buf, sizeof(buf) - 1, 0);
+			if (s <= 0) {
+				close(ps->sched);
+				continue;
+			}
+
+			if (!sscanf(buf, "%s %*s %*s", key))
+				continue;
+
+			strncpy(ps->name, key, 16);
+			/* discard line 2 */
+			m = bufgetline(buf);
+			if (!m)
+				continue;
+
+			m = bufgetline(m);
+			if (!m)
+				continue;
+
+			if (!sscanf(m, "%*s %*s %s", t))
+				continue;
+
+			ps->starttime = strtod(t, NULL) / 1000.0;
+
+			/* ppid */
+			sprintf(filename, "/proc/%d/stat", pid);
+			st = fopen(filename, "r");
+			if (!st)
+				continue;
+			if (!fscanf(st, "%*s %*s %*s %i", &p)) {
+				fclose(st);
+				continue;
+			}
+			fclose(st);
+			ps->ppid = p;
+
+			/*
+			 * setup child pointers
+			 *
+			 * these are used to paint the tree coherently later
+			 * each parent has a LL of children, and a LL of siblings
+			 */
+			if (pid == 1)
+				continue; /* nothing to do for init atm */
+
+			/* kthreadd has ppid=0, which breaks our tree ordering */
+			if (ps->ppid == 0)
+				ps->ppid = 1;
+
+			parent = ps_first;
+			while ((parent->next_ps && parent->pid != ps->ppid))
+				parent = parent->next_ps;
+
+			if ((!parent) || (parent->pid != ps->ppid)) {
+				/* orphan */
+				ps->ppid = 1;
+				parent = ps_first->next_ps;
+			}
+
+			ps->parent = parent;
+
+			if (!parent->children) {
+				/* it's the first child */
+				parent->children = ps;
+			} else {
+				/* walk all children and append */
+				struct ps_struct *children;
+				children = parent->children;
+				while (children->next)
+					children = children->next;
+				children->next = ps;
+			}
+		}
+
+		/* else -> found pid, append data in ps */
+
+		/* below here is all continuous logging parts - we get here on every
+		 * iteration */
+
+		/* rt, wt */
+		if (!ps->schedstat) {
+			sprintf(filename, "/proc/%d/schedstat", pid);
+			ps->schedstat = open(filename, O_RDONLY);
+			if (ps->schedstat == -1)
+				continue;
+		}
+
+		if (pread(ps->schedstat, buf, sizeof(buf) - 1, 0) <= 0) {
+			/* clean up our file descriptors - assume that the process exited */
+			close(ps->schedstat);
+			if (ps->sched)
+				close(ps->sched);
+			//if (ps->smaps)
+			//	fclose(ps->smaps);
+			continue;
+		}
+		if (!sscanf(buf, "%s %s %*s", rt, wt))
+			continue;
+
+		ps->last = sample;
+		ps->sample[sample].runtime = atoll(rt);
+		ps->sample[sample].waittime = atoll(wt);
+
+		ps->total = (ps->sample[ps->last].runtime
+				 - ps->sample[ps->first].runtime)
+				 / 1000000000.0;
+
+		if (!pss)
+			goto catch_rename;
+		/* Pss */
+		if (!ps->smaps) {
+			sprintf(filename, "/proc/%d/smaps", pid);
+			ps->smaps = fopen(filename, "r");
+			setvbuf(ps->smaps, smaps_buf, _IOFBF, sizeof(smaps_buf));
+			if (!ps->smaps)
+				continue;
+		} else {
+			rewind(ps->smaps);
+		}
+
+		while (1) {
+			int pss_kb;
+
+			/* skip one line, this contains the object mapped */
+			if (fgets(buf, sizeof(buf), ps->smaps) == NULL)
+				break;
+			/* then there's a 28 char 14 line block */
+			if (fread(buf, 1, 28 * 14, ps->smaps) != 28 * 14)
+				break;
+
+			pss_kb = atoi(&buf[61]);
+			ps->sample[sample].pss += pss_kb;
+		}
+
+		if (ps->sample[sample].pss > ps->pss_max)
+			ps->pss_max = ps->sample[sample].pss;
+
+catch_rename:
+		/* catch process rename, try to randomize time */
+		mod = (hz < 4.0) ? 4.0 : (hz / 4.0);
+		if (((samples - ps->first) + pid) % (int)(mod) == 0) {
+
+			/* re-fetch name */
+			/* get name, start time */
+			if (!ps->sched) {
+				sprintf(filename, "/proc/%d/sched", pid);
+				ps->sched = open(filename, O_RDONLY);
+				if (ps->sched == -1)
+					continue;
+			}
+			if (pread(ps->sched, buf, sizeof(buf) - 1, 0) <= 0) {
+				/* clean up file descriptors */
+				close(ps->sched);
+				if (ps->schedstat)
+					close(ps->schedstat);
+				//if (ps->smaps)
+				//	fclose(ps->smaps);
+				continue;
+			}
+
+			if (!sscanf(buf, "%s %*s %*s", key))
+				continue;
+
+			strncpy(ps->name, key, 16);
+		}
+	}
+}