/*
** CBLM v3
** $Id: data.c,v 1.35 2011/10/27 20:57:52 sella Exp $
** Copyright (c) 2004-2007 James M. Sella. All Rights Reserved.
** Released under the GPL Version 2 License.
** http://www.digitalgenesis.com
** 
** Cheap Bastard Latency Monitor v3
*/

static const char rcsid[] = "$Id: data.c,v 1.35 2011/10/27 20:57:52 sella Exp $";

#include "data.h"
#include "util.h"
#include "prefs.h"

#include <stdio.h>
#include <string.h>
#include <limits.h>
#include <mysql/mysql.h>

#define JITTER_VAR_AVG

/* Global Variables */
struct data_list_t *data_list = NULL;
struct data_list_t **dl_end = &data_list;
uint32_t d_count = 0;

/* External Global Variables */
extern struct args_t args;
extern struct prefs_t prefs;
extern pthread_mutex_t snmptrap_mutex, db_mutex, hash_mutex;
extern uint32_t probe_count;

//FIXME: Break each "test" into a seperate function.
void process_packet(struct packet_t *packet) {
	char buf[INET_ADDRSTRLEN];
	int32_t jtr;
	struct data_t d, *data;
	struct triptime_t *tt_tmp;
	uint64_t tx, usec, tt, last, last_warn, last_alarm;
	uint64_t now = time(NULL);

	pthread_mutex_lock(&hash_mutex);

	/* Build struct to search with */
	d.dst = packet->dst;
	d.tos = packet->tx_tos;
	d.pri = packet->tx_pri;

	if ((data = hash_find(&d)) == NULL) {
		/* Initialize memory. */
		memset(&d, 0, sizeof(struct data_t));
		d.src = packet->src;
		d.dst = packet->dst;
		d.tx = ntohs(packet->tot);
		d.set = packet->set;
		d.tos = packet->tx_tos;
		d.pri = packet->tx_pri;
		d.rtt_min = d.ott_min = d.jtr_min = INT_MAX;

		/* Setup data pointer. */
		data = &d;

		/* Insert into hash */
		hash_add(data);
	}

	/* Are we still on the same set? */
	if (packet->set + 1 == data->set) {
		if (prefs.trk_ooo == 1) {
			data->pkt_ooo++; /* Track out of order packet. */

			if (now > data->last_warn || args.debug >= DEBUG_DUMP) {
				data->last_warn = now + WARNING_DELAY;

				syslog(LOG_WARNING, "Recieved a packet from a previous set. Discarding packet from %s", inet_ntop(AF_INET, &(packet->dst), buf, sizeof(buf)));
				
			}
		}

		goto error;
	} else if (packet->set != data->set) {
		/* Finalize the data. */
		if (data->rx > 0) {
			data->ott_avg = data->ott_avg / data->rx;
			data->rtt_avg = data->rtt_avg / data->rx;

			data->ott_min = data->ott_min == INT_MAX ? 0 : data->ott_min;
			data->rtt_min = data->rtt_min == INT_MAX ? 0 : data->rtt_min;

#ifdef JITTER_VAR_AVG
			data->jtr_min = INT_MAX;
			data->jtr_max = data->jtr_avg = 0;
			while (data->tt != NULL) {
				if (abs(data->tt->rtt - data->rtt_avg) < data->jtr_min) {
					data->jtr_min = abs(data->tt->rtt - data->rtt_avg);
				}

				if (abs(data->tt->rtt - data->rtt_avg) > data->jtr_max) {
					data->jtr_max = abs(data->tt->rtt - data->rtt_avg);
				}

				data->jtr_avg += abs(data->tt->rtt - data->rtt_avg);

				/* Free triptime struct and move to next */
				tt_tmp = data->tt;
				data->tt = tt_tmp->next;
				free(tt_tmp);
			}
			data->jtr_avg = data->jtr_avg / data->rx;
#else
			data->jtr_avg = data->jtr_avg / data->rx;
			data->jtr_min = data->jtr_min == INT_MAX ? 0 : data->jtr_min;
#endif

			data->timestamp = time(NULL);

			/* Add data to data_list. */
			if (d_count > probe_count * QUEUE_DEPTH_MULT) {
				if (args.debug >= DEBUG_ERROR) {
					syslog(LOG_WARNING, "Maximum data queue depth reached. Tail dropping new entry for '%s'", inet_ntop(AF_INET, &(data->dst), buf, sizeof(buf)));
				}
			} else {
				*dl_end = malloc(sizeof(struct data_list_t));
				memcpy(&(*dl_end)->data, data, sizeof(struct data_t));

				(*dl_end)->next = NULL;
				dl_end = &(*dl_end)->next;
				d_count++;
			}
		}

		/* Save last warn/alarm */
		last_warn = data->last_warn;
		last_alarm = data->last_alarm;

		/* Reset data */
		memset(data, 0, sizeof(struct data_t));

		/* Restore last warn/alarm */
		data->last_warn = last_warn;
		data->last_alarm = last_alarm;

		/* Initialize the new data set. */
		data->src = packet->src;
		data->dst = packet->dst;
		data->tx = ntohs(packet->tot);
		data->set = packet->set;
		data->tos = packet->tx_tos;
		data->pri = packet->tx_pri;
		data->rtt_min = data->ott_min = data->jtr_min = INT_MAX;
	}

	/* Track out of order packets */
	if (prefs.trk_ooo == 1) {
		if (ntohs(packet->seq) < data->seq) {
			data->pkt_ooo++;

			if (now > data->last_warn || args.debug >= DEBUG_DUMP) {
				data->last_warn = now + WARNING_DELAY;

				syslog(LOG_WARNING, "Out of order packet received from %s (packet: %lu, data: %lu)", inet_ntop(AF_INET, &(packet->dst), buf, sizeof(buf)), packet->seq, data->seq);
			}
		}
	}

	/* Track duplicate packets */
	if (prefs.trk_dup == 1) {
		if (ntohs(packet->seq) == data->seq && packet->seq != 0) {
			data->pkt_dup++;

			if (now > data->last_warn || args.debug >= DEBUG_DUMP) {
				data->last_warn = now + WARNING_DELAY;

				syslog(LOG_WARNING, "Recieved a duplicate response from %s.", inet_ntop(AF_INET, &(packet->dst), buf, sizeof(buf)));
			}
		}
	}

	/* Recieved */
	data->rx++;
	data->seq = ntohs(packet->seq);

	/* Capture the tx time. */
	tx = ntohll(packet->tx);

	/* RTT */
	usec = ntohll(packet->rx);
	usec -= tx;
	data->rtt_lst = tt = last = usec;
	if (tt < MAX_RTT) {
		data->rtt_min = tt < data->rtt_min && tt > 0 ? tt : data->rtt_min;
		data->rtt_max = tt > data->rtt_max && tt > 0 ? tt : data->rtt_max;
		data->rtt_avg = tt > 0 ? data->rtt_avg + tt : data->rtt_avg;
	} else if (now > data->last_warn || args.debug >= DEBUG_DUMP) {
		data->last_warn = now + WARNING_DELAY;

		syslog(LOG_WARNING, "RTT to %s of %d (usec) is over max %d (usec). Discarding sample.", inet_ntop(AF_INET, &(packet->dst), buf, sizeof(buf)), tt, MAX_RTT);
	}

	/* OTT */
	usec = ntohll(packet->ox);
	usec -= tx;
	data->ott_lst = tt = usec;
	if (prefs.trk_ott == 1) {
		if (tt < last && tt < MAX_OTT) { /* OTT can't be greater than the RTT, or MAX_OTT. */
			data->ott_min = tt < data->ott_min && tt > 0 ? tt : data->ott_min;
			data->ott_max = tt > data->ott_max && tt > 0 ? tt : data->ott_max;
			data->ott_avg = tt > 0 ? data->ott_avg + tt : data->ott_avg;
		} else if (now > data->last_warn || args.debug >= DEBUG_DUMP) {
			data->last_warn = now + WARNING_DELAY;

			syslog(LOG_WARNING, "OTT to %s of %d (usec) is over max %d or RTT %d (usec). Discarding sample.", inet_ntop(AF_INET, &(packet->dst), buf, sizeof(buf)), tt, MAX_OTT, last);
		}
	}

	/* JTR (RTT) */
#ifdef JITTER_VAR_AVG
	if (prefs.trk_jtr == 1) {
		/* Insert new triptime struct */
		tt_tmp = (struct triptime_t*) malloc(sizeof(struct triptime_t));
		tt_tmp->next = data->tt;
		data->tt = tt_tmp;

		data->tt->rtt = data->rtt_lst;
		data->tt->ott = data->ott_lst; // Not used
	}
#else
	if (prefs.trk_jtr == 1) {
		if (data->ott_lst == 0) {
			data->ott_lst = tt;
		} else {
			jtr = abs(data->ott_lst - tt);
			if (jtr < MAX_JTR) {
				data->jtr_min = jtr < data->jtr_min ? jtr : data->jtr_min;
				data->jtr_max = jtr > data->jtr_max ? jtr : data->jtr_max;
				data->jtr_avg += jtr;
			} else if (now > data->last_warn || args.debug >= DEBUG_DUMP) {
				data->last_warn = now + WARNING_DELAY;

				syslog(LOG_WARNING, "Jitter to %s of %d (usec) is over max %d (usec). Discarding sample.\n", inet_ntop(AF_INET, &(packet->dst), buf, sizeof(buf)), jtr, MAX_JTR);
			}
		}
	}
#endif

	if (prefs.trk_tos == 1) {
		/* ToS A to Z error. */
		if (packet->tx_tos != packet->ox_tos) {
			data->tos_a2z++;

			if (now > data->last_warn || args.debug >= DEBUG_DUMP) {
				data->last_warn = now + WARNING_DELAY;

				syslog(LOG_WARNING, "ToS marking transmitted to %s changed from %u to %u (failed A to Z).\n", inet_ntop(AF_INET, &(packet->dst), buf, sizeof(buf)), packet->tx_tos, packet->ox_tos);
			}
		}

		/* ToS Z to A error. */
		if (packet->tx_tos != packet->rx_tos) {
			data->tos_z2a++;

			if (now > data->last_warn || args.debug >= DEBUG_DUMP) {
				data->last_warn = now + WARNING_DELAY;

				syslog(LOG_WARNING, "ToS marking transmitted from %s changed from %u to %u (failed Z to A).", inet_ntop(AF_INET, &(packet->dst), buf, sizeof(buf)), packet->tx_tos, packet->rx_tos);
			}
		}
	}

	if (prefs.trk_pri == 1) {
		/* Prioroty A to Z error. */
		if (packet->tx_pri != packet->ox_pri) {
			data->pri_a2z++;

			if (now > data->last_warn || args.debug >= DEBUG_DUMP) {
				data->last_warn = now + WARNING_DELAY;

				syslog(LOG_WARNING, "Priority marking transmitted to %s changed from %u to %u (failed A to Z).\n", inet_ntop(AF_INET, &(packet->dst), buf, sizeof(buf)), packet->tx_pri, packet->ox_pri);
			}
		}

		/* Priority Z to A error. */
		if (packet->tx_pri != packet->rx_pri) {
			data->pri_z2a++;

			if (now > data->last_warn || args.debug >= DEBUG_DUMP) {
				data->last_warn = now + WARNING_DELAY;

				syslog(LOG_WARNING, "Priority marking transmitted from %s changed from %u to %u (failed Z to A).", inet_ntop(AF_INET, &(packet->dst), buf, sizeof(buf)), packet->tx_pri, packet->rx_pri);
			}
		}
	}

	if (args.debug >= DEBUG_DUMP) {
		syslog(LOG_DEBUG, "Processed packet");
		display_data(data);
	}

error:
	pthread_mutex_unlock(&hash_mutex);
}

int write_data(struct data_t *data) {
	int found = 0, retval = 1;
	char sql[1024], src[INET_ADDRSTRLEN], dst[INET_ADDRSTRLEN], rtt[132], ott[132], jtr[132];
	MYSQL_RES* res;
	struct data_t d;

	assert(data != NULL);

   pthread_mutex_lock(&db_mutex);

	if (db_isconnected() != 1) {
		if (args.debug >= DEBUG_INTERNAL) {
			syslog(LOG_DEBUG, "Skipping write() - no database connection.");
		}

		goto exit;
	}

	inet_ntop(AF_INET, &data->src, src, sizeof(src));
	inet_ntop(AF_INET, &data->dst, dst, sizeof(dst));

	if (args.debug >= DEBUG_INTERNAL) {
		syslog(LOG_INFO, "Writting data for destination %s with ToS %u, PRI %u.", dst, data->tos, data->pri);
	}

	if (args.debug >= DEBUG_INTERNAL) {
		display_data(data);
	}

	found = is_probe(dst);

	/* Write data if destination is still a registered probe. */
	if (found != 1) {
		if (args.debug >= DEBUG_NOTICE) {
			syslog(LOG_NOTICE, "Discard data for %s. Not registered any longer or registration is too new.", dst);
		}
	} else {
		snprintf(sql, sizeof(sql), "INSERT INTO data (src, dst, timestamp, tos, pri, secs, tx, rx, rtt_min, rtt_max, rtt_avg, ott_min, ott_max, ott_avg, jtr_min, jtr_max, jtr_avg, tos_a2z, tos_z2a, pri_a2z, pri_z2a, pkt_ooo, pkt_dup) VALUES (INET_ATON('%s'), INET_ATON('%s'), SUBTIME(NOW(),SEC_TO_TIME(%llu)), %u, %u, %u, %u, %u, %d, %d, %llu, %d, %d, %llu, %d, %d, %llu, %u, %u, %u, %u, %u, %u)", src, dst, time(NULL) - data->timestamp, data->tos, data->pri, DATA_INTERVAL_MIN, data->tx, data->rx, data->rtt_min, data->rtt_max, data->rtt_avg, data->ott_min, data->ott_max, data->ott_avg, data->jtr_min, data->jtr_max, data->jtr_avg, data->tos_a2z, data->tos_z2a, data->pri_a2z, data->pri_z2a, data->pkt_ooo, data->pkt_dup);

		if (db_query(&res, sql) != 0) {
			syslog(LOG_CRIT, "write_data() failed to insert data.");

			goto exit;
		}
		mysql_free_result(res);
	}

	retval = 0;

exit:
   pthread_mutex_unlock(&db_mutex);

	return retval;
}

int summarize_data() {
	char sql[1024];
	int retval = 1;
	static uint64_t last_summary = 0;
	int64_t last_min = 0, last_hour = 0, last_day = 0;
	MYSQL_RES* res;
   MYSQL_ROW row;

	/* Don't try to summarize if it isn't time. */
	if (time(NULL) < last_summary) {
		return 0;
	}

	/* Allow time for all probes to write back data after a reconnect. */
	if (time(NULL) < DATA_ROLLUP_DELAY + db_last_connect()) {
		return 0;
	}

   pthread_mutex_lock(&db_mutex);

	if (db_isconnected() != 1) {
		if (args.debug >= DEBUG_INTERNAL) {
			syslog(LOG_DEBUG, "Skipping summarize_data() - no database connection.");
		}

		goto exit;
	}

	if (args.debug >= DEBUG_INTERNAL) {
		syslog(LOG_INFO, "Starting to summarize samples in database.");
	}

	/* Update time for next summary attempt. */
	last_summary = time(NULL) + SUMMARY_INTERVAL;

	/* Others may read, but may not write to data. */
	if (db_query(&res, "LOCK TABLES data WRITE, data AS data_src WRITE, state WRITE, prefs WRITE") != 0) {
		syslog(LOG_CRIT, "summarize_data() failed to lock data, state and prefs tables.");

		goto unlock;
	}
	mysql_free_result(res);

	/* Grab intervals update times. */
	if (db_query(&res, "SELECT var, val - UNIX_TIMESTAMP() FROM state WHERE var = 'last_min' OR var = 'last_hour' OR var = 'last_day'") != 0) {
		syslog(LOG_CRIT, "summarize_data() failed to select from state table.");

		goto unlock;
	}

	while (row = mysql_fetch_row(res)) {
		if (args.debug >= DEBUG_DUMP) {
			syslog(LOG_DEBUG, "read val: %s, var: %s", row[0], row[1]);
		}

		if (!strncmp(row[0], "last_min", 8)) {
			last_min = atoll(row[1]);
		} else if (!strncmp(row[0], "last_hour", 9)) {
			last_hour = atoll(row[1]);
		} else if (!strncmp(row[0], "last_day", 8)) {
			last_day = atoll(row[1]);
		}
	}
	mysql_free_result(res);

	/* Delete old 60 sec samples. */
	if (last_min < 1) {
		snprintf(sql, sizeof(sql), "DELETE FROM data WHERE secs = %lu AND UNIX_TIMESTAMP(timestamp) < UNIX_TIMESTAMP() - %lu", DATA_INTERVAL_MIN, DATA_INTERVAL_HOUR * DATA_MULTIPLIER);
		if (db_query(&res, sql) != 0) {
			syslog(LOG_CRIT, "summarize_data() failed to delete stale minute entries from data table.");

			goto unlock;
		}
		mysql_free_result(res);

		/* Delete old value, if any. */
		if (db_query(&res, "DELETE FROM state WHERE var = 'last_min'") != 0) {
			syslog(LOG_CRIT, "summarize_data() failed to delete 'last_min' from state table.");

			goto unlock;
		}
		mysql_free_result(res);

		/* Insert new time to summarize this interval. */
		snprintf(sql, sizeof(sql), "INSERT INTO state (var, val) VALUES('last_min', UNIX_TIMESTAMP() + %lu)", SUMMARY_INTERVAL);
		if (db_query(&res, sql) != 0) {
			syslog(LOG_CRIT, "summarize_data() failed to insert 'last_min' into state table.");

			goto unlock;
		}
		mysql_free_result(res);
	}

	/* Roll up 60 seconds samples into hour samples, and delete old hour samples. */
	if (last_hour < 1) {
		snprintf(sql, sizeof(sql), "INSERT INTO data (src, dst, timestamp, tos, pri, secs, tx, rx, rtt_min, rtt_max, rtt_avg, ott_min, ott_max, ott_avg, jtr_min, jtr_max, jtr_avg, tos_a2z, tos_z2a, pri_a2z, pri_z2a, pkt_ooo, pkt_dup) SELECT src, dst, MAX(timestamp), tos, pri, %lu, SUM(tx), SUM(rx), MIN(rtt_min), MAX(rtt_max), AVG(rtt_avg), MIN(ott_min), MAX(ott_max), AVG(ott_avg), MIN(jtr_min), MAX(jtr_max), AVG(jtr_avg), SUM(tos_a2z), SUM(tos_z2a), SUM(pri_a2z), SUM(pri_z2a), SUM(pkt_ooo), SUM(pkt_dup) FROM data AS data_src WHERE UNIX_TIMESTAMP(timestamp) > UNIX_TIMESTAMP() - %lu AND secs = %lu GROUP by src, dst, tos, pri ORDER by src, dst", DATA_INTERVAL_HOUR, DATA_INTERVAL_HOUR, DATA_INTERVAL_MIN);
		if (db_query(&res, sql) != 0) {
			syslog(LOG_CRIT, "summarize_data() failed to insert rolled up minute data into data table.");

			goto unlock;
		}
		mysql_free_result(res);

		snprintf(sql, sizeof(sql), "DELETE FROM data WHERE secs = %lu AND UNIX_TIMESTAMP(timestamp) < UNIX_TIMESTAMP() - %lu", DATA_INTERVAL_HOUR, DATA_INTERVAL_DAY * DATA_MULTIPLIER);
		if (db_query(&res, sql) != 0) {
			syslog(LOG_CRIT, "summarize_data() failed to delete stale hourly entries from data table.");

			goto unlock;
		}
		mysql_free_result(res);

		/* Delete old value, if any. */
		if (db_query(&res, "DELETE FROM state WHERE var = 'last_hour'") != 0) {
			syslog(LOG_CRIT, "summarize_data() failed to delete 'last_hour' from state table.");

			goto unlock;
		}
		mysql_free_result(res);

		/* Insert new time to summarize this interval. */
		snprintf(sql, sizeof(sql), "INSERT INTO state (var, val) VALUES('last_hour', UNIX_TIMESTAMP() + %lu)", DATA_INTERVAL_HOUR);
		if (db_query(&res, sql) != 0) {
			syslog(LOG_CRIT, "summarize_data() failed to insert 'last_hour' into state table.");

			goto unlock;
		}
		mysql_free_result(res);
	}

	/* Roll up 3600 seconds samples into day samples, and delete old day samples. */
	if (last_day < 1) {
		snprintf(sql, sizeof(sql), "INSERT INTO data (src, dst, timestamp, tos, pri, secs, tx, rx, rtt_min, rtt_max, rtt_avg, ott_min, ott_max, ott_avg, jtr_min, jtr_max, jtr_avg, tos_a2z, tos_z2a, pri_a2z, pri_z2a, pkt_ooo, pkt_dup) SELECT src, dst, MAX(timestamp), tos, pri, %lu, SUM(tx), SUM(rx), MIN(rtt_min), MAX(rtt_max), AVG(rtt_avg), MIN(ott_min), MAX(ott_max), AVG(ott_avg), MIN(jtr_min), MAX(jtr_max), AVG(jtr_avg), SUM(tos_a2z), SUM(tos_z2a), SUM(pri_a2z), SUM(pri_z2a), SUM(pkt_ooo), SUM(pkt_dup) FROM data AS data_src WHERE UNIX_TIMESTAMP(timestamp) > UNIX_TIMESTAMP() - %lu AND secs = %lu GROUP by src, dst, tos, pri ORDER by src, dst", DATA_INTERVAL_DAY, DATA_INTERVAL_DAY, DATA_INTERVAL_HOUR);
		if (db_query(&res, sql) != 0) {
			syslog(LOG_CRIT, "summarize_data() failed to insert rolled up hourly data into data table.");

			goto unlock;
		}
		mysql_free_result(res);

		snprintf(sql, sizeof(sql), "DELETE FROM data WHERE secs = %lu AND UNIX_TIMESTAMP(timestamp) < UNIX_TIMESTAMP() - %lu", DATA_INTERVAL_DAY, DATA_EXPIRATION);
		if (db_query(&res, sql) != 0) {
			syslog(LOG_CRIT, "summarize_data() failed to delete stale daily entries from data table.");

			goto unlock;
		}
		mysql_free_result(res);

		/* Delete old value, if any. */
		if (db_query(&res, "DELETE FROM state WHERE var = 'last_day'") != 0) {
			syslog(LOG_CRIT, "summarize_data() failed to delete 'last_day' from state table.");

			goto unlock;
		}
		mysql_free_result(res);

		/* Insert new time to summarize this interval. */
		snprintf(sql, sizeof(sql), "INSERT INTO state (var, val) VALUES('last_day', UNIX_TIMESTAMP() + %lu)", DATA_INTERVAL_DAY);
		if (db_query(&res, sql) != 0) {
			syslog(LOG_CRIT, "summarize_data() failed to insert 'last_day' into state table.");

			goto unlock;
		}
		mysql_free_result(res);

		if (db_query(&res, "UNLOCK TABLES") != 0) {
			syslog(LOG_CRIT, "summarize_data() failed to unlock data, state and prefs tables.");

			goto exit;
		}
		mysql_free_result(res);

		if (args.debug >= DEBUG_INTERNAL) {
			syslog(LOG_INFO, "Optimizing database tables.");
		}

		if (db_query(&res, "OPTIMIZE TABLE data") != 0) {
			syslog(LOG_CRIT, "summarize_data() failed to optimize data tables.");

			goto unlock;
		}
		mysql_free_result(res);
	}

	if (last_min < 1 || last_hour < 1 || last_day < 1) {
		/* Allow other probes to do next update. */
		last_summary += random() % (SUMMARY_INTERVAL / 8);
	}

unlock:
	if (db_query(&res, "UNLOCK TABLES") != 0) {
		syslog(LOG_CRIT, "summarize_data() failed to unlock data, state and prefs tables.");

		goto exit;
	}
	mysql_free_result(res);

	retval = 0;

exit:
   pthread_mutex_unlock(&db_mutex);

	if (args.debug >= DEBUG_INTERNAL) {
		syslog(LOG_INFO, "Completed summarization of samples in database. (Processed %s%s%s%s)", last_min < 1 ? "min " : "", last_hour < 1 ? "hour " : "", last_day < 1 ? "day " : "", last_min > 0 && last_hour > 0 && last_day > 0 ? "none" : "");
	}

	return retval;
}

/*
** Local Variables:
** c-basic-offset: 3
** tab-width: 3
** End:
** vim: noet ts=3 sw=3
*/
