Merge pull request #204 from kanoi/icarus-timer

Icarus: high accuracy timing and other bitstream speed support

FPGA-README

@@ -0,0 +1,56 @@
+
+This README contains extended details about FPGA mining with cgminer
+
+
+Icarus
+
+There is a hidden option in cgminer when Icarus support is compiled in:
+
+--icarus-timing <arg> Set how the Icarus timing is calculated - one setting/value for all or comma separated
+           default[=N]   Use the default Icarus hash time (2.6316ns)
+           short         Calculate the hash time and stop adjusting it at ~315 difficulty 1 shares (~1hr)
+           long          Re-calculate the hash time continuously
+           value[=N]     Specify the hash time in nanoseconds (e.g. 2.6316) and abort time (e.g. 2.6316=80)
+
+     Icarus timing is required for devices that do not exactly match a default Icarus Rev3 in
+     processing speed
+     If you have an Icarus Rev3 you should not normally need to use --icarus-timing since the
+     default values will maximise the MH/s and display it correctly
+
+     Icarus timing is used to determine the number of hashes that have been checked when it aborts
+     a nonce range (including on a LongPoll)
+     It is also used to determine the elapsed time when it should abort a nonce range to avoid
+     letting the Icarus go idle, but also to safely maximise that time
+
+     'short' or 'long' mode should only be used on a computer that has enough CPU available to run
+     cgminer without any CPU delays (an active desktop or swapping computer would not be stable enough)
+     Any CPU delays while calculating the hash time will affect the result
+     'short' mode only requires the computer to be stable until it has completed ~315 difficulty 1 shares
+     'long' mode requires it to always be stable to ensure accuracy, however, over time it continually
+     corrects itself
+
+     When in 'short' or 'long' mode, it will report the hash time value each time it is re-calculated
+     In 'short' or 'long' mode, the scan abort time starts at 5 seconds and uses the default 2.6316ns
+     scan hash time, for the first 5 nonce's or one minute (whichever is longer)
+
+     In 'default' or 'value' mode the 'constants' are calculated once at the start, based on the default
+     value or the value specified
+     The optional additional =N specifies to set the default abort at N 1/10ths of a second, not the
+     calculated value, which is 112 for 2.6316ns
+
+     To determine the hash time value for a non Icarus Rev3 device or an Icarus Rev3 with a different
+     bitstream to the default one, use 'long' mode and give it at least a few hundred shares, or use
+     'short' mode and take note of the final hash time value (Hs) calculated
+     You can also use the RPC API 'stats' command to see the current hash time (Hs) at any time
+
+     The Icarus code currently only works with a dual FPGA device that supports the same commands as
+     Icarus Rev3 requires and also is less than ~840MH/s and greater than 2MH/s
+     If a dual FPGA device does hash faster than ~840MH/s it should work correctly if you supply the
+     correct hash time nanoseconds value
+
+     The timing code itself will affect the Icarus performance since it increases the delay after
+     work is completed or aborted until it starts again
+     The increase is, however, extremely small and the actual increase is reported with the
+     RPC API 'stats' command (a very slow CPU will make it more noticeable)
+     Using the 'short' mode will remove this delay after 'short' mode completes
+     The delay doesn't affect the calculation of the correct hash time

README

@@ -209,6 +209,8 @@ FPGA mining boards(BitForce, Icarus, Ztex) only options:
      On windows <arg> is usually of the format \\.\COMn
        (where n = the correct device number for the FPGA device)
 
+For other FPGA details see the FPGA-README
+
 
 CPU only options (deprecated, not included in binaries!):
 

cgminer.c

@@ -134,6 +134,7 @@ bool opt_api_listen;
 bool opt_api_network;
 bool opt_delaynet;
 bool opt_disable_pool = true;
+char *opt_icarus_timing = NULL;
 
 char *opt_kernel_path;
 char *cgminer_path;
@@ -678,6 +679,15 @@ static char *set_api_description(const char *arg)
 	return NULL;
 }
 
+#ifdef USE_ICARUS
+static char *set_icarus_timing(const char *arg)
+{
+	opt_set_charp(arg, &opt_icarus_timing);
+
+	return NULL;
+}
+#endif
+
 /* These options are available from config file or commandline */
 static struct opt_table opt_config_table[] = {
 #ifdef WANT_CPUMINE
@@ -814,6 +824,11 @@ static struct opt_table opt_config_table[] = {
 	OPT_WITH_ARG("--kernel|-k",
 		     set_kernel, NULL, NULL,
 		     "Override kernel to use (diablo, poclbm, phatk or diakgcn) - one value or comma separated"),
+#endif
+#ifdef USE_ICARUS
+	OPT_WITH_ARG("--icarus-timing",
+		     set_icarus_timing, NULL, NULL,
+		     opt_hidden),
 #endif
 	OPT_WITHOUT_ARG("--load-balance",
 		     set_loadbalance, &pool_strategy,
@@ -2769,6 +2784,8 @@ void write_config(FILE *fcfg)
 		fprintf(fcfg, ",\n\"api-allow\" : \"%s\"", opt_api_allow);
 	if (strcmp(opt_api_description, PACKAGE_STRING) != 0)
 		fprintf(fcfg, ",\n\"api-description\" : \"%s\"", opt_api_description);
+	if (opt_icarus_timing)
+		fprintf(fcfg, ",\n\"icarus-timing\" : \"%s\"", opt_icarus_timing);
 	fputs("\n}", fcfg);
 }
 

driver-icarus.c

@@ -51,17 +51,136 @@
 #include "elist.h"
 #include "miner.h"
 
-// This is valid for a standard Icarus Rev 3
-// Assuming each hash pair takes 5.26ns then a whole nonce range would take 11.3s
-// Giving a little leaway 11.1s would be best
-//#define ICARUS_READ_COUNT_DEFAULT	111
-#define ICARUS_READ_COUNT_DEFAULT	80
+// The serial I/O speed - Linux uses a define 'B115200' in bits/termios.h
+#define ICARUS_IO_SPEED 115200
 
-// 2 x 11.1 / (5.26 x 10^-9)
-//#define ESTIMATE_HASHES	0xFB90365E
+// The size of a successful nonce read
+#define ICARUS_READ_SIZE 4
 
-// This is the 8s value
-#define ESTIMATE_HASHES	0xB54E9147
+// Ensure the sizes are correct for the Serial read
+#if (ICARUS_READ_SIZE != 4)
+#error ICARUS_READ_SIZE must be 4
+#endif
+#define ASSERT1(condition) __maybe_unused static char sizeof_uint32_t_must_be_4[(condition)?1:-1]
+ASSERT1(sizeof(uint32_t) == 4);
+
+#define ICARUS_READ_TIME ((double)ICARUS_READ_SIZE * (double)8.0 / (double)ICARUS_IO_SPEED)
+
+// Fraction of a second, USB timeout is measured in
+// i.e. 10 means 1/10 of a second
+#define TIME_FACTOR 10
+// In Linux it's 10 per second, thus value = 10/TIME_FACTOR =
+#define LINUX_TIMEOUT_VALUE 1
+// In Windows it's 1000 per second, thus value = 1000/TIME_FACTOR =
+#define WINDOWS_TIMEOUT_VALUE 100
+
+// In timing mode: Default starting value until an estimate can be obtained
+// 5 seconds allows for up to a ~840MH/s device
+#define ICARUS_READ_COUNT_TIMING	(5 * TIME_FACTOR)
+
+// For a standard Icarus REV3 (to 5 places)
+// Since this rounds up a the last digit - it is a slight overestimate
+// Thus the hash rate will be a VERY slight underestimate
+// (by a lot less than the displayed accuracy)
+#define ICARUS_REV3_HASH_TIME 0.0000000026316
+#define NANOSEC 1000000000.0
+
+// Icarus Rev3 doesn't send a completion message when it finishes
+// the full nonce range, so to avoid being idle we must abort the
+// work (by starting a new work) shortly before it finishes
+//
+// Thus we need to estimate 2 things:
+//	1) How many hashes were done if the work was aborted
+//	2) How high can the timeout be before the Icarus is idle,
+//		to minimise the number of work started
+//	We set 2) to 'the calculated estimate' - 1
+//	to ensure the estimate ends before idle
+//
+// The simple calculation used is:
+//	Tn = Total time in seconds to calculate n hashes
+//	Hs = seconds per hash
+//	Xn = number of hashes
+//	W  = code overhead per work
+//
+// Rough but reasonable estimate:
+//	Tn = Hs * Xn + W	(of the form y = mx + b)
+//
+// Thus:
+//	Line of best fit (using least squares)
+//
+//	Hs = (n*Sum(XiTi)-Sum(Xi)*Sum(Ti))/(n*Sum(Xi^2)-Sum(Xi)^2)
+//	W = Sum(Ti)/n - (Hs*Sum(Xi))/n
+//
+// N.B. W is less when aborting work since we aren't waiting for the reply
+//	to be transferred back (ICARUS_READ_TIME)
+//	Calculating the hashes aborted at n seconds is thus just n/Hs
+//	(though this is still a slight overestimate due to code delays)
+//
+
+// Both below must be exceeded to complete a set of data
+// Minimum how long after the first, the last data point must be
+#define HISTORY_SEC 60
+// Minimum how many points a single ICARUS_HISTORY should have
+#define MIN_DATA_COUNT 5
+// The value above used is doubled each history until it exceeds:
+#define MAX_MIN_DATA_COUNT 100
+
+static struct timeval history_sec = { HISTORY_SEC, 0 };
+
+// Store the last INFO_HISTORY data sets
+// [0] = current data, not yet ready to be included as an estimate
+// Each new data set throws the last old set off the end thus
+// keeping a ongoing average of recent data
+#define INFO_HISTORY 10
+
+struct ICARUS_HISTORY {
+	struct timeval finish;
+	double sumXiTi;
+	double sumXi;
+	double sumTi;
+	double sumXi2;
+	uint32_t values;
+	uint32_t hash_count_min;
+	uint32_t hash_count_max;
+};
+
+enum timing_mode { MODE_DEFAULT, MODE_SHORT, MODE_LONG, MODE_VALUE };
+
+static const char *MODE_DEFAULT_STR = "default";
+static const char *MODE_SHORT_STR = "short";
+static const char *MODE_LONG_STR = "long";
+static const char *MODE_VALUE_STR = "value";
+static const char *MODE_UNKNOWN_STR = "unknown";
+
+struct ICARUS_INFO {
+	// time to calculate the golden_ob
+	uint64_t golden_hashes;
+	struct timeval golden_tv;
+
+	struct ICARUS_HISTORY history[INFO_HISTORY+1];
+	uint32_t min_data_count;
+
+	// seconds per Hash
+	double Hs;
+	int read_count;
+
+	enum timing_mode timing_mode;
+	bool do_icarus_timing;
+
+	double fullnonce;
+	int count;
+	double W;
+	uint32_t values;
+	uint64_t hash_count_range;
+
+	// Determine the cost of history processing
+	// (which will only affect W)
+	uint64_t history_count;
+	struct timeval history_time;
+};
+
+// One for each possible device
+static struct ICARUS_INFO *icarus_info[MAX_DEVICES];
 
 struct device_api icarus_api;
 
@@ -98,7 +217,7 @@ static int icarus_open(const char *devpath)
 				ISTRIP | INLCR | IGNCR | ICRNL | IXON);
 	my_termios.c_oflag &= ~OPOST;
 	my_termios.c_lflag &= ~(ECHO | ECHONL | ICANON | ISIG | IEXTEN);
-	my_termios.c_cc[VTIME] = 1; /* block 0.1 second */
+	my_termios.c_cc[VTIME] = LINUX_TIMEOUT_VALUE; /* how long to block */
 	my_termios.c_cc[VMIN] = 0;
 	tcsetattr(serialfd, TCSANOW, &my_termios);
 
@@ -119,7 +238,7 @@ static int icarus_open(const char *devpath)
 	comCfg.dwSize = sizeof(COMMCONFIG);
 	comCfg.wVersion = 1;
 	comCfg.dcb.DCBlength = sizeof(DCB);
-	comCfg.dcb.BaudRate = 115200;
+	comCfg.dcb.BaudRate = ICARUS_IO_SPEED;
 	comCfg.dcb.fBinary = 1;
 	comCfg.dcb.fDtrControl = DTR_CONTROL_ENABLE;
 	comCfg.dcb.fRtsControl = RTS_CONTROL_ENABLE;
@@ -127,42 +246,57 @@ static int icarus_open(const char *devpath)
 
 	SetCommConfig(hSerial, &comCfg, sizeof(comCfg));
 
-	// block 0.1 second
-	COMMTIMEOUTS cto = {100, 0, 100, 0, 100};
+	// How long to block
+	COMMTIMEOUTS cto = {WINDOWS_TIMEOUT_VALUE, 0, WINDOWS_TIMEOUT_VALUE, 0, WINDOWS_TIMEOUT_VALUE};
 	SetCommTimeouts(hSerial, &cto);
 
 	return _open_osfhandle((LONG)hSerial, 0);
 #endif
 }
 
-static int icarus_gets(unsigned char *buf, size_t bufLen, int fd, int thr_id, int read_count)
+static int icarus_gets(unsigned char *buf, int fd, struct timeval *tv_finish, int thr_id, int read_count)
 {
 	ssize_t ret = 0;
 	int rc = 0;
+	int read_amount = ICARUS_READ_SIZE;
+	bool first = true;
 
-	while (bufLen) {
+	// Read reply 1 byte at a time to get earliest tv_finish
+	while (true) {
 		ret = read(fd, buf, 1);
-		if (ret == 1) {
-			bufLen--;
-			buf++;
+
+		if (first)
+			gettimeofday(tv_finish, NULL);
+
+		if (ret >= read_amount)
+			return 0;
+
+		if (ret > 0) {
+			buf += ret;
+			read_amount -= ret;
+			first = false;
 			continue;
 		}
-
+			
 		rc++;
 		if (rc >= read_count) {
-			applog(LOG_DEBUG,
-			       "Icarus Read: No data in %.2f seconds", (float)(rc/10.0f));
+			if (opt_debug) {
+				applog(LOG_DEBUG,
+					"Icarus Read: No data in %.2f seconds",
+					(float)rc/(float)TIME_FACTOR);
+			}
 			return 1;
 		}
 
 		if (thr_id >= 0 && work_restart[thr_id].restart) {
-			applog(LOG_DEBUG,
-			       "Icarus Read: Work restart at %.2f seconds", (float)(rc/10.0f));
+			if (opt_debug) {
+				applog(LOG_DEBUG,
+					"Icarus Read: Work restart at %.2f seconds",
+					(float)(rc)/(float)TIME_FACTOR);
+			}
 			return 1;
 		}
 	}
-
-	return 0;
 }
 
 static int icarus_write(int fd, const void *buf, size_t bufLen)
@@ -178,8 +312,110 @@ static int icarus_write(int fd, const void *buf, size_t bufLen)
 
 #define icarus_close(fd) close(fd)
 
+static const char *timing_mode_str(enum timing_mode timing_mode)
+{
+	switch(timing_mode) {
+	case MODE_DEFAULT:
+		return MODE_DEFAULT_STR;
+	case MODE_SHORT:
+		return MODE_SHORT_STR;
+	case MODE_LONG:
+		return MODE_LONG_STR;
+	case MODE_VALUE:
+		return MODE_VALUE_STR;
+	default:
+		return MODE_UNKNOWN_STR;
+	}
+}
+
+static void set_timing_mode(struct cgpu_info *icarus)
+{
+	struct ICARUS_INFO *info = icarus_info[icarus->device_id];
+	double Hs;
+	char buf[BUFSIZ+1];
+	char *ptr, *comma, *eq;
+	size_t max;
+	int i;
+
+	if (opt_icarus_timing == NULL)
+		buf[0] = '\0';
+	else {
+		ptr = opt_icarus_timing;
+		for (i = 0; i < icarus->device_id; i++) {
+			comma = strchr(ptr, ',');
+			if (comma == NULL)
+				break;
+			ptr = comma + 1;
+		}
+
+		comma = strchr(ptr, ',');
+		if (comma == NULL)
+			max = strlen(ptr);
+		else
+			max = comma - ptr;
+
+		if (max > BUFSIZ)
+			max = BUFSIZ;
+		strncpy(buf, ptr, max);
+		buf[max] = '\0';
+	}
+
+	info->Hs = 0;
+	info->read_count = 0;
+
+	if (strcasecmp(buf, MODE_SHORT_STR) == 0) {
+		info->Hs = ICARUS_REV3_HASH_TIME;
+		info->read_count = ICARUS_READ_COUNT_TIMING;
+
+		info->timing_mode = MODE_SHORT;
+		info->do_icarus_timing = true;
+	} else if (strcasecmp(buf, MODE_LONG_STR) == 0) {
+		info->Hs = ICARUS_REV3_HASH_TIME;
+		info->read_count = ICARUS_READ_COUNT_TIMING;
+
+		info->timing_mode = MODE_LONG;
+		info->do_icarus_timing = true;
+	} else if ((Hs = atof(buf)) != 0) {
+		info->Hs = Hs / NANOSEC;
+		info->fullnonce = info->Hs * (((double)0xffffffff) + 1);
+
+		if ((eq = strchr(buf, '=')) != NULL)
+			info->read_count = atoi(eq+1);
+
+		if (info->read_count < 1)
+			info->read_count = (int)(info->fullnonce * TIME_FACTOR) - 1;
+
+		if (unlikely(info->read_count < 1))
+			info->read_count = 1;
+
+		info->timing_mode = MODE_VALUE;
+		info->do_icarus_timing = false;
+	} else {
+		// Anything else in buf just uses DEFAULT mode
+
+		info->Hs = ICARUS_REV3_HASH_TIME;
+		info->fullnonce = info->Hs * (((double)0xffffffff) + 1);
+
+		if ((eq = strchr(buf, '=')) != NULL)
+			info->read_count = atoi(eq+1);
+
+		if (info->read_count < 1)
+			info->read_count = (int)(info->fullnonce * TIME_FACTOR) - 1;
+
+		info->timing_mode = MODE_DEFAULT;
+		info->do_icarus_timing = false;
+	}
+
+	info->min_data_count = MIN_DATA_COUNT;
+
+	applog(LOG_DEBUG, "Icarus: Init: %d mode=%s read_count=%d Hs=%e",
+		icarus->device_id, timing_mode_str(info->timing_mode), info->read_count, info->Hs);
+
+}
+
 static bool icarus_detect_one(const char *devpath)
 {
+	struct ICARUS_INFO *info;
 	struct timeval tv_start, tv_finish;
 	int fd;
 
@@ -194,8 +430,9 @@ static bool icarus_detect_one(const char *devpath)
 		"0000000087320b1a1426674f2fa722ce";
 
 	const char golden_nonce[] = "000187a2";
+	const uint32_t golden_nonce_val = 0x000187a2;
 
-	unsigned char ob_bin[64], nonce_bin[4];
+	unsigned char ob_bin[64], nonce_bin[ICARUS_READ_SIZE];
 	char *nonce_hex;
 
 	if (total_devices == MAX_DEVICES)
@@ -212,8 +449,7 @@ static bool icarus_detect_one(const char *devpath)
 	gettimeofday(&tv_start, NULL);
 
 	memset(nonce_bin, 0, sizeof(nonce_bin));
-	icarus_gets(nonce_bin, sizeof(nonce_bin), fd, -1, 1);
-	gettimeofday(&tv_finish, NULL);
+	icarus_gets(nonce_bin, fd, &tv_finish, -1, 1);
 
 	icarus_close(fd);
 
@@ -221,16 +457,16 @@ static bool icarus_detect_one(const char *devpath)
 	if (nonce_hex) {
 		if (strncmp(nonce_hex, golden_nonce, 8)) {
 			applog(LOG_ERR, 
-			       "Icarus Detect: "
-			       "Test failed at %s: get %s, should: %s",
-			       devpath, nonce_hex, golden_nonce);
+				"Icarus Detect: "
+				"Test failed at %s: get %s, should: %s",
+				devpath, nonce_hex, golden_nonce);
 			free(nonce_hex);
 			return false;
 		}
 		applog(LOG_DEBUG, 
-		       "Icarus Detect: "
-		       "Test succeeded at %s: got %s",
-			       devpath, nonce_hex);
+			"Icarus Detect: "
+			"Test succeeded at %s: got %s",
+				devpath, nonce_hex);
 		free(nonce_hex);
 	} else
 		return false;
@@ -244,7 +480,23 @@ static bool icarus_detect_one(const char *devpath)
 	add_cgpu(icarus);
 
 	applog(LOG_INFO, "Found Icarus at %s, mark as %d",
-	       devpath, icarus->device_id);
+		devpath, icarus->device_id);
+
+	if (icarus_info[icarus->device_id] == NULL) {
+		icarus_info[icarus->device_id] = (struct ICARUS_INFO *)malloc(sizeof(struct ICARUS_INFO));
+		if (unlikely(!(icarus_info[icarus->device_id])))
+			quit(1, "Failed to malloc ICARUS_INFO");
+	}
+
+	info = icarus_info[icarus->device_id];
+
+	// Initialise everything to zero for a new device
+	memset(info, 0, sizeof(struct ICARUS_INFO));
+
+	info->golden_hashes = (golden_nonce_val & 0x7fffffff) << 1;
+	timersub(&tv_finish, &tv_start, &(info->golden_tv));
+
+	set_timing_mode(icarus);
 
 	return true;
 }
@@ -295,11 +547,24 @@ static uint64_t icarus_scanhash(struct thr_info *thr, struct work *work,
 	int fd;
 	int ret;
 
-	unsigned char ob_bin[64], nonce_bin[4];
-	char *ob_hex, *nonce_hex;
+	struct ICARUS_INFO *info;
+
+	unsigned char ob_bin[64], nonce_bin[ICARUS_READ_SIZE];
+	char *ob_hex;
 	uint32_t nonce;
-	uint32_t hash_count;
+	uint64_t hash_count;
 	struct timeval tv_start, tv_finish, elapsed;
+	struct timeval tv_history_start, tv_history_finish;
+	double Ti, Xi;
+	int i;
+
+	struct ICARUS_HISTORY *history0, *history;
+	int count;
+	double Hs, W, fullnonce;
+	int read_count;
+	uint64_t estimate_hashes;
+	uint32_t values;
+	uint64_t hash_count_range;
 
 	elapsed.tv_sec = elapsed.tv_usec = 0;
 
@@ -315,39 +580,49 @@ static uint64_t icarus_scanhash(struct thr_info *thr, struct work *work,
 	tcflush(fd, TCOFLUSH);
 #endif
 	ret = icarus_write(fd, ob_bin, sizeof(ob_bin));
-	if (opt_debug)
-		gettimeofday(&tv_start, NULL);
 	if (ret)
 		return 0;	/* This should never happen */
 
+	gettimeofday(&tv_start, NULL);
+
 	if (opt_debug) {
 		ob_hex = bin2hex(ob_bin, sizeof(ob_bin));
 		if (ob_hex) {
 			applog(LOG_DEBUG, "Icarus %d sent: %s",
-			       icarus->device_id, ob_hex);
+				icarus->device_id, ob_hex);
 			free(ob_hex);
 		}
 	}
 
-	/* Icarus will return 4 bytes nonces or nothing */
+	/* Icarus will return 4 bytes (ICARUS_READ_SIZE) nonces or nothing */
 	memset(nonce_bin, 0, sizeof(nonce_bin));
-	ret = icarus_gets(nonce_bin, sizeof(nonce_bin), fd, thr_id,
-						ICARUS_READ_COUNT_DEFAULT);
-
-	if (opt_debug)
-		gettimeofday(&tv_finish, NULL);
+	info = icarus_info[icarus->device_id];
+	ret = icarus_gets(nonce_bin, fd, &tv_finish, thr_id, info->read_count);
 
 	work->blk.nonce = 0xffffffff;
 	memcpy((char *)&nonce, nonce_bin, sizeof(nonce_bin));
 
 	// aborted before becoming idle, get new work
-        if (nonce == 0 && ret) {
+	if (nonce == 0 && ret) {
+		timersub(&tv_finish, &tv_start, &elapsed);
+
+		// ONLY up to just when it aborted
+		// We didn't read a reply so we don't subtract ICARUS_READ_TIME
+		estimate_hashes = ((double)(elapsed.tv_sec)
+					+ ((double)(elapsed.tv_usec))/((double)1000000)) / info->Hs;
+
+		// If some Serial-USB delay allowed the full nonce range to
+		// complete it can't have done more than a full nonce
+		if (unlikely(estimate_hashes > 0xffffffff))
+			estimate_hashes = 0xffffffff;
+
 		if (opt_debug) {
-			timersub(&tv_finish, &tv_start, &elapsed);
-			applog(LOG_DEBUG, "Icarus %d no nonce = 0x%08x hashes (%ld.%06lds)",
-				icarus->device_id, ESTIMATE_HASHES, elapsed.tv_sec, elapsed.tv_usec);
+			applog(LOG_DEBUG, "Icarus %d no nonce = 0x%08llx hashes (%ld.%06lds)",
+					icarus->device_id, estimate_hashes,
+					elapsed.tv_sec, elapsed.tv_usec);
 		}
-		return ESTIMATE_HASHES;
+
+		return estimate_hashes;
 	}
 
 #if !defined (__BIG_ENDIAN__) && !defined(MIPSEB)
@@ -356,28 +631,136 @@ static uint64_t icarus_scanhash(struct thr_info *thr, struct work *work,
 
 	submit_nonce(thr, work, nonce);
 
-	if (opt_debug) {
-		timersub(&tv_finish, &tv_start, &elapsed);
-
-		nonce_hex = bin2hex(nonce_bin, sizeof(nonce_bin));
-		if (nonce_hex) {
-			applog(LOG_DEBUG, "Icarus %d returned (elapsed %ld.%06ld seconds): %s",
-			       icarus->device_id, elapsed.tv_sec, elapsed.tv_usec, nonce_hex);
-			free(nonce_hex);
-		}
-	}
-
 	hash_count = (nonce & 0x7fffffff);
 	if (hash_count++ == 0x7fffffff)
 		hash_count = 0xffffffff;
 	else
 		hash_count <<= 1;
 
-	if (opt_debug)
-		applog(LOG_DEBUG, "Icarus %d nonce = 0x%08x = 0x%08x hashes (%ld.%06lds)",
+	if (opt_debug || info->do_icarus_timing)
+		timersub(&tv_finish, &tv_start, &elapsed);
+
+	if (opt_debug) {
+		applog(LOG_DEBUG, "Icarus %d nonce = 0x%08x = 0x%08llx hashes (%ld.%06lds)",
 				icarus->device_id, nonce, hash_count, elapsed.tv_sec, elapsed.tv_usec);
+	}
 
-        return hash_count;
+	// ignore possible end condition values
+	if (info->do_icarus_timing && (nonce & 0x7fffffff) > 0x000fffff && (nonce & 0x7fffffff) < 0x7ff00000) {
+		gettimeofday(&tv_history_start, NULL);
+
+		history0 = &(info->history[0]);
+
+		if (history0->values == 0)
+			timeradd(&tv_start, &history_sec, &(history0->finish));
+
+		Ti = (double)(elapsed.tv_sec)
+			+ ((double)(elapsed.tv_usec))/((double)1000000)
+			- ICARUS_READ_TIME;
+		Xi = (double)hash_count;
+		history0->sumXiTi += Xi * Ti;
+		history0->sumXi += Xi;
+		history0->sumTi += Ti;
+		history0->sumXi2 += Xi * Xi;
+
+		history0->values++;
+
+		if (history0->hash_count_max < hash_count)
+			history0->hash_count_max = hash_count;
+		if (history0->hash_count_min > hash_count || history0->hash_count_min == 0)
+			history0->hash_count_min = hash_count;
+
+		if (history0->values >= info->min_data_count
+		&&  timercmp(&tv_start, &(history0->finish), >)) {
+			for (i = INFO_HISTORY; i > 0; i--)
+				memcpy(&(info->history[i]),
+					&(info->history[i-1]),
+					sizeof(struct ICARUS_HISTORY));
+
+			// Initialise history0 to zero for summary calculation
+			memset(history0, 0, sizeof(struct ICARUS_HISTORY));
+
+			// We just completed a history data set
+			// So now recalc read_count based on the whole history thus we will
+			// initially get more accurate until it completes INFO_HISTORY
+			// total data sets
+			count = 0;
+			for (i = 1 ; i <= INFO_HISTORY; i++) {
+				history = &(info->history[i]);
+				if (history->values >= MIN_DATA_COUNT) {
+					count++;
+
+					history0->sumXiTi += history->sumXiTi;
+					history0->sumXi += history->sumXi;
+					history0->sumTi += history->sumTi;
+					history0->sumXi2 += history->sumXi2;
+					history0->values += history->values;
+
+					if (history0->hash_count_max < history->hash_count_max)
+						history0->hash_count_max = history->hash_count_max;
+					if (history0->hash_count_min > history->hash_count_min || history0->hash_count_min == 0)
+						history0->hash_count_min = history->hash_count_min;
+				}
+			}
+
+			// All history data
+			Hs = (history0->values*history0->sumXiTi - history0->sumXi*history0->sumTi)
+				/ (history0->values*history0->sumXi2 - history0->sumXi*history0->sumXi);
+			W = history0->sumTi/history0->values - Hs*history0->sumXi/history0->values;
+			hash_count_range = history0->hash_count_max - history0->hash_count_min;
+			values = history0->values;
+			
+			// Initialise history0 to zero for next data set
+			memset(history0, 0, sizeof(struct ICARUS_HISTORY));
+
+			fullnonce = W + Hs * (((double)0xffffffff) + 1);
+			read_count = (int)(fullnonce * TIME_FACTOR) - 1;
+
+			info->Hs = Hs;
+			info->read_count = read_count;
+
+			info->fullnonce = fullnonce;
+			info->count = count;
+			info->W = W;
+			info->values = values;
+			info->hash_count_range = hash_count_range;
+
+			if (info->min_data_count < MAX_MIN_DATA_COUNT)
+				info->min_data_count *= 2;
+			else if (info->timing_mode == MODE_SHORT)
+				info->do_icarus_timing = false;
+
+//			applog(LOG_WARNING, "Icarus %d Re-estimate: read_count=%d fullnonce=%fs history count=%d Hs=%e W=%e values=%d hash range=0x%08lx min data count=%u", icarus->device_id, read_count, fullnonce, count, Hs, W, values, hash_count_range, info->min_data_count);
+			applog(LOG_WARNING, "Icarus %d Re-estimate: Hs=%e W=%e read_count=%d fullnonce=%.3fs",
+					icarus->device_id, Hs, W, read_count, fullnonce);
+		}
+		info->history_count++;
+		gettimeofday(&tv_history_finish, NULL);
+
+		timersub(&tv_history_finish, &tv_history_start, &tv_history_finish);
+		timeradd(&tv_history_finish, &(info->history_time), &(info->history_time));
+	}
+
+	return hash_count;
+}
+
+static void icarus_api_stats(char *buf, struct cgpu_info *cgpu, bool isjson)
+{
+	struct ICARUS_INFO *info = icarus_info[cgpu->device_id];
+
+	// Warning, access to these is not locked - but we don't really
+	// care since hashing performance is way more important than
+	// locking access to displaying API debug 'stats'
+	sprintf(buf, isjson
+		? "\"read_count\":%d,\"fullnonce\":%f,\"count\":%d,\"Hs\":%.15f,\"W\":%f,\"total_values\":%u,\"range\":%ld,\"history_count\":%lu,\"history_time\":%f,\"min_data_count\":%u,\"timing_values\":%u"
+		: "read_count=%d,fullnonce=%f,count=%d,Hs=%.15f,W=%f,total_values=%u,range=%ld,history_count=%lu,history_time=%f,min_data_count=%u,timing_values=%u",
+		info->read_count, info->fullnonce,
+		info->count, info->Hs, info->W,
+		info->values, info->hash_count_range,
+		info->history_count,
+		(double)(info->history_time.tv_sec)
+			+ ((double)(info->history_time.tv_usec))/((double)1000000),
+		info->min_data_count, info->history[0].values);
 }
 
 static void icarus_shutdown(struct thr_info *thr)
@@ -390,6 +773,7 @@ struct device_api icarus_api = {
 	.dname = "icarus",
 	.name = "ICA",
 	.api_detect = icarus_detect,
+	.get_api_stats = icarus_api_stats,
 	.thread_prepare = icarus_prepare,
 	.scanhash = icarus_scanhash,
 	.thread_shutdown = icarus_shutdown,

miner.h

@@ -514,6 +514,7 @@ extern bool opt_api_listen;
 extern bool opt_api_network;
 extern bool opt_delaynet;
 extern bool opt_restart;
+extern char *opt_icarus_timing;
 
 extern pthread_rwlock_t netacc_lock;