More corrections to float printing
Testing suggests all f32s are now printed accurately.
This commit is contained in:
parent
03a0dfbeca
commit
262b7428cf
|
@ -32,7 +32,7 @@ pub fn errol3(value: f64, buffer: []u8) -> FloatDecimal {
|
||||||
fn errol3u(val: f64, buffer: []u8) -> FloatDecimal {
|
fn errol3u(val: f64, buffer: []u8) -> FloatDecimal {
|
||||||
// check if in integer or fixed range
|
// check if in integer or fixed range
|
||||||
|
|
||||||
if (val >= 9.007199254740992e15 and val < 3.40282366920938e+38) {
|
if (val > 9.007199254740992e15 and val < 3.40282366920938e+38) {
|
||||||
return errolInt(val, buffer);
|
return errolInt(val, buffer);
|
||||||
} else if (val >= 16.0 and val < 9.007199254740992e15) {
|
} else if (val >= 16.0 and val < 9.007199254740992e15) {
|
||||||
return errolFixed(val, buffer);
|
return errolFixed(val, buffer);
|
||||||
|
@ -235,7 +235,7 @@ fn hpMul10(hp: &HP) {
|
||||||
fn errolInt(val: f64, buffer: []u8) -> FloatDecimal {
|
fn errolInt(val: f64, buffer: []u8) -> FloatDecimal {
|
||||||
const pow19 = u128(1e19);
|
const pow19 = u128(1e19);
|
||||||
|
|
||||||
assert((val >= 9.007199254740992e15) and val < (3.40282366920938e38));
|
assert((val > 9.007199254740992e15) and val < (3.40282366920938e38));
|
||||||
|
|
||||||
var mid = u128(val);
|
var mid = u128(val);
|
||||||
var low: u128 = mid - fpeint((fpnext(val) - val) / 2.0);
|
var low: u128 = mid - fpeint((fpnext(val) - val) / 2.0);
|
||||||
|
@ -510,10 +510,6 @@ fn u64toa(value_param: u64, buffer: []u8) -> usize {
|
||||||
buf_index += 1;
|
buf_index += 1;
|
||||||
buffer[buf_index] = c_digits_lut[d8];
|
buffer[buf_index] = c_digits_lut[d8];
|
||||||
buf_index += 1;
|
buf_index += 1;
|
||||||
buffer[buf_index] = c_digits_lut[d8];
|
|
||||||
buf_index += 1;
|
|
||||||
buffer[buf_index] = c_digits_lut[d8];
|
|
||||||
buf_index += 1;
|
|
||||||
buffer[buf_index] = c_digits_lut[d8 + 1];
|
buffer[buf_index] = c_digits_lut[d8 + 1];
|
||||||
buf_index += 1;
|
buf_index += 1;
|
||||||
} else {
|
} else {
|
||||||
|
@ -613,7 +609,7 @@ fn fpeint(from: f64) -> u128 {
|
||||||
const bits = @bitCast(u64, from);
|
const bits = @bitCast(u64, from);
|
||||||
assert((bits & ((1 << 52) - 1)) == 0);
|
assert((bits & ((1 << 52) - 1)) == 0);
|
||||||
|
|
||||||
return u64(1) << u6(((bits >> 52) - 1023));
|
return u128(1) << @truncate(u7, (bits >> 52) -% 1023);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -250,20 +250,17 @@ pub fn formatFloat(value: var, context: var, output: fn(@typeOf(context), []cons
|
||||||
if (math.isNan(x)) {
|
if (math.isNan(x)) {
|
||||||
return output(context, "NaN");
|
return output(context, "NaN");
|
||||||
}
|
}
|
||||||
if (math.isPositiveInf(x)) {
|
if (math.signbit(x)) {
|
||||||
return output(context, "Infinity");
|
|
||||||
}
|
|
||||||
if (math.isNegativeInf(x)) {
|
|
||||||
return output(context, "-Infinity");
|
|
||||||
}
|
|
||||||
if (x == 0.0) {
|
|
||||||
return output(context, "0.0");
|
|
||||||
}
|
|
||||||
if (x < 0.0) {
|
|
||||||
if (!output(context, "-"))
|
if (!output(context, "-"))
|
||||||
return false;
|
return false;
|
||||||
x = -x;
|
x = -x;
|
||||||
}
|
}
|
||||||
|
if (math.isPositiveInf(x)) {
|
||||||
|
return output(context, "Infinity");
|
||||||
|
}
|
||||||
|
if (x == 0.0) {
|
||||||
|
return output(context, "0.0");
|
||||||
|
}
|
||||||
|
|
||||||
var buffer: [32]u8 = undefined;
|
var buffer: [32]u8 = undefined;
|
||||||
const float_decimal = errol3(x, buffer[0..]);
|
const float_decimal = errol3(x, buffer[0..]);
|
||||||
|
@ -272,8 +269,12 @@ pub fn formatFloat(value: var, context: var, output: fn(@typeOf(context), []cons
|
||||||
if (!output(context, "."))
|
if (!output(context, "."))
|
||||||
return false;
|
return false;
|
||||||
if (float_decimal.digits.len > 1) {
|
if (float_decimal.digits.len > 1) {
|
||||||
const num_digits = if (@typeOf(value) == f32) { usize(8) } else { usize(17) };
|
const num_digits = if (@typeOf(value) == f32) {
|
||||||
if (!output(context, float_decimal.digits[1 .. math.min(num_digits, float_decimal.digits.len)]))
|
math.min(usize(9), float_decimal.digits.len)
|
||||||
|
} else {
|
||||||
|
float_decimal.digits.len
|
||||||
|
};
|
||||||
|
if (!output(context, float_decimal.digits[1 .. num_digits]))
|
||||||
return false;
|
return false;
|
||||||
} else {
|
} else {
|
||||||
if (!output(context, "0"))
|
if (!output(context, "0"))
|
||||||
|
|
Loading…
Reference in New Issue
Block a user