chore: make the new alternative_estimate_fee be configurable, change its name from Median to Block

This commit is contained in:
grdddj
2025-04-14 13:57:35 +02:00
parent 23cbb397dd
commit 68359d06b1
6 changed files with 320 additions and 208 deletions

View File

@@ -157,10 +157,10 @@ func (b *BitcoinRPC) Initialize() error {
// disable AlternativeEstimateFee logic
b.alternativeFeeProvider = nil
}
} else if b.ChainConfig.AlternativeEstimateFee == "mempoolspacemedian" {
glog.Info("Using MempoolSpaceMedianFee")
if b.alternativeFeeProvider, err = NewMempoolSpaceMedianFee(b, b.ChainConfig.AlternativeEstimateFeeParams); err != nil {
glog.Error("MempoolSpaceMedianFee error ", err, " Reverting to default estimateFee functionality")
} else if b.ChainConfig.AlternativeEstimateFee == "mempoolspaceblock" {
glog.Info("Using MempoolSpaceBlockFee")
if b.alternativeFeeProvider, err = NewMempoolSpaceBlockFee(b, b.ChainConfig.AlternativeEstimateFeeParams); err != nil {
glog.Error("MempoolSpaceBlockFee error ", err, " Reverting to default estimateFee functionality")
// disable AlternativeEstimateFee logic
b.alternativeFeeProvider = nil
}

View File

@@ -0,0 +1,180 @@
package btc
import (
"encoding/json"
"math"
"net/http"
"strconv"
"time"
"github.com/golang/glog"
"github.com/juju/errors"
"github.com/trezor/blockbook/bchain"
"github.com/trezor/blockbook/common"
)
// https://mempool.space/api/v1/fees/mempool-blocks returns a list of upcoming blocks and their medianFee.
// Example response:
// [
// {
// "blockSize": 1604493,
// "blockVSize": 997944.75,
// "nTx": 3350,
// "totalFees": 8333539,
// "medianFee": 3.0073509137538332,
// "feeRange": [
// 2.0444444444444443,
// 2.2135922330097086,
// 2.608695652173913,
// 3.016042780748663,
// 4.0048289738430585,
// 9.27631139325092,
// 201.06951871657753
// ]
// },
// ...
// ]
type mempoolSpaceBlockFeeResult struct {
BlockSize float64 `json:"blockSize"`
BlockVSize float64 `json:"blockVSize"`
NTx int `json:"nTx"`
TotalFees int `json:"totalFees"`
MedianFee float64 `json:"medianFee"`
// 2nd, 10th, 25th, 50th, 75th, 90th, 98th percentiles
FeeRange []float64 `json:"feeRange"`
}
type mempoolSpaceBlockFeeParams struct {
URL string `json:"url"`
PeriodSeconds int `json:"periodSeconds"`
// Either number, then take the specified index. If null or missing, take the medianFee
FeeRangeIndex *int `json:"feeRangeIndex,omitempty"`
}
type mempoolSpaceBlockFeeProvider struct {
*alternativeFeeProvider
params mempoolSpaceBlockFeeParams
}
// NewMempoolSpaceBlockFee initializes the provider.
func NewMempoolSpaceBlockFee(chain bchain.BlockChain, params string) (alternativeFeeProviderInterface, error) {
p := &mempoolSpaceBlockFeeProvider{alternativeFeeProvider: &alternativeFeeProvider{}}
err := json.Unmarshal([]byte(params), &p.params)
if err != nil {
return nil, err
}
// Check mandatory parameters
if p.params.URL == "" {
return nil, errors.New("NewMempoolSpaceBlockFee: Missing url")
}
if p.params.PeriodSeconds == 0 {
return nil, errors.New("NewMempoolSpaceBlockFee: Missing periodSeconds")
}
// Report on what is used
if p.params.FeeRangeIndex == nil {
glog.Info("NewMempoolSpaceBlockFee: Using median fee")
} else {
index := *p.params.FeeRangeIndex
if index < 0 || index > 6 {
return nil, errors.New("NewMempoolSpaceBlockFee: feeRangeIndex must be between 0 and 6")
}
glog.Infof("NewMempoolSpaceBlockFee: Using feeRangeIndex %d", index)
}
p.chain = chain
go p.mempoolSpaceBlockFeeDownloader()
return p, nil
}
func (p *mempoolSpaceBlockFeeProvider) mempoolSpaceBlockFeeDownloader() {
period := time.Duration(p.params.PeriodSeconds) * time.Second
timer := time.NewTimer(period)
counter := 0
for {
var data []mempoolSpaceBlockFeeResult
err := p.mempoolSpaceBlockFeeGetData(&data)
if err != nil {
glog.Error("mempoolSpaceBlockFeeGetData ", err)
} else {
if p.mempoolSpaceBlockFeeProcessData(&data) {
if counter%60 == 0 {
p.compareToDefault()
}
counter++
}
}
<-timer.C
timer.Reset(period)
}
}
func (p *mempoolSpaceBlockFeeProvider) mempoolSpaceBlockFeeProcessData(data *[]mempoolSpaceBlockFeeResult) bool {
if len(*data) == 0 {
glog.Error("mempoolSpaceBlockFeeProcessData: empty data")
return false
}
p.mux.Lock()
defer p.mux.Unlock()
p.fees = make([]alternativeFeeProviderFee, 0, len(*data))
for i, block := range *data {
var fee float64
if p.params.FeeRangeIndex == nil {
fee = block.MedianFee
} else {
feeRange := block.FeeRange
index := *p.params.FeeRangeIndex
if len(feeRange) > index {
fee = feeRange[index]
} else {
glog.Warningf("Block %d has too short feeRange (len=%d, required=%d). Replacing by medianFee", i, len(feeRange), index)
fee = block.MedianFee
}
}
if fee < 1 {
glog.Warningf("Skipping block at index %d due to invalid fee: %f", i, fee)
continue
}
// TODO: it might make sense to not include _every_ block, but only e.g. first 20 and then some hardcoded ones like 50, 100, 200, etc.
// But even storing thousands of elements in []alternativeFeeProviderFee should not make a big performance overhead
// Depends on Suite requirements
// We want to convert the fee to 3 significant digits
feeRounded := common.RoundToSignificantDigits(fee, 3)
feePerKB := int(math.Round(feeRounded * 1000))
p.fees = append(p.fees, alternativeFeeProviderFee{
blocks: i + 1,
feePerKB: feePerKB,
})
}
p.lastSync = time.Now()
return true
}
func (p *mempoolSpaceBlockFeeProvider) mempoolSpaceBlockFeeGetData(res interface{}) error {
httpReq, err := http.NewRequest("GET", p.params.URL, nil)
if err != nil {
return err
}
httpRes, err := http.DefaultClient.Do(httpReq)
if httpRes != nil {
defer httpRes.Body.Close()
}
if err != nil {
return err
}
if httpRes.StatusCode != http.StatusOK {
return errors.New(p.params.URL + " returned status " + strconv.Itoa(httpRes.StatusCode))
}
return common.SafeDecodeResponseFromReader(httpRes.Body, res)
}

View File

@@ -0,0 +1,134 @@
//go:build unittest
package btc
import (
"math/big"
"strconv"
"testing"
)
var testBlocks = []mempoolSpaceBlockFeeResult{
{
BlockSize: 1800000,
BlockVSize: 997931,
NTx: 2500,
TotalFees: 6000000,
MedianFee: 25.1,
FeeRange: []float64{1, 5, 10, 20, 30, 50, 300},
},
{
BlockSize: 1750000,
BlockVSize: 997930,
NTx: 2200,
TotalFees: 4500000,
MedianFee: 7.31,
FeeRange: []float64{1, 2, 5, 10, 15, 20, 150},
},
{
BlockSize: 1700000,
BlockVSize: 997929,
NTx: 2000,
TotalFees: 3000000,
MedianFee: 3.14,
FeeRange: []float64{1, 1.5, 2, 5, 7, 10, 100},
},
{
BlockSize: 1650000,
BlockVSize: 997928,
NTx: 1800,
TotalFees: 2000000,
MedianFee: 1.34,
FeeRange: []float64{1, 1.2, 1.5, 3, 4, 5, 50},
},
{
BlockSize: 1600000,
BlockVSize: 997927,
NTx: 1500,
TotalFees: 1500000,
MedianFee: 1.11,
FeeRange: []float64{1, 1.05, 1.1, 1.5, 1.8, 2, 20},
},
}
var estimateFeeTestCasesMedian = []struct {
blocks int
want big.Int
}{
{0, *big.NewInt(25100)},
{1, *big.NewInt(25100)},
{2, *big.NewInt(7310)},
{3, *big.NewInt(3140)},
{4, *big.NewInt(1340)},
{5, *big.NewInt(1110)},
{6, *big.NewInt(1110)},
{7, *big.NewInt(1110)},
{10, *big.NewInt(1110)},
{36, *big.NewInt(1110)},
{100, *big.NewInt(1110)},
{201, *big.NewInt(1110)},
{501, *big.NewInt(1110)},
{5000000, *big.NewInt(1110)},
}
var estimateFeeTestCasesFeeRangeIndex5 = []struct {
blocks int
want big.Int
}{
{0, *big.NewInt(50000)},
{1, *big.NewInt(50000)},
{2, *big.NewInt(20000)},
{3, *big.NewInt(10000)},
{4, *big.NewInt(5000)},
{5, *big.NewInt(2000)},
{6, *big.NewInt(2000)},
{7, *big.NewInt(2000)},
{10, *big.NewInt(2000)},
{36, *big.NewInt(2000)},
{100, *big.NewInt(2000)},
{201, *big.NewInt(2000)},
{501, *big.NewInt(2000)},
{5000000, *big.NewInt(2000)},
}
func runEstimateFeeTest(t *testing.T, testName string, feeRangeIndex *int, expected []struct {
blocks int
want big.Int
}) {
m := &mempoolSpaceBlockFeeProvider{alternativeFeeProvider: &alternativeFeeProvider{}}
m.params.FeeRangeIndex = feeRangeIndex
success := m.mempoolSpaceBlockFeeProcessData(&testBlocks)
if !success {
t.Fatalf("[%s] Expected data to be processed successfully", testName)
}
for _, tt := range expected {
t.Run(testName+"_"+strconv.Itoa(tt.blocks), func(t *testing.T) {
got, err := m.estimateFee(tt.blocks)
if err != nil {
t.Errorf("[%s] estimateFee returned error: %v", testName, err)
}
if got.Cmp(&tt.want) != 0 {
t.Errorf("[%s] estimateFee(%d) = %v, want %v", testName, tt.blocks, got, tt.want)
}
})
}
}
func Test_mempoolSpaceBlockFeeProviderMedian(t *testing.T) {
// Taking the median explicitly
runEstimateFeeTest(t, "median", nil, estimateFeeTestCasesMedian)
}
func Test_mempoolSpaceBlockFeeProviderSecondLargestIndex(t *testing.T) {
// Taking the valid index
index := 5
runEstimateFeeTest(t, "feeRangeIndex_5", &index, estimateFeeTestCasesFeeRangeIndex5)
}
func Test_mempoolSpaceBlockFeeProviderInvalidIndexTooHigh(t *testing.T) {
// Index is too high, will fallback to median
index := 555
runEstimateFeeTest(t, "invalidFeeRangeIndex_555", &index, estimateFeeTestCasesMedian)
}

View File

@@ -1,142 +0,0 @@
package btc
import (
"encoding/json"
"math"
"net/http"
"strconv"
"time"
"github.com/golang/glog"
"github.com/juju/errors"
"github.com/trezor/blockbook/bchain"
"github.com/trezor/blockbook/common"
)
// https://mempool.space/api/v1/fees/mempool-blocks returns a list of upcoming blocks and their medianFee.
// Example response:
// [
// {
// "blockSize": 1589235,
// "blockVSize": 997914,
// "nTx": 4224,
// "totalFees": 6935988,
// "medianFee": 3.622,
// "feeRange": [ ... ]
// },
// ...
// ]
type mempoolSpaceMedianFeeResult struct {
BlockSize float64 `json:"blockSize"`
BlockVSize float64 `json:"blockVSize"`
NTx int `json:"nTx"`
TotalFees int `json:"totalFees"`
MedianFee float64 `json:"medianFee"`
FeeRange []float64 `json:"feeRange"`
}
type mempoolSpaceMedianFeeParams struct {
URL string `json:"url"`
PeriodSeconds int `json:"periodSeconds"`
}
type mempoolSpaceMedianFeeProvider struct {
*alternativeFeeProvider
params mempoolSpaceMedianFeeParams
}
// NewMempoolSpaceMedianFee initializes the median-fee provider using mempool.space data.
func NewMempoolSpaceMedianFee(chain bchain.BlockChain, params string) (alternativeFeeProviderInterface, error) {
p := &mempoolSpaceMedianFeeProvider{alternativeFeeProvider: &alternativeFeeProvider{}}
err := json.Unmarshal([]byte(params), &p.params)
if err != nil {
return nil, err
}
if p.params.URL == "" || p.params.PeriodSeconds == 0 {
return nil, errors.New("NewMempoolSpaceMedianFee: Missing parameters")
}
p.chain = chain
go p.mempoolSpaceMedianFeeDownloader()
return p, nil
}
func (p *mempoolSpaceMedianFeeProvider) mempoolSpaceMedianFeeDownloader() {
period := time.Duration(p.params.PeriodSeconds) * time.Second
timer := time.NewTimer(period)
counter := 0
for {
var data []mempoolSpaceMedianFeeResult
err := p.mempoolSpaceMedianFeeGetData(&data)
if err != nil {
glog.Error("mempoolSpaceMedianFeeGetData ", err)
} else {
if p.mempoolSpaceMedianFeeProcessData(&data) {
if counter%60 == 0 {
p.compareToDefault()
}
counter++
}
}
<-timer.C
timer.Reset(period)
}
}
func (p *mempoolSpaceMedianFeeProvider) mempoolSpaceMedianFeeProcessData(data *[]mempoolSpaceMedianFeeResult) bool {
if len(*data) == 0 {
glog.Error("mempoolSpaceMedianFeeProcessData: empty data")
return false
}
p.mux.Lock()
defer p.mux.Unlock()
p.fees = make([]alternativeFeeProviderFee, 0, len(*data))
zeroReplacement := 1.05
for i, block := range *data {
if block.MedianFee == 0 {
glog.Infof("Replacing zero medianFee by: %f", zeroReplacement)
block.MedianFee = zeroReplacement
} else if block.MedianFee < 1 {
glog.Warningf("Skipping block at index %d due to invalid medianFee: %f", i, block.MedianFee)
continue
}
// TODO: it might make sense to not include _every_ block, but only e.g. first 20 and then some hardcoded ones like 50, 100, 200, etc.
// But even storing thousands of elements in []alternativeFeeProviderFee should not make a big performance overhead
// Depends on Suite requirements
// We want to convert the median fee to 3 significant digits
medianFee := common.RoundToSignificantDigits(block.MedianFee, 3)
feePerKB := int(math.Round(medianFee * 1000)) // convert sat/vB to sat/KB
p.fees = append(p.fees, alternativeFeeProviderFee{
blocks: i + 1, // simple mapping: index 0 -> 1 block, etc.
feePerKB: feePerKB,
})
}
p.lastSync = time.Now()
return true
}
func (p *mempoolSpaceMedianFeeProvider) mempoolSpaceMedianFeeGetData(res interface{}) error {
httpReq, err := http.NewRequest("GET", p.params.URL, nil)
if err != nil {
return err
}
httpRes, err := http.DefaultClient.Do(httpReq)
if httpRes != nil {
defer httpRes.Body.Close()
}
if err != nil {
return err
}
if httpRes.StatusCode != http.StatusOK {
return errors.New(p.params.URL + " returned status " + strconv.Itoa(httpRes.StatusCode))
}
return common.SafeDecodeResponseFromReader(httpRes.Body, res)
}

View File

@@ -1,62 +0,0 @@
//go:build unittest
package btc
import (
"math/big"
"strconv"
"testing"
)
func Test_mempoolSpaceMedianFeeProvider(t *testing.T) {
m := &mempoolSpaceMedianFeeProvider{alternativeFeeProvider: &alternativeFeeProvider{}}
testBlocks := []mempoolSpaceMedianFeeResult{
{MedianFee: 5.123456},
{MedianFee: 4.456789},
{MedianFee: 3.789012},
{MedianFee: 2.012345},
{MedianFee: 1.345678},
}
success := m.mempoolSpaceMedianFeeProcessData(&testBlocks)
if !success {
t.Fatal("Expected data to be processed successfully")
}
tests := []struct {
blocks int
want big.Int
}{
{0, *big.NewInt(5120)},
{1, *big.NewInt(5120)},
{2, *big.NewInt(4460)},
{3, *big.NewInt(3790)},
{4, *big.NewInt(2010)},
{5, *big.NewInt(1350)},
{6, *big.NewInt(1350)},
{7, *big.NewInt(1350)},
{10, *big.NewInt(1350)},
{18, *big.NewInt(1350)},
{19, *big.NewInt(1350)},
{36, *big.NewInt(1350)},
{37, *big.NewInt(1350)},
{100, *big.NewInt(1350)},
{101, *big.NewInt(1350)},
{200, *big.NewInt(1350)},
{201, *big.NewInt(1350)},
{500, *big.NewInt(1350)},
{501, *big.NewInt(1350)},
{5000000, *big.NewInt(1350)},
}
for _, tt := range tests {
t.Run(strconv.Itoa(tt.blocks), func(t *testing.T) {
got, err := m.estimateFee(tt.blocks)
if err != nil {
t.Error("estimateFee returned error ", err)
}
if got.Cmp(&tt.want) != 0 {
t.Errorf("estimateFee(%d) = %v, want %v", tt.blocks, got, tt.want)
}
})
}
}

View File

@@ -64,6 +64,8 @@
"xpub_magic_segwit_native": 73342198,
"slip44": 1,
"additional_params": {
"alternative_estimate_fee": "mempoolspaceblock",
"alternative_estimate_fee_params": "{\"url\": \"https://mempool.space/api/v1/fees/mempool-blocks\", \"periodSeconds\": 20, \"feeRangeIndex\": 5}",
"block_golomb_filter_p": 20,
"block_filter_scripts": "taproot-noordinals",
"block_filter_use_zeroed_key": true,