From 68359d06b19e8212e48db330adcb7cf3fbe4b31c Mon Sep 17 00:00:00 2001 From: grdddj Date: Mon, 14 Apr 2025 13:57:35 +0200 Subject: [PATCH] chore: make the new alternative_estimate_fee be configurable, change its name from Median to Block --- bchain/coins/btc/bitcoinrpc.go | 8 +- bchain/coins/btc/mempoolspaceblock.go | 180 ++++++++++++++++++++ bchain/coins/btc/mempoolspaceblock_test.go | 134 +++++++++++++++ bchain/coins/btc/mempoolspacemedian.go | 142 --------------- bchain/coins/btc/mempoolspacemedian_test.go | 62 ------- configs/coins/bitcoin_regtest.json | 2 + 6 files changed, 320 insertions(+), 208 deletions(-) create mode 100644 bchain/coins/btc/mempoolspaceblock.go create mode 100644 bchain/coins/btc/mempoolspaceblock_test.go delete mode 100644 bchain/coins/btc/mempoolspacemedian.go delete mode 100644 bchain/coins/btc/mempoolspacemedian_test.go diff --git a/bchain/coins/btc/bitcoinrpc.go b/bchain/coins/btc/bitcoinrpc.go index cff3b5d8..00b3f468 100644 --- a/bchain/coins/btc/bitcoinrpc.go +++ b/bchain/coins/btc/bitcoinrpc.go @@ -157,10 +157,10 @@ func (b *BitcoinRPC) Initialize() error { // disable AlternativeEstimateFee logic b.alternativeFeeProvider = nil } - } else if b.ChainConfig.AlternativeEstimateFee == "mempoolspacemedian" { - glog.Info("Using MempoolSpaceMedianFee") - if b.alternativeFeeProvider, err = NewMempoolSpaceMedianFee(b, b.ChainConfig.AlternativeEstimateFeeParams); err != nil { - glog.Error("MempoolSpaceMedianFee error ", err, " Reverting to default estimateFee functionality") + } else if b.ChainConfig.AlternativeEstimateFee == "mempoolspaceblock" { + glog.Info("Using MempoolSpaceBlockFee") + if b.alternativeFeeProvider, err = NewMempoolSpaceBlockFee(b, b.ChainConfig.AlternativeEstimateFeeParams); err != nil { + glog.Error("MempoolSpaceBlockFee error ", err, " Reverting to default estimateFee functionality") // disable AlternativeEstimateFee logic b.alternativeFeeProvider = nil } diff --git a/bchain/coins/btc/mempoolspaceblock.go b/bchain/coins/btc/mempoolspaceblock.go new file mode 100644 index 00000000..abcc43a0 --- /dev/null +++ b/bchain/coins/btc/mempoolspaceblock.go @@ -0,0 +1,180 @@ +package btc + +import ( + "encoding/json" + "math" + "net/http" + "strconv" + "time" + + "github.com/golang/glog" + "github.com/juju/errors" + "github.com/trezor/blockbook/bchain" + "github.com/trezor/blockbook/common" +) + +// https://mempool.space/api/v1/fees/mempool-blocks returns a list of upcoming blocks and their medianFee. +// Example response: +// [ +// { +// "blockSize": 1604493, +// "blockVSize": 997944.75, +// "nTx": 3350, +// "totalFees": 8333539, +// "medianFee": 3.0073509137538332, +// "feeRange": [ +// 2.0444444444444443, +// 2.2135922330097086, +// 2.608695652173913, +// 3.016042780748663, +// 4.0048289738430585, +// 9.27631139325092, +// 201.06951871657753 +// ] +// }, +// ... +// ] + +type mempoolSpaceBlockFeeResult struct { + BlockSize float64 `json:"blockSize"` + BlockVSize float64 `json:"blockVSize"` + NTx int `json:"nTx"` + TotalFees int `json:"totalFees"` + MedianFee float64 `json:"medianFee"` + // 2nd, 10th, 25th, 50th, 75th, 90th, 98th percentiles + FeeRange []float64 `json:"feeRange"` +} + +type mempoolSpaceBlockFeeParams struct { + URL string `json:"url"` + PeriodSeconds int `json:"periodSeconds"` + // Either number, then take the specified index. If null or missing, take the medianFee + FeeRangeIndex *int `json:"feeRangeIndex,omitempty"` +} + +type mempoolSpaceBlockFeeProvider struct { + *alternativeFeeProvider + params mempoolSpaceBlockFeeParams +} + +// NewMempoolSpaceBlockFee initializes the provider. +func NewMempoolSpaceBlockFee(chain bchain.BlockChain, params string) (alternativeFeeProviderInterface, error) { + p := &mempoolSpaceBlockFeeProvider{alternativeFeeProvider: &alternativeFeeProvider{}} + err := json.Unmarshal([]byte(params), &p.params) + if err != nil { + return nil, err + } + + // Check mandatory parameters + if p.params.URL == "" { + return nil, errors.New("NewMempoolSpaceBlockFee: Missing url") + } + if p.params.PeriodSeconds == 0 { + return nil, errors.New("NewMempoolSpaceBlockFee: Missing periodSeconds") + } + + // Report on what is used + if p.params.FeeRangeIndex == nil { + glog.Info("NewMempoolSpaceBlockFee: Using median fee") + } else { + index := *p.params.FeeRangeIndex + if index < 0 || index > 6 { + return nil, errors.New("NewMempoolSpaceBlockFee: feeRangeIndex must be between 0 and 6") + } + glog.Infof("NewMempoolSpaceBlockFee: Using feeRangeIndex %d", index) + } + + p.chain = chain + go p.mempoolSpaceBlockFeeDownloader() + return p, nil +} + +func (p *mempoolSpaceBlockFeeProvider) mempoolSpaceBlockFeeDownloader() { + period := time.Duration(p.params.PeriodSeconds) * time.Second + timer := time.NewTimer(period) + counter := 0 + for { + var data []mempoolSpaceBlockFeeResult + err := p.mempoolSpaceBlockFeeGetData(&data) + if err != nil { + glog.Error("mempoolSpaceBlockFeeGetData ", err) + } else { + if p.mempoolSpaceBlockFeeProcessData(&data) { + if counter%60 == 0 { + p.compareToDefault() + } + counter++ + } + } + <-timer.C + timer.Reset(period) + } +} + +func (p *mempoolSpaceBlockFeeProvider) mempoolSpaceBlockFeeProcessData(data *[]mempoolSpaceBlockFeeResult) bool { + if len(*data) == 0 { + glog.Error("mempoolSpaceBlockFeeProcessData: empty data") + return false + } + + p.mux.Lock() + defer p.mux.Unlock() + + p.fees = make([]alternativeFeeProviderFee, 0, len(*data)) + + for i, block := range *data { + var fee float64 + + if p.params.FeeRangeIndex == nil { + fee = block.MedianFee + } else { + feeRange := block.FeeRange + index := *p.params.FeeRangeIndex + if len(feeRange) > index { + fee = feeRange[index] + } else { + glog.Warningf("Block %d has too short feeRange (len=%d, required=%d). Replacing by medianFee", i, len(feeRange), index) + fee = block.MedianFee + } + } + + if fee < 1 { + glog.Warningf("Skipping block at index %d due to invalid fee: %f", i, fee) + continue + } + + // TODO: it might make sense to not include _every_ block, but only e.g. first 20 and then some hardcoded ones like 50, 100, 200, etc. + // But even storing thousands of elements in []alternativeFeeProviderFee should not make a big performance overhead + // Depends on Suite requirements + + // We want to convert the fee to 3 significant digits + feeRounded := common.RoundToSignificantDigits(fee, 3) + feePerKB := int(math.Round(feeRounded * 1000)) + + p.fees = append(p.fees, alternativeFeeProviderFee{ + blocks: i + 1, + feePerKB: feePerKB, + }) + } + + p.lastSync = time.Now() + return true +} + +func (p *mempoolSpaceBlockFeeProvider) mempoolSpaceBlockFeeGetData(res interface{}) error { + httpReq, err := http.NewRequest("GET", p.params.URL, nil) + if err != nil { + return err + } + httpRes, err := http.DefaultClient.Do(httpReq) + if httpRes != nil { + defer httpRes.Body.Close() + } + if err != nil { + return err + } + if httpRes.StatusCode != http.StatusOK { + return errors.New(p.params.URL + " returned status " + strconv.Itoa(httpRes.StatusCode)) + } + return common.SafeDecodeResponseFromReader(httpRes.Body, res) +} diff --git a/bchain/coins/btc/mempoolspaceblock_test.go b/bchain/coins/btc/mempoolspaceblock_test.go new file mode 100644 index 00000000..2dba097b --- /dev/null +++ b/bchain/coins/btc/mempoolspaceblock_test.go @@ -0,0 +1,134 @@ +//go:build unittest + +package btc + +import ( + "math/big" + "strconv" + "testing" +) + +var testBlocks = []mempoolSpaceBlockFeeResult{ + { + BlockSize: 1800000, + BlockVSize: 997931, + NTx: 2500, + TotalFees: 6000000, + MedianFee: 25.1, + FeeRange: []float64{1, 5, 10, 20, 30, 50, 300}, + }, + { + BlockSize: 1750000, + BlockVSize: 997930, + NTx: 2200, + TotalFees: 4500000, + MedianFee: 7.31, + FeeRange: []float64{1, 2, 5, 10, 15, 20, 150}, + }, + { + BlockSize: 1700000, + BlockVSize: 997929, + NTx: 2000, + TotalFees: 3000000, + MedianFee: 3.14, + FeeRange: []float64{1, 1.5, 2, 5, 7, 10, 100}, + }, + { + BlockSize: 1650000, + BlockVSize: 997928, + NTx: 1800, + TotalFees: 2000000, + MedianFee: 1.34, + FeeRange: []float64{1, 1.2, 1.5, 3, 4, 5, 50}, + }, + { + BlockSize: 1600000, + BlockVSize: 997927, + NTx: 1500, + TotalFees: 1500000, + MedianFee: 1.11, + FeeRange: []float64{1, 1.05, 1.1, 1.5, 1.8, 2, 20}, + }, +} + +var estimateFeeTestCasesMedian = []struct { + blocks int + want big.Int +}{ + {0, *big.NewInt(25100)}, + {1, *big.NewInt(25100)}, + {2, *big.NewInt(7310)}, + {3, *big.NewInt(3140)}, + {4, *big.NewInt(1340)}, + {5, *big.NewInt(1110)}, + {6, *big.NewInt(1110)}, + {7, *big.NewInt(1110)}, + {10, *big.NewInt(1110)}, + {36, *big.NewInt(1110)}, + {100, *big.NewInt(1110)}, + {201, *big.NewInt(1110)}, + {501, *big.NewInt(1110)}, + {5000000, *big.NewInt(1110)}, +} + +var estimateFeeTestCasesFeeRangeIndex5 = []struct { + blocks int + want big.Int +}{ + {0, *big.NewInt(50000)}, + {1, *big.NewInt(50000)}, + {2, *big.NewInt(20000)}, + {3, *big.NewInt(10000)}, + {4, *big.NewInt(5000)}, + {5, *big.NewInt(2000)}, + {6, *big.NewInt(2000)}, + {7, *big.NewInt(2000)}, + {10, *big.NewInt(2000)}, + {36, *big.NewInt(2000)}, + {100, *big.NewInt(2000)}, + {201, *big.NewInt(2000)}, + {501, *big.NewInt(2000)}, + {5000000, *big.NewInt(2000)}, +} + +func runEstimateFeeTest(t *testing.T, testName string, feeRangeIndex *int, expected []struct { + blocks int + want big.Int +}) { + m := &mempoolSpaceBlockFeeProvider{alternativeFeeProvider: &alternativeFeeProvider{}} + m.params.FeeRangeIndex = feeRangeIndex + + success := m.mempoolSpaceBlockFeeProcessData(&testBlocks) + if !success { + t.Fatalf("[%s] Expected data to be processed successfully", testName) + } + + for _, tt := range expected { + t.Run(testName+"_"+strconv.Itoa(tt.blocks), func(t *testing.T) { + got, err := m.estimateFee(tt.blocks) + if err != nil { + t.Errorf("[%s] estimateFee returned error: %v", testName, err) + } + if got.Cmp(&tt.want) != 0 { + t.Errorf("[%s] estimateFee(%d) = %v, want %v", testName, tt.blocks, got, tt.want) + } + }) + } +} + +func Test_mempoolSpaceBlockFeeProviderMedian(t *testing.T) { + // Taking the median explicitly + runEstimateFeeTest(t, "median", nil, estimateFeeTestCasesMedian) +} + +func Test_mempoolSpaceBlockFeeProviderSecondLargestIndex(t *testing.T) { + // Taking the valid index + index := 5 + runEstimateFeeTest(t, "feeRangeIndex_5", &index, estimateFeeTestCasesFeeRangeIndex5) +} + +func Test_mempoolSpaceBlockFeeProviderInvalidIndexTooHigh(t *testing.T) { + // Index is too high, will fallback to median + index := 555 + runEstimateFeeTest(t, "invalidFeeRangeIndex_555", &index, estimateFeeTestCasesMedian) +} diff --git a/bchain/coins/btc/mempoolspacemedian.go b/bchain/coins/btc/mempoolspacemedian.go deleted file mode 100644 index 8341ed99..00000000 --- a/bchain/coins/btc/mempoolspacemedian.go +++ /dev/null @@ -1,142 +0,0 @@ -package btc - -import ( - "encoding/json" - "math" - "net/http" - "strconv" - "time" - - "github.com/golang/glog" - "github.com/juju/errors" - "github.com/trezor/blockbook/bchain" - "github.com/trezor/blockbook/common" -) - -// https://mempool.space/api/v1/fees/mempool-blocks returns a list of upcoming blocks and their medianFee. -// Example response: -// [ -// { -// "blockSize": 1589235, -// "blockVSize": 997914, -// "nTx": 4224, -// "totalFees": 6935988, -// "medianFee": 3.622, -// "feeRange": [ ... ] -// }, -// ... -// ] - -type mempoolSpaceMedianFeeResult struct { - BlockSize float64 `json:"blockSize"` - BlockVSize float64 `json:"blockVSize"` - NTx int `json:"nTx"` - TotalFees int `json:"totalFees"` - MedianFee float64 `json:"medianFee"` - FeeRange []float64 `json:"feeRange"` -} - -type mempoolSpaceMedianFeeParams struct { - URL string `json:"url"` - PeriodSeconds int `json:"periodSeconds"` -} - -type mempoolSpaceMedianFeeProvider struct { - *alternativeFeeProvider - params mempoolSpaceMedianFeeParams -} - -// NewMempoolSpaceMedianFee initializes the median-fee provider using mempool.space data. -func NewMempoolSpaceMedianFee(chain bchain.BlockChain, params string) (alternativeFeeProviderInterface, error) { - p := &mempoolSpaceMedianFeeProvider{alternativeFeeProvider: &alternativeFeeProvider{}} - err := json.Unmarshal([]byte(params), &p.params) - if err != nil { - return nil, err - } - if p.params.URL == "" || p.params.PeriodSeconds == 0 { - return nil, errors.New("NewMempoolSpaceMedianFee: Missing parameters") - } - p.chain = chain - go p.mempoolSpaceMedianFeeDownloader() - return p, nil -} - -func (p *mempoolSpaceMedianFeeProvider) mempoolSpaceMedianFeeDownloader() { - period := time.Duration(p.params.PeriodSeconds) * time.Second - timer := time.NewTimer(period) - counter := 0 - for { - var data []mempoolSpaceMedianFeeResult - err := p.mempoolSpaceMedianFeeGetData(&data) - if err != nil { - glog.Error("mempoolSpaceMedianFeeGetData ", err) - } else { - if p.mempoolSpaceMedianFeeProcessData(&data) { - if counter%60 == 0 { - p.compareToDefault() - } - counter++ - } - } - <-timer.C - timer.Reset(period) - } -} - -func (p *mempoolSpaceMedianFeeProvider) mempoolSpaceMedianFeeProcessData(data *[]mempoolSpaceMedianFeeResult) bool { - if len(*data) == 0 { - glog.Error("mempoolSpaceMedianFeeProcessData: empty data") - return false - } - - p.mux.Lock() - defer p.mux.Unlock() - - p.fees = make([]alternativeFeeProviderFee, 0, len(*data)) - - zeroReplacement := 1.05 - - for i, block := range *data { - if block.MedianFee == 0 { - glog.Infof("Replacing zero medianFee by: %f", zeroReplacement) - block.MedianFee = zeroReplacement - } else if block.MedianFee < 1 { - glog.Warningf("Skipping block at index %d due to invalid medianFee: %f", i, block.MedianFee) - continue - } - - // TODO: it might make sense to not include _every_ block, but only e.g. first 20 and then some hardcoded ones like 50, 100, 200, etc. - // But even storing thousands of elements in []alternativeFeeProviderFee should not make a big performance overhead - // Depends on Suite requirements - - // We want to convert the median fee to 3 significant digits - medianFee := common.RoundToSignificantDigits(block.MedianFee, 3) - feePerKB := int(math.Round(medianFee * 1000)) // convert sat/vB to sat/KB - - p.fees = append(p.fees, alternativeFeeProviderFee{ - blocks: i + 1, // simple mapping: index 0 -> 1 block, etc. - feePerKB: feePerKB, - }) - } - - p.lastSync = time.Now() - return true -} - -func (p *mempoolSpaceMedianFeeProvider) mempoolSpaceMedianFeeGetData(res interface{}) error { - httpReq, err := http.NewRequest("GET", p.params.URL, nil) - if err != nil { - return err - } - httpRes, err := http.DefaultClient.Do(httpReq) - if httpRes != nil { - defer httpRes.Body.Close() - } - if err != nil { - return err - } - if httpRes.StatusCode != http.StatusOK { - return errors.New(p.params.URL + " returned status " + strconv.Itoa(httpRes.StatusCode)) - } - return common.SafeDecodeResponseFromReader(httpRes.Body, res) -} diff --git a/bchain/coins/btc/mempoolspacemedian_test.go b/bchain/coins/btc/mempoolspacemedian_test.go deleted file mode 100644 index ca08f6fe..00000000 --- a/bchain/coins/btc/mempoolspacemedian_test.go +++ /dev/null @@ -1,62 +0,0 @@ -//go:build unittest - -package btc - -import ( - "math/big" - "strconv" - "testing" -) - -func Test_mempoolSpaceMedianFeeProvider(t *testing.T) { - m := &mempoolSpaceMedianFeeProvider{alternativeFeeProvider: &alternativeFeeProvider{}} - testBlocks := []mempoolSpaceMedianFeeResult{ - {MedianFee: 5.123456}, - {MedianFee: 4.456789}, - {MedianFee: 3.789012}, - {MedianFee: 2.012345}, - {MedianFee: 1.345678}, - } - - success := m.mempoolSpaceMedianFeeProcessData(&testBlocks) - if !success { - t.Fatal("Expected data to be processed successfully") - } - - tests := []struct { - blocks int - want big.Int - }{ - {0, *big.NewInt(5120)}, - {1, *big.NewInt(5120)}, - {2, *big.NewInt(4460)}, - {3, *big.NewInt(3790)}, - {4, *big.NewInt(2010)}, - {5, *big.NewInt(1350)}, - {6, *big.NewInt(1350)}, - {7, *big.NewInt(1350)}, - {10, *big.NewInt(1350)}, - {18, *big.NewInt(1350)}, - {19, *big.NewInt(1350)}, - {36, *big.NewInt(1350)}, - {37, *big.NewInt(1350)}, - {100, *big.NewInt(1350)}, - {101, *big.NewInt(1350)}, - {200, *big.NewInt(1350)}, - {201, *big.NewInt(1350)}, - {500, *big.NewInt(1350)}, - {501, *big.NewInt(1350)}, - {5000000, *big.NewInt(1350)}, - } - for _, tt := range tests { - t.Run(strconv.Itoa(tt.blocks), func(t *testing.T) { - got, err := m.estimateFee(tt.blocks) - if err != nil { - t.Error("estimateFee returned error ", err) - } - if got.Cmp(&tt.want) != 0 { - t.Errorf("estimateFee(%d) = %v, want %v", tt.blocks, got, tt.want) - } - }) - } -} diff --git a/configs/coins/bitcoin_regtest.json b/configs/coins/bitcoin_regtest.json index 825dbd8b..0f75ad04 100644 --- a/configs/coins/bitcoin_regtest.json +++ b/configs/coins/bitcoin_regtest.json @@ -64,6 +64,8 @@ "xpub_magic_segwit_native": 73342198, "slip44": 1, "additional_params": { + "alternative_estimate_fee": "mempoolspaceblock", + "alternative_estimate_fee_params": "{\"url\": \"https://mempool.space/api/v1/fees/mempool-blocks\", \"periodSeconds\": 20, \"feeRangeIndex\": 5}", "block_golomb_filter_p": 20, "block_filter_scripts": "taproot-noordinals", "block_filter_use_zeroed_key": true,