Use new db column blockaddresses to support UTXO chain block disconnect

bchain/baseparser.go

@@ -35,6 +35,11 @@ func (p *BaseParser) PackedTxidLen() int {
 	return 32
 }
 
+// KeepBlockAddresses returns number of blocks which are to be kept in blockaddresses column
+func (p *BaseParser) KeepBlockAddresses() int {
+	return 100
+}
+
 // PackTxid packs txid to byte array
 func (p *BaseParser) PackTxid(txid string) ([]byte, error) {
 	return hex.DecodeString(txid)

bchain/coins/eth/ethparser.go

@@ -272,3 +272,9 @@ func (p *EthereumParser) UnpackBlockHash(buf []byte) (string, error) {
 func (p *EthereumParser) IsUTXOChain() bool {
 	return false
 }
+
+// KeepBlockAddresses returns number of blocks which are to be kept in blockaddresses column
+// do not use the blockaddresses for eth
+func (p *EthereumParser) KeepBlockAddresses() int {
+	return 0
+}

bchain/types.go

@@ -134,6 +134,10 @@ type BlockChainParser interface {
 	// UTXO chains need "inputs" column in db, that map transactions to transactions that spend them
 	// non UTXO chains have mapping of address to input and output transactions directly in "outputs" column in db
 	IsUTXOChain() bool
+	// KeepBlockAddresses returns number of blocks which are to be kept in blockaddresses column
+	// and used in case of fork
+	// if 0 the blockaddresses column is not used at all (usually non UTXO chains)
+	KeepBlockAddresses() int
 	// address id conversions
 	GetAddrIDFromVout(output *Vout) ([]byte, error)
 	GetAddrIDFromAddress(address string) ([]byte, error)

db/rocksdb.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/bsm/go-vlq"
 	"github.com/golang/glog"
+	"github.com/juju/errors"
 
 	"github.com/tecbot/gorocksdb"
 )
@@ -40,9 +41,10 @@ const (
 	cfAddresses
 	cfUnspentTxs
 	cfTransactions
+	cfBlockAddresses
 )
 
-var cfNames = []string{"default", "height", "addresses", "unspenttxs", "transactions"}
+var cfNames = []string{"default", "height", "addresses", "unspenttxs", "transactions", "blockaddresses"}
 
 func openDB(path string) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error) {
 	c := gorocksdb.NewLRUCache(8 << 30) // 8GB
@@ -80,7 +82,7 @@ func openDB(path string) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error)
 	optsOutputs.SetMaxOpenFiles(25000)
 	optsOutputs.SetCompression(gorocksdb.NoCompression)
 
-	fcOptions := []*gorocksdb.Options{opts, opts, optsOutputs, opts, opts}
+	fcOptions := []*gorocksdb.Options{opts, opts, optsOutputs, opts, opts, opts}
 
 	db, cfh, err := gorocksdb.OpenDbColumnFamilies(opts, path, cfNames, fcOptions)
 	if err != nil {
@@ -239,6 +241,9 @@ type outpoint struct {
 }
 
 func (d *RocksDB) writeAddressRecords(wb *gorocksdb.WriteBatch, block *bchain.Block, op int, records map[string][]outpoint) error {
+	keep := d.chainParser.KeepBlockAddresses()
+	blockAddresses := make([]byte, 0)
+	vBuf := make([]byte, vlq.MaxLen32)
 	for addrID, outpoints := range records {
 		key, err := packOutputKey([]byte(addrID), block.Height)
 		if err != nil {
@@ -253,10 +258,36 @@ func (d *RocksDB) writeAddressRecords(wb *gorocksdb.WriteBatch, block *bchain.Bl
 				continue
 			}
 			wb.PutCF(d.cfh[cfAddresses], key, val)
+			if keep > 0 {
+				// collect all addresses to be stored in blockaddresses
+				vl := packVarint(int32(len([]byte(addrID))), vBuf)
+				blockAddresses = append(blockAddresses, vBuf[0:vl]...)
+				blockAddresses = append(blockAddresses, []byte(addrID)...)
+			}
 		case opDelete:
 			wb.DeleteCF(d.cfh[cfAddresses], key)
 		}
 	}
+	if keep > 0 && op == opInsert {
+		// write new block address
+		key := packUint(block.Height)
+		wb.PutCF(d.cfh[cfBlockAddresses], key, blockAddresses)
+		// cleanup old block address
+		if block.Height > uint32(keep) {
+			for rh := block.Height - uint32(keep); rh < block.Height; rh-- {
+				key = packUint(rh)
+				val, err := d.db.GetCF(d.ro, d.cfh[cfBlockAddresses], key)
+				if err != nil {
+					return err
+				}
+				if val.Size() == 0 {
+					break
+				}
+				val.Free()
+				d.db.DeleteCF(d.wo, d.cfh[cfBlockAddresses], key)
+			}
+		}
+	}
 	return nil
 }
 
@@ -305,7 +336,7 @@ func appendPackedAddrID(txAddrs []byte, addrID []byte, n uint32, remaining int)
 }
 
 func findAndRemoveUnspentAddr(unspentAddrs []byte, vout uint32) ([]byte, uint32, []byte) {
-	// the addresses are packed as lenaddrID:addrID:vout, where lenaddrID and vout are varints
+	// the addresses are packed as lenaddrID addrID vout, where lenaddrID and vout are varints
 	for i := 0; i < len(unspentAddrs); {
 		l, lv1 := unpackVarint(unspentAddrs[i:])
 		// index of vout of address in unspentAddrs
@@ -326,6 +357,11 @@ func findAndRemoveUnspentAddr(unspentAddrs []byte, vout uint32) ([]byte, uint32,
 }
 
 func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Block, op int) error {
+	if op == opDelete {
+		// block does not contain mapping tx-> input address, which is necessary to recreate
+		// unspentTxs; therefore it is not possible to DisconnectBlocks this way
+		return errors.New("DisconnectBlock is not supported for UTXO chains")
+	}
 	addresses := make(map[string][]outpoint)
 	unspentTxs := make(map[string][]byte)
 	btxIDs := make([][]byte, len(block.Txs))
@@ -355,7 +391,7 @@ func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Blo
 		}
 		unspentTxs[string(btxID)] = txAddrs
 	}
-	// locate unspent addresses and add them to addresses map them in format txid ^index
+	// locate addresses spent by this tx and add them to addresses map them in format txid ^index
 	for txi, tx := range block.Txs {
 		spendingTxid := btxIDs[txi]
 		for i, input := range tx.Vin {
@@ -393,15 +429,10 @@ func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Blo
 	}
 	// save unspent txs from current block
 	for tx, val := range unspentTxs {
-		switch op {
-		case opInsert:
-			if len(val) == 0 {
-				wb.DeleteCF(d.cfh[cfUnspentTxs], []byte(tx))
-			} else {
-				wb.PutCF(d.cfh[cfUnspentTxs], []byte(tx), val)
-			}
-		case opDelete:
+		if len(val) == 0 {
 			wb.DeleteCF(d.cfh[cfUnspentTxs], []byte(tx))
+		} else {
+			wb.PutCF(d.cfh[cfUnspentTxs], []byte(tx), val)
 		}
 	}
 	return nil

db/rocksdb_test.go

@@ -9,6 +9,7 @@ import (
 	"reflect"
 	"sort"
 	"strconv"
+	"strings"
 	"testing"
 
 	"github.com/juju/errors"
@@ -41,14 +42,17 @@ func addressToPubKeyHex(addr string, t *testing.T, d *RocksDB) string {
 	return hex.EncodeToString(b)
 }
 
-func addressToPubKeyHexWithLenght(addr string, t *testing.T, d *RocksDB) string {
+func addressToPubKeyHexWithLength(addr string, t *testing.T, d *RocksDB) string {
 	h := addressToPubKeyHex(addr, t, d)
 	// length is signed varint, therefore 2 times big, we can take len(h) as the correct value
 	return strconv.FormatInt(int64(len(h)), 16) + h
 }
 
+// keyPair is used to compare given key value in DB with expected
+// for more complicated compares it is possible to specify CompareFunc
 type keyPair struct {
-	Key, Value string
+	Key, Value  string
+	CompareFunc func(string) bool
 }
 
 func checkColumn(d *RocksDB, col int, kp []keyPair) error {
@@ -67,7 +71,13 @@ func checkColumn(d *RocksDB, col int, kp []keyPair) error {
 			return errors.Errorf("Incorrect key %v found in column %v row %v, expecting %v", key, col, i, kp[i].Key)
 		}
 		val := hex.EncodeToString(it.Value().Data())
-		if val != kp[i].Value {
+		var valOK bool
+		if kp[i].CompareFunc == nil {
+			valOK = val == kp[i].Value
+		} else {
+			valOK = kp[i].CompareFunc(val)
+		}
+		if !valOK {
 			return errors.Errorf("Incorrect value %v found in column %v row %v, expecting %v", val, col, i, kp[i].Value)
 		}
 		i++
@@ -78,7 +88,7 @@ func checkColumn(d *RocksDB, col int, kp []keyPair) error {
 	return nil
 }
 
-func getTestBlock1(t *testing.T, d *RocksDB) *bchain.Block {
+func getTestUTXOBlock1(t *testing.T, d *RocksDB) *bchain.Block {
 	return &bchain.Block{
 		BlockHeader: bchain.BlockHeader{
 			Height: 225493,
@@ -123,7 +133,7 @@ func getTestBlock1(t *testing.T, d *RocksDB) *bchain.Block {
 	}
 }
 
-func getTestBlock2(t *testing.T, d *RocksDB) *bchain.Block {
+func getTestUTXOBlock2(t *testing.T, d *RocksDB) *bchain.Block {
 	return &bchain.Block{
 		BlockHeader: bchain.BlockHeader{
 			Height: 225494,
@@ -188,9 +198,9 @@ func getTestBlock2(t *testing.T, d *RocksDB) *bchain.Block {
 	}
 }
 
-func verifyAfterBlock1(t *testing.T, d *RocksDB) {
+func verifyAfterUTXOBlock1(t *testing.T, d *RocksDB) {
 	if err := checkColumn(d, cfHeight, []keyPair{
-		keyPair{"000370d5", "0000000076fbbed90fd75b0e18856aa35baa984e9c9d444cf746ad85e94e2997"},
+		keyPair{"000370d5", "0000000076fbbed90fd75b0e18856aa35baa984e9c9d444cf746ad85e94e2997", nil},
 	}); err != nil {
 		{
 			t.Fatal(err)
@@ -198,10 +208,10 @@ func verifyAfterBlock1(t *testing.T, d *RocksDB) {
 	}
 	// the vout is encoded as signed varint, i.e. value * 2 for non negative values
 	if err := checkColumn(d, cfAddresses, []keyPair{
-		keyPair{addressToPubKeyHex("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d) + "000370d5", "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + "00"},
-		keyPair{addressToPubKeyHex("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "000370d5", "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + "02"},
-		keyPair{addressToPubKeyHex("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "00"},
-		keyPair{addressToPubKeyHex("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "02"},
+		keyPair{addressToPubKeyHex("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d) + "000370d5", "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + "00", nil},
+		keyPair{addressToPubKeyHex("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "000370d5", "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + "02", nil},
+		keyPair{addressToPubKeyHex("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "00", nil},
+		keyPair{addressToPubKeyHex("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "02", nil},
 	}); err != nil {
 		{
 			t.Fatal(err)
@@ -210,11 +220,38 @@ func verifyAfterBlock1(t *testing.T, d *RocksDB) {
 	if err := checkColumn(d, cfUnspentTxs, []keyPair{
 		keyPair{
 			"00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840",
-			addressToPubKeyHexWithLenght("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d) + "00" + addressToPubKeyHexWithLenght("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "02",
+			addressToPubKeyHexWithLength("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d) + "00" + addressToPubKeyHexWithLength("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "02",
+			nil,
 		},
 		keyPair{
 			"effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75",
-			addressToPubKeyHexWithLenght("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "00" + addressToPubKeyHexWithLenght("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "02",
+			addressToPubKeyHexWithLength("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "00" + addressToPubKeyHexWithLength("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "02",
+			nil,
+		},
+	}); err != nil {
+		{
+			t.Fatal(err)
+		}
+	}
+	// the values in cfBlockAddresses has random order, must use CompareFunc
+	if err := checkColumn(d, cfBlockAddresses, []keyPair{
+		keyPair{"000370d5", "",
+			func(v string) bool {
+				expected := []string{
+					addressToPubKeyHexWithLength("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d),
+					addressToPubKeyHexWithLength("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d),
+					addressToPubKeyHexWithLength("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d),
+					addressToPubKeyHexWithLength("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d),
+				}
+				for _, e := range expected {
+					lb := len(v)
+					v = strings.Replace(v, e, "", 1)
+					if lb == len(v) {
+						return false
+					}
+				}
+				return len(v) == 0
+			},
 		},
 	}); err != nil {
 		{
@@ -223,27 +260,27 @@ func verifyAfterBlock1(t *testing.T, d *RocksDB) {
 	}
 }
 
-func verifyAfterBlock2(t *testing.T, d *RocksDB) {
+func verifyAfterUTXOBlock2(t *testing.T, d *RocksDB) {
 	if err := checkColumn(d, cfHeight, []keyPair{
-		keyPair{"000370d5", "0000000076fbbed90fd75b0e18856aa35baa984e9c9d444cf746ad85e94e2997"},
-		keyPair{"000370d6", "00000000eb0443fd7dc4a1ed5c686a8e995057805f9a161d9a5a77a95e72b7b6"},
+		keyPair{"000370d5", "0000000076fbbed90fd75b0e18856aa35baa984e9c9d444cf746ad85e94e2997", nil},
+		keyPair{"000370d6", "00000000eb0443fd7dc4a1ed5c686a8e995057805f9a161d9a5a77a95e72b7b6", nil},
 	}); err != nil {
 		{
 			t.Fatal(err)
 		}
 	}
 	if err := checkColumn(d, cfAddresses, []keyPair{
-		keyPair{addressToPubKeyHex("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d) + "000370d5", "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + "00"},
-		keyPair{addressToPubKeyHex("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "000370d5", "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + "02"},
-		keyPair{addressToPubKeyHex("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "00"},
-		keyPair{addressToPubKeyHex("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "02"},
-		keyPair{addressToPubKeyHex("mzB8cYrfRwFRFAGTDzV8LkUQy5BQicxGhX", t, d) + "000370d6", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + "00" + "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + "01"},
-		keyPair{addressToPubKeyHex("mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL", t, d) + "000370d6", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + "02"},
-		keyPair{addressToPubKeyHex("mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC", t, d) + "000370d6", "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + "00"},
-		keyPair{addressToPubKeyHex("mmJx9Y8ayz9h14yd9fgCW1bUKoEpkBAquP", t, d) + "000370d6", "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + "02"},
-		keyPair{addressToPubKeyHex("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "000370d6", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + "01"},
-		keyPair{addressToPubKeyHex("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "000370d6", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + "03"},
-		keyPair{addressToPubKeyHex("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "000370d6", "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + "03"},
+		keyPair{addressToPubKeyHex("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d) + "000370d5", "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + "00", nil},
+		keyPair{addressToPubKeyHex("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "000370d5", "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + "02", nil},
+		keyPair{addressToPubKeyHex("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "00", nil},
+		keyPair{addressToPubKeyHex("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "02", nil},
+		keyPair{addressToPubKeyHex("mzB8cYrfRwFRFAGTDzV8LkUQy5BQicxGhX", t, d) + "000370d6", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + "00" + "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + "01", nil},
+		keyPair{addressToPubKeyHex("mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL", t, d) + "000370d6", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + "02", nil},
+		keyPair{addressToPubKeyHex("mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC", t, d) + "000370d6", "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + "00", nil},
+		keyPair{addressToPubKeyHex("mmJx9Y8ayz9h14yd9fgCW1bUKoEpkBAquP", t, d) + "000370d6", "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + "02", nil},
+		keyPair{addressToPubKeyHex("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "000370d6", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + "01", nil},
+		keyPair{addressToPubKeyHex("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "000370d6", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + "03", nil},
+		keyPair{addressToPubKeyHex("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "000370d6", "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + "03", nil},
 	}); err != nil {
 		{
 			t.Fatal(err)
@@ -252,15 +289,45 @@ func verifyAfterBlock2(t *testing.T, d *RocksDB) {
 	if err := checkColumn(d, cfUnspentTxs, []keyPair{
 		keyPair{
 			"00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840",
-			addressToPubKeyHexWithLenght("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d) + "00",
+			addressToPubKeyHexWithLength("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d) + "00",
+			nil,
 		},
 		keyPair{
 			"7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25",
-			addressToPubKeyHexWithLenght("mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL", t, d) + "02",
+			addressToPubKeyHexWithLength("mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL", t, d) + "02",
+			nil,
 		},
 		keyPair{
 			"3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71",
-			addressToPubKeyHexWithLenght("mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC", t, d) + "00" + addressToPubKeyHexWithLenght("mmJx9Y8ayz9h14yd9fgCW1bUKoEpkBAquP", t, d) + "02",
+			addressToPubKeyHexWithLength("mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC", t, d) + "00" + addressToPubKeyHexWithLength("mmJx9Y8ayz9h14yd9fgCW1bUKoEpkBAquP", t, d) + "02",
+			nil,
+		},
+	}); err != nil {
+		{
+			t.Fatal(err)
+		}
+	}
+	if err := checkColumn(d, cfBlockAddresses, []keyPair{
+		keyPair{"000370d6", "",
+			func(v string) bool {
+				expected := []string{
+					addressToPubKeyHexWithLength("mzB8cYrfRwFRFAGTDzV8LkUQy5BQicxGhX", t, d),
+					addressToPubKeyHexWithLength("mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL", t, d),
+					addressToPubKeyHexWithLength("mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC", t, d),
+					addressToPubKeyHexWithLength("mmJx9Y8ayz9h14yd9fgCW1bUKoEpkBAquP", t, d),
+					addressToPubKeyHexWithLength("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d),
+					addressToPubKeyHexWithLength("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d),
+					addressToPubKeyHexWithLength("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d),
+				}
+				for _, e := range expected {
+					lb := len(v)
+					v = strings.Replace(v, e, "", 1)
+					if lb == len(v) {
+						return false
+					}
+				}
+				return len(v) == 0
+			},
 		},
 	}); err != nil {
 		{
@@ -291,31 +358,39 @@ func verifyGetTransactions(t *testing.T, d *RocksDB, addr string, low, high uint
 	}
 }
 
+type testBitcoinParser struct {
+	*btc.BitcoinParser
+}
+
+// override btc.KeepBlockAddresses to keep only one blockaddress
+func (p *testBitcoinParser) KeepBlockAddresses() int {
+	return 1
+}
+
 // TestRocksDB_Index_UTXO is a composite test probing the whole indexing functionality for UTXO chains
 // It does the following:
 // 1) Connect two blocks (inputs from 2nd block are spending some outputs from the 1st block)
 // 2) GetTransactions for various addresses / low-high ranges
-// 3) Disconnect block 2
-// 4) GetTransactions for various addresses
-// 5) Connect the block 2 back
+// 3) Disconnect block 2 - expect error
+// 4) Disconnect the block 2 using full scan
 // After each step, the whole content of DB is examined and any difference against expected state is regarded as failure
 func TestRocksDB_Index_UTXO(t *testing.T) {
-	d := setupRocksDB(t, &btc.BitcoinParser{Params: btc.GetChainParams("test")})
+	d := setupRocksDB(t, &testBitcoinParser{BitcoinParser: &btc.BitcoinParser{Params: btc.GetChainParams("test")}})
 	defer closeAnddestroyRocksDB(t, d)
 
 	// connect 1st block - will log warnings about missing UTXO transactions in cfUnspentTxs column
-	block1 := getTestBlock1(t, d)
+	block1 := getTestUTXOBlock1(t, d)
 	if err := d.ConnectBlock(block1); err != nil {
 		t.Fatal(err)
 	}
-	verifyAfterBlock1(t, d)
+	verifyAfterUTXOBlock1(t, d)
 
 	// connect 2nd block - use some outputs from the 1st block as the inputs and 1 input uses tx from the same block
-	block2 := getTestBlock2(t, d)
+	block2 := getTestUTXOBlock2(t, d)
 	if err := d.ConnectBlock(block2); err != nil {
 		t.Fatal(err)
 	}
-	verifyAfterBlock2(t, d)
+	verifyAfterUTXOBlock2(t, d)
 
 	// get transactions for various addresses / low-high ranges
 	verifyGetTransactions(t, d, "mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", 0, 1000000, []txidVoutOutput{
@@ -334,4 +409,13 @@ func TestRocksDB_Index_UTXO(t *testing.T) {
 	}, nil)
 	verifyGetTransactions(t, d, "mtGXQvBowMkBpnhLckhxhbwYK44Gs9eBad", 500000, 1000000, []txidVoutOutput{}, errors.New("checksum mismatch"))
 
+	// DisconnectBlock for UTXO chains is not possible
+	err := d.DisconnectBlock(block2)
+	if err == nil || err.Error() != "DisconnectBlock is not supported for UTXO chains" {
+		t.Fatal(err)
+	}
+	verifyAfterUTXOBlock2(t, d)
+
+	// disconnect the 2nd block, verify that the db contains only the 1st block
+
 }

db/sync.go

@@ -382,6 +382,11 @@ func (w *SyncWorker) getBlockChain(out chan blockResult, done chan struct{}) {
 // otherwise doing full scan
 func (w *SyncWorker) DisconnectBlocks(lower uint32, higher uint32, hashes []string) error {
 	glog.Infof("sync: disconnecting blocks %d-%d", lower, higher)
+	// if the chain uses Block to Addresses mapping, always use DisconnectBlocksFullScan
+	// the full scan will be optimized using the mapping
+	if w.chain.GetChainParser().KeepBlockAddresses() > 0 {
+		return w.db.DisconnectBlocksFullScan(lower, higher)
+	}
 	blocks := make([]*bchain.Block, len(hashes))
 	var err error
 	// get all blocks first to see if we can avoid full scan