From 8cb238fd60ce0fe0368a17c825b28d5b8e1e49ec Mon Sep 17 00:00:00 2001 From: George Tankersley Date: Sat, 15 Dec 2018 01:27:24 -0500 Subject: [PATCH] storage: some minor storage & logging tweaks --- cmd/ingest/main.go | 11 ++++++----- storage/sqlite3.go | 17 +++++++++-------- storage/sqlite3_test.go | 40 ++++++++++++++++++++++++++++++++++++++-- 3 files changed, 53 insertions(+), 15 deletions(-) diff --git a/cmd/ingest/main.go b/cmd/ingest/main.go index 3b79667..87d5d2e 100644 --- a/cmd/ingest/main.go +++ b/cmd/ingest/main.go @@ -205,14 +205,14 @@ func handleBlock(db *sql.DB, sequence int, blockData []byte) { "block_hash": blockHash, "block_version": block.GetVersion(), "tx_count": block.GetTxCount(), - "has_sapling": block.HasSaplingTransactions(), + "sapling": block.HasSaplingTransactions(), "error": err, }) if err != nil { - entry.Error("error storing block") + entry.Error("new block") } else { - entry.Info("received new block") + entry.Info("new block") } for index, tx := range block.Transactions() { @@ -229,11 +229,12 @@ func handleBlock(db *sql.DB, sequence int, blockData []byte) { "block_height": block.GetHeight(), "block_hash": blockHash, "tx_index": index, - "has_sapling": tx.HasSaplingTransactions(), + "tx_size": len(tx.Bytes()), + "sapling": tx.HasSaplingTransactions(), "error": err, }) if err != nil { - entry.Error("error storing tx") + entry.Error("storing tx") } else { entry.Debug("storing tx") } diff --git a/storage/sqlite3.go b/storage/sqlite3.go index 831677f..f0fdf4b 100644 --- a/storage/sqlite3.go +++ b/storage/sqlite3.go @@ -27,9 +27,9 @@ func CreateTables(conn *sql.DB) error { blockTable := ` CREATE TABLE IF NOT EXISTS blocks ( - height INTEGER PRIMARY KEY, - hash TEXT, - has_sapling_tx BOOL, + block_height INTEGER PRIMARY KEY, + block_hash TEXT, + sapling BOOL, compact_encoding BLOB ); ` @@ -65,7 +65,7 @@ func GetCurrentHeight(ctx context.Context, db *sql.DB) (int, error) { func GetBlock(ctx context.Context, db *sql.DB, height int) ([]byte, error) { var blockBytes []byte // avoid a copy with *RawBytes - query := "SELECT compact_encoding from blocks WHERE height = ?" + query := "SELECT compact_encoding from blocks WHERE block_height = ?" err := db.QueryRowContext(ctx, query, height).Scan(&blockBytes) if err != nil { return nil, err @@ -75,7 +75,7 @@ func GetBlock(ctx context.Context, db *sql.DB, height int) ([]byte, error) { func GetBlockByHash(ctx context.Context, db *sql.DB, hash string) ([]byte, error) { var blockBytes []byte // avoid a copy with *RawBytes - query := "SELECT compact_encoding from blocks WHERE hash = ?" + query := "SELECT compact_encoding from blocks WHERE block_hash = ?" err := db.QueryRowContext(ctx, query, hash).Scan(&blockBytes) if err != nil { return nil, errors.Wrap(err, fmt.Sprintf("getting block with hash %s", hash)) @@ -92,7 +92,7 @@ func GetBlockRange(ctx context.Context, db *sql.DB, blockOut chan<- []byte, errO return } - query := "SELECT compact_encoding from blocks WHERE (height BETWEEN ? AND ?)" + query := "SELECT compact_encoding from blocks WHERE (block_height BETWEEN ? AND ?)" result, err := db.QueryContext(ctx, query, start, end) if err != nil { errOut <- err @@ -122,7 +122,7 @@ func GetBlockRange(ctx context.Context, db *sql.DB, blockOut chan<- []byte, errO } func StoreBlock(conn *sql.DB, height int, hash string, sapling bool, encoded []byte) error { - insertBlock := "INSERT INTO blocks (height, hash, has_sapling_tx, compact_encoding) values (?, ?, ?, ?)" + insertBlock := "INSERT INTO blocks (block_height, block_hash, sapling, compact_encoding) values (?, ?, ?, ?)" _, err := conn.Exec(insertBlock, height, hash, sapling, encoded) if err != nil { return errors.Wrap(err, fmt.Sprintf("storing compact block %d", height)) @@ -130,13 +130,14 @@ func StoreBlock(conn *sql.DB, height int, hash string, sapling bool, encoded []b currentHeight, err := GetCurrentHeight(context.Background(), conn) if err != nil || height > currentHeight { + // TODO database transactions! err = SetCurrentHeight(conn, height) } return err } func SetCurrentHeight(conn *sql.DB, height int) error { - update := "UPDATE state SET current_height=?, timestamp=CURRENT_TIMESTAMP WHERE rowid = 1" + update := "UPDATE state SET current_height=? WHERE rowid = 1" result, err := conn.Exec(update, height) if err != nil { return errors.Wrap(err, "updating state row") diff --git a/storage/sqlite3_test.go b/storage/sqlite3_test.go index e7f0777..f9fae66 100644 --- a/storage/sqlite3_test.go +++ b/storage/sqlite3_test.go @@ -98,16 +98,17 @@ func TestSqliteStorage(t *testing.T) { } lastBlockTest := compactTests[len(compactTests)-1] + if blockHeight != lastBlockTest.BlockHeight { t.Errorf("Wrong block height, got: %d", blockHeight) } - retBlock, err := GetBlock(ctx, db, blockHeight) + storedBlock, err := GetBlock(ctx, db, blockHeight) if err != nil { t.Error(errors.Wrap(err, "retrieving stored block")) } cblock := &rpc.CompactBlock{} - err = proto.Unmarshal(retBlock, cblock) + err = proto.Unmarshal(storedBlock, cblock) if err != nil { t.Fatal(err) } @@ -194,6 +195,41 @@ func TestSqliteStorage(t *testing.T) { if count > 0 { t.Errorf("got some blocks that shouldn't be there") } + } + + // Transaction storage + { + blockData, _ := hex.DecodeString(compactTests[0].Full) + block := parser.NewBlock() + _, _ = block.ParseFromSlice(blockData) + tx := block.Transactions()[0] + + blockHash := hex.EncodeToString(block.GetEncodableHash()) + txHash := hex.EncodeToString(tx.GetEncodableHash()) + err = StoreTransaction( + db, + block.GetHeight(), + blockHash, + 0, + txHash, + tx.Bytes(), + ) + + if err != nil { + t.Error(err) + } + + var storedBytes []byte + getTx := "SELECT tx_bytes FROM transactions WHERE tx_hash = ?" + err = db.QueryRow(getTx, txHash).Scan(&storedBytes) + if err != nil { + t.Error(errors.Wrap(err, fmt.Sprintf("error getting a full transaction"))) + } + + if len(storedBytes) != len(tx.Bytes()) { + t.Errorf("Wrong tx size, want %d got %d", len(tx.Bytes()), storedBytes) + } } + }