storage: begin implementing sqlite3 storage

go.mod

@@ -3,6 +3,7 @@ module github.com/gtank/ctxd
 require (
 	github.com/dvyukov/go-fuzz v0.0.0-20180902053217-4aff8368ef19 // indirect
 	github.com/golang/protobuf v1.2.0
+	github.com/mattn/go-sqlite3 v1.10.0
 	github.com/pebbe/zmq4 v1.0.0
 	github.com/pkg/errors v0.8.0
 	golang.org/x/net v0.0.0-20180926154720-4dfa2610cdf3

go.sum

@@ -9,6 +9,8 @@ github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb
 github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK860o=
+github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/pebbe/zmq4 v1.0.0 h1:D+MSmPpqkL5PSSmnh8g51ogirUCyemThuZzLW7Nrt78=
 github.com/pebbe/zmq4 v1.0.0/go.mod h1:7N4y5R18zBiu3l0vajMUWQgZyjv464prE8RCyBcmnZM=
 github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=

parser/block.go

@@ -35,6 +35,15 @@ func (b *block) getEncodableHash() []byte {
 	return b.hdr.getEncodableHash()
 }
 
+func (b *block) HasSaplingTransactions() bool {
+	for _, tx := range b.vtx {
+		if tx.HasSaplingTransactions() {
+			return true
+		}
+	}
+	return false
+}
+
 // GetHeight() extracts the block height from the coinbase transaction. See
 // BIP34. Returns block height on success, or -1 on error.
 func (b *block) GetHeight() int {

parser/block_test.go

@@ -6,7 +6,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
-	"log"
 	"os"
 	"testing"
 
@@ -57,12 +56,12 @@ func TestCompactBlocks(t *testing.T) {
 
 	blockJSON, err := ioutil.ReadFile("testdata/compact_blocks.json")
 	if err != nil {
-		log.Fatal(err)
+		t.Fatal(err)
 	}
 
 	err = json.Unmarshal(blockJSON, &compactTests)
 	if err != nil {
-		log.Fatal(err)
+		t.Fatal(err)
 	}
 
 	for _, test := range compactTests {

parser/transaction.go

@@ -296,6 +296,10 @@ func (tx *transaction) getEncodableHash() []byte {
 	return digest[:]
 }
 
+func (tx *transaction) HasSaplingTransactions() bool {
+	return tx.version >= 4 && (len(tx.shieldedSpends)+len(tx.shieldedOutputs)) > 0
+}
+
 func (tx *transaction) ToCompact(index int) *proto.CompactTx {
 	ctx := &proto.CompactTx{
 		Index:   uint64(index), // index is contextual

storage/sqlite3.go

@@ -0,0 +1,17 @@
+package storage
+
+import "database/sql"
+
+func createBlockTable(conn *sql.DB) error {
+	tableCreation := `
+		CREATE TABLE IF NOT EXISTS blocks (
+			height INTEGER,
+			hash TEXT,
+			has_sapling_tx BOOL,
+			compact_encoding BLOB,
+			PRIMARY KEY (height, hash)
+		);
+	`
+	_, err := conn.Exec(tableCreation)
+	return err
+}

storage/sqlite3_test.go

@@ -0,0 +1,85 @@
+package storage
+
+import (
+	"database/sql"
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"testing"
+
+	protobuf "github.com/golang/protobuf/proto"
+	"github.com/gtank/ctxd/parser"
+	_ "github.com/mattn/go-sqlite3"
+	"github.com/pkg/errors"
+)
+
+func TestMain(m *testing.M) {
+	conn, err := sql.Open("sqlite3", "testdata/blocks.db")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	err = createBlockTable(conn)
+	if err != nil {
+		conn.Close()
+		log.Fatal(err)
+	}
+
+	ret := m.Run()
+	conn.Close()
+	//os.Remove("testdata/blocks.db")
+	os.Exit(ret)
+}
+
+func TestFillDB(t *testing.T) {
+	type compactTest struct {
+		BlockHeight int    `json:"block"`
+		BlockHash   string `json:"hash"`
+		Full        string `json:"full"`
+		Compact     string `json:"compact"`
+	}
+	var compactTests []compactTest
+
+	blockJSON, err := ioutil.ReadFile("testdata/compact_blocks.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = json.Unmarshal(blockJSON, &compactTests)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	conn, err := sql.Open("sqlite3", "testdata/blocks.db")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer conn.Close()
+
+	for _, test := range compactTests {
+		blockData, _ := hex.DecodeString(test.Full)
+		block := parser.NewBlock()
+		blockData, err = block.ParseFromSlice(blockData)
+		if err != nil {
+			t.Error(errors.Wrap(err, fmt.Sprintf("parsing testnet block %d", test.BlockHeight)))
+			continue
+		}
+
+		height := block.GetHeight()
+		hash := hex.EncodeToString(block.GetHash())
+		hasSapling := block.HasSaplingTransactions()
+		marshaled, _ := protobuf.Marshal(block.ToCompact())
+		compactB64 := base64.RawStdEncoding.EncodeToString(marshaled)
+
+		insertBlock := "INSERT INTO blocks (height, hash, has_sapling_tx, compact_encoding) values (?, ?, ?, ?)"
+		_, err := conn.Exec(insertBlock, height, hash, hasSapling, compactB64)
+		if err != nil {
+			t.Error(errors.Wrap(err, fmt.Sprintf("storing compact block %d", height)))
+			continue
+		}
+	}
+}