Merge branch 'master' into remove_0mq

.gitignore

@@ -0,0 +1,6 @@
+*.conf
+*.config
+*.log
+*.sqlite
+*.pem
+*.key

.gitlab-ci.yml

@@ -0,0 +1,109 @@
+# /************************************************************************
+ #  File: .gitlab-ci.yml
+ #  Author: mdr0id
+ #  Date: 7/16/2019
+ #  Description:  Used to setup runners/jobs for lightwalletd
+ #  Usage: Commit source and the pipeline will trigger the according jobs.
+ #         
+ #  Known bugs/missing features:
+ #
+ #  IMPORTANT NOTE: any job with preceeding '.'' is ignored in pipeline 
+ # ************************************************************************/
+ 
+image: golang:1.11-alpine
+  
+stages:
+  - build
+  - test
+  - deploy
+  - monitor
+
+# ************************************************************************/
+# BUILD
+# ************************************************************************/
+.lint-check:
+  stage: build
+  script:
+    - make lint
+
+.build-docs:
+  stage: build
+  script:
+    - made docs
+
+build-linux:
+  stage: build
+  script:
+    - make
+
+.build-windows:
+  stage: build
+  script:
+    - make
+
+.build-mac:
+  stage: build
+  script:
+    - make
+
+# Build against latest Golang
+.build-latest:
+  stage: build
+  image: golang:latest-alpine
+  script:
+    - make
+  allow_failure: true 
+
+# Likely don't need a nightly in always deployable model; will revisit after requirement gathering
+.build-nightly:
+  stage: build
+  script:
+    - make
+  allow_failure: true
+
+# ************************************************************************/
+# TEST
+# ************************************************************************/
+test-coverage:
+  stage: test
+  script:
+    - make coverage
+  allow_failure: true  
+
+test-coverage-report:
+  stage: test
+  script:
+    - make coverage_report
+  allow_failure: true
+
+test-coverage-report-html:
+  stage: test
+  script:
+    - make coverage_html
+  allow_failure: true
+
+# ************************************************************************/
+# DEPLOY
+# ************************************************************************/
+.release-candidate:
+  stage: deploy
+  script:
+    - echo "Generating v0.0.1-rc"
+  when: manual
+
+.release-production:
+  stage: deploy
+  script:
+    - echo "Generating v0.0.1"
+  when: manual
+
+
+# ************************************************************************/
+# MONITOR
+# ************************************************************************/
+.monitor-release:
+  stage: deploy
+  script:
+    - echo "Building docker image for v0.0.0"
+    - make image
+  when: manual

Makefile

@@ -0,0 +1,77 @@
+# /************************************************************************
+ #  File: Makefile
+ #  Author: mdr0id
+ #  Date: 7/16/2019
+ #  Description:  Used for local and container dev in CI deployments
+ #  Usage: make <target_name>
+ #
+ #  Known bugs/missing features:
+ #
+ # ************************************************************************/
+PROJECT_NAME := "lightwalletd"
+GO_FILES := $(shell find . -name '*.go' | grep -v /vendor/ | grep -v '*_test.go')
+GO_TEST_FILES := $(shell find . -name '*_test.go' -type f | rev | cut -d "/" -f2- | rev | sort -u)
+GO_BUILD_FILES := $(shell find . -name 'main.go')
+
+#PKG_LIST := $(shell go list | grep -v /vendor/)
+#GO_COV_REPORTS :=
+
+.PHONY: all dep build clean test coverage coverhtml lint
+
+all: build
+
+# Lint golang files
+lint:
+	@golint -set_exit_status
+
+show_tests:
+	@echo ${GO_TEST_FILES}
+
+# Run unittests
+test:
+	@go test -v -short ${GO_TEST_FILES}
+
+# Run data race detector
+race: dep
+	@go test -v -race -short ${GO_TEST_FILES}
+
+# Run memory sanitizer (need to ensure proper build flag is set)
+msan: dep
+	@go test -v -msan -short ${GO_TEST_FILES}
+
+# Generate global code coverage report
+coverage:
+	@go test -coverprofile=coverage.out ${GO_TEST_FILES}
+
+# Generate code coverage report
+coverage_report:
+	@go tool cover -func=coverage.out 
+
+# Generate code coverage report in HTML
+coverage_html: 
+	@go tool cover -html=coverage.out
+
+# Generate documents
+docs:
+	@echo "Generating docs..."
+
+# Generate docker image
+image:
+	@echo "Building lightwalletd image..."
+
+# Get dependencies
+dep:
+	@go get -v -d ./...
+
+# Build binary
+build:
+	@go build -i -v ./cmd/ingest
+	@go build -i -v ./cmd/server
+
+# Install binaries into Go path
+install:
+	go install ./...
+
+clean:
+	@echo "clean project..."
+	#rm -f $(PROJECT_NAME)

README.md

@@ -81,7 +81,9 @@ To see the other command line options, run `go run cmd/server/main.go --help`.
 
 **What should I watch out for?**
 
-Not much! This is a very simple piece of software. Make sure you point it at the same storage as the ingester. See the "Production" section for some caveats.
+x509 Certificates! This software relies on the confidentiality and integrity of a modern TLS connection between incoming clients and the front-end. Without an x509 certificate that incoming clients accurately authenticate, the security properties of this software are lost.
+
+Otherwise, not much! This is a very simple piece of software. Make sure you point it at the same storage as the ingester. See the "Production" section for some caveats.
 
 Support for users sending transactions will require the ability to make JSON-RPC calls to a zcashd instance. By default the frontend tries to pull RPC credentials from your zcashd.conf file, but you can specify other credentials via command line flag. In the future, it should be possible to do this with environment variables [(#2)](https://github.com/zcash-hackworks/lightwalletd/issues/2).
 
@@ -103,6 +105,30 @@ sqlite is extremely reliable for what it is, but it isn't good at high concurren
 
 ⚠️ This is informational documentation about a piece of alpha software. It has not yet undergone audits or been subject to rigorous testing. It lacks some affordances necessary for production-level reliability. We do not recommend using it to handle customer funds at this time (March 2019).
 
+**x509 Certificates**
+You will need to supply an x509 certificate that connecting clients will have good reason to trust (hint: do not use a self-signed one, our SDK will reject those unless you distribute them to the client out-of-band). We suggest that you be sure to buy a reputable one from a supplier that uses a modern hashing algorithm (NOT md5 or sha1) and that uses Certificate Transparency (OID 1.3.6.1.4.1.11129.2.4.2 will be present in the certificate).
+
+To check a given certificate's (cert.pem) hashing algorithm:
+```
+openssl x509 -text -in certificate.crt | grep "Signature Algorithm"
+```
+
+To check if a given certificate (cert.pem) contains a Certificate Transparency OID:
+```
+echo "1.3.6.1.4.1.11129.2.4.2 certTransparency Certificate Transparency" > oid.txt
+openssl asn1parse -in cert.pem -oid ./oid.txt | grep 'Certificate Transparency'
+```
+
+To use Let's Encrypt to generate a free certificate for your frontend, one method is to:
+1) Install certbot
+2) Open port 80 to your host
+3) Point some forward dns to that host (some.forward.dns.com)
+4) Run
+```
+certbot certonly --standalone --preferred-challenges http -d some.forward.dns.com
+```
+5) Pass the resulting certificate and key to frontend using the -tls-cert and -tls-key options.
+
 **Dependencies**
 
 The first-order dependencies of this code are:

cmd/ingest/main.go

@@ -208,7 +208,7 @@ func getBlock(rpcClient *rpcclient.Client, height int) (*parser.Block, error) {
 	if err != nil{
 		return nil, errors.Wrap(err, "error reading JSON response")
 	}
-
+  
 	blockData, err := hex.DecodeString(blockDataHex)
 	if err != nil {
 		return nil, errors.Wrap(err, "error decoding getblock output")

cmd/server/main.go

@@ -5,7 +5,8 @@ import (
 	"flag"
 	"net"
 	"os"
-	"os/signal"
+	"fmt"
+ 	"os/signal"
 	"syscall"
 	"time"
 
@@ -29,9 +30,15 @@ func init() {
 		DisableLevelTruncation: true,
 	})
 
+	onexit := func () {
+		fmt.Printf("Lightwalletd died with a Fatal error. Check logfile for details.\n")
+	}
+
 	log = logger.WithFields(logrus.Fields{
 		"app": "frontend-grpc",
 	})
+
+	logrus.RegisterExitHandler(onexit)
 }
 
 // TODO stream logging
@@ -75,31 +82,55 @@ func loggerFromContext(ctx context.Context) *logrus.Entry {
 }
 
 type Options struct {
-	bindAddr      string `json:"bind_address,omitempty"`
-	dbPath        string `json:"db_path"`
-	tlsCertPath   string `json:"tls_cert_path,omitempty"`
-	tlsKeyPath    string `json:"tls_cert_key,omitempty"`
-	logLevel      uint64 `json:"log_level,omitempty"`
-	logPath       string `json:"log_file,omitempty"`
-	zcashConfPath string `json:"zcash_conf,omitempty"`
+	bindAddr      string  `json:"bind_address,omitempty"`
+	dbPath        string  `json:"db_path"`
+	tlsCertPath   string  `json:"tls_cert_path,omitempty"`
+	tlsKeyPath    string  `json:"tls_cert_key,omitempty"`
+	logLevel      uint64  `json:"log_level,omitempty"`
+	logPath       string  `json:"log_file,omitempty"`
+	zcashConfPath string  `json:"zcash_conf,omitempty"`
+	veryInsecure  bool    `json:"very_insecure,omitempty"`
+}
+
+func fileExists(filename string) bool {
+    info, err := os.Stat(filename)
+    if os.IsNotExist(err) {
+        return false
+    }
+    return !info.IsDir()
 }
 
 func main() {
 	opts := &Options{}
 	flag.StringVar(&opts.bindAddr, "bind-addr", "127.0.0.1:9067", "the address to listen on")
-	flag.StringVar(&opts.dbPath, "db-path", "", "the path to a sqlite database file")
-	flag.StringVar(&opts.tlsCertPath, "tls-cert", "", "the path to a TLS certificate (optional)")
-	flag.StringVar(&opts.tlsKeyPath, "tls-key", "", "the path to a TLS key file (optional)")
+	flag.StringVar(&opts.dbPath, "db-path", "./database.sqlite", "the path to a sqlite database file")
+	flag.StringVar(&opts.tlsCertPath, "tls-cert", "./cert.pem", "the path to a TLS certificate")
+	flag.StringVar(&opts.tlsKeyPath, "tls-key", "./cert.key", "the path to a TLS key file")
 	flag.Uint64Var(&opts.logLevel, "log-level", uint64(logrus.InfoLevel), "log level (logrus 1-7)")
-	flag.StringVar(&opts.logPath, "log-file", "", "log file to write to")
-	flag.StringVar(&opts.zcashConfPath, "conf-file", "", "conf file to pull RPC creds from")
+	flag.StringVar(&opts.logPath, "log-file", "./server.log", "log file to write to")
+	flag.StringVar(&opts.zcashConfPath, "conf-file", "./zcash.conf", "conf file to pull RPC creds from")
+	flag.BoolVar(&opts.veryInsecure, "very-insecure", false, "run without the required TLS certificate, only for debugging, DO NOT use in production")
 	// TODO prod metrics
 	// TODO support config from file and env vars
 	flag.Parse()
 
-	if opts.dbPath == "" || opts.zcashConfPath == "" {
-		flag.Usage()
-		os.Exit(1)
+	filesThatShouldExist := []string {
+		opts.dbPath,
+		opts.tlsCertPath,
+		opts.tlsKeyPath,
+		opts.logPath,
+		opts.zcashConfPath,
+	}
+
+	for _, filename := range filesThatShouldExist {
+		if (opts.veryInsecure && (filename == opts.tlsCertPath || filename == opts.tlsKeyPath)) {
+			continue
+		}
+		if !fileExists(filename) {
+			os.Stderr.WriteString(fmt.Sprintf("\n  ** File does not exist: %s\n\n", filename))
+			flag.Usage()
+			os.Exit(1)
+		}
 	}
 
 	if opts.logPath != "" {
@@ -132,7 +163,9 @@ func main() {
 		}
 		server = grpc.NewServer(grpc.Creds(transportCreds), LoggingInterceptor())
 	} else {
-		server = grpc.NewServer(LoggingInterceptor())
+		if (opts.veryInsecure) {
+			server = grpc.NewServer(LoggingInterceptor())
+		}
 	}
 
 	// Enable reflection for debugging