Swap main database to bolt; refactor sp list sorting

README.md

@@ -6,8 +6,8 @@ remotely through encrypted transportation. All of which is deployable locally or
 platforms such as GCP and AWS.
 
 It is still currently in development. However, the server is already capable of serving a gRPC
-based API using mutual TLS encryption, backed by Redis and Docker. It is also capable of being
+based API using mutual TLS encryption, backed by BoltDB. It is also capable
-deployed in a semi-automated fashion locally and to GCP thanks to Docker.
+of being deployed in a semi-automated fashion locally and to GCP thanks to Docker.
 
 **Server Roadmap**
 
@@ -64,6 +64,6 @@ using GUIs, with all the same safety and encryption as the CLI.
 - Go-Kit: services
 - gRPC/Protobuf: all
 - Cobra Commander & Viper Config: sp
-- Redis: services
+- BoltDB/Redis: services
 - Docker: services
 - Debian: docker images & machines

services/Makefile

@@ -31,7 +31,6 @@ machine-create-google:
 	    --google-username selfpass \
 	    --google-zone us-west1-c \
 	    selfpass01
-	$(MAKE) machine-put-redis.conf
 	$(MAKE) machine-put-data
 	$(MAKE) machine-install-stackdriver-agent
 	$(MAKE) machine-add-grpc-server-tag
@@ -42,14 +41,11 @@ machine-rm:
 machine-ssh:
 	docker-machine ssh selfpass01
 
-machine-put-redis.conf:
-	docker-machine scp ./redis.conf selfpass01:redis.conf
-
 machine-put-data:
-	docker-machine scp ./data/appendonly.aof selfpass01:data/
+	docker-machine scp ./data/bolt.db selfpass01:data
 
 machine-get-data:
-	docker-machine scp selfpass01:data/appendonly.aof ./data/
+	docker-machine scp selfpass01:data/bolt.db ./data
 
 machine-add-grpc-server-tag:
 	gcloud compute instances add-tags selfpass01 \

services/cmd/main.go

@@ -57,7 +57,7 @@ func main() {
 		},
 	})
 
-	db, err := repositories.NewRedisConn("tcp", "redis:6379", 2)
+	db, err := repositories.OpenBoltDB("/home/selfpass/data/bolt.db", 0600, nil)
 	check(err)
 
 	var svc types.Service

services/credentials/repositories/bolt.go

@@ -0,0 +1,139 @@
+package repositories
+
+import (
+	"bytes"
+	"context"
+	"encoding/gob"
+	"os"
+	"sync"
+
+	"go.etcd.io/bbolt"
+
+	"github.com/mitchell/selfpass/services/credentials/types"
+)
+
+func OpenBoltDB(file string, mode os.FileMode, opts *bbolt.Options) (out BoltDB, err error) {
+	db, err := bbolt.Open(file, mode, opts)
+	if err != nil {
+		return out, err
+	}
+
+	return BoltDB{bolt: db}, nil
+}
+
+type BoltDB struct {
+	bolt *bbolt.DB
+}
+
+func (db BoltDB) GetAllMetadata(ctx context.Context, sourceHost string, errch chan<- error) (output <-chan types.Metadata) {
+	mdch := make(chan types.Metadata, 1)
+
+	go func() {
+		defer close(mdch)
+
+		err := db.bolt.View(func(tx *bbolt.Tx) error {
+			bkt := tx.Bucket([]byte(credentialsBkt))
+			if bkt == nil {
+				return nil
+			}
+
+			var wg sync.WaitGroup
+			err := bkt.ForEach(func(_, value []byte) error {
+				wg.Add(1)
+
+				go func(value []byte) {
+					defer wg.Done()
+
+					var cred types.Credential
+
+					err := gobUnmarshal(value, &cred)
+					if err != nil {
+						errch <- err
+						return
+					}
+
+					if sourceHost == "" || sourceHost == cred.SourceHost {
+						mdch <- cred.Metadata
+					}
+				}(value)
+
+				return nil
+			})
+			if err != nil {
+				return err
+			}
+
+			wg.Wait()
+
+			return nil
+		})
+		if err != nil {
+			errch <- err
+			return
+		}
+	}()
+
+	return mdch
+}
+
+func (db BoltDB) Get(ctx context.Context, id string) (output types.Credential, err error) {
+	err = db.bolt.View(func(tx *bbolt.Tx) error {
+		bkt := tx.Bucket([]byte(credentialsBkt))
+		if bkt == nil {
+			return nil
+		}
+
+		value := bkt.Get([]byte(id))
+		if value == nil {
+			return nil
+		}
+
+		return gobUnmarshal(value, &output)
+	})
+
+	return output, err
+}
+
+func (db BoltDB) Put(ctx context.Context, c types.Credential) (err error) {
+	err = db.bolt.Update(func(tx *bbolt.Tx) error {
+		bkt, err := tx.CreateBucketIfNotExists([]byte(credentialsBkt))
+		if err != nil {
+			return err
+		}
+
+		value, err := gobMarshal(c)
+		if err != nil {
+			return err
+		}
+
+		return bkt.Put([]byte(c.ID), value)
+	})
+
+	return err
+}
+
+func (db BoltDB) Delete(ctx context.Context, id string) (err error) {
+	err = db.bolt.Update(func(tx *bbolt.Tx) error {
+		bkt := tx.Bucket([]byte(credentialsBkt))
+		if bkt == nil {
+			return nil
+		}
+
+		return bkt.Delete([]byte(id))
+	})
+
+	return err
+}
+
+const credentialsBkt = "credentials"
+
+func gobMarshal(v interface{}) (bs []byte, err error) {
+	buf := bytes.NewBuffer(nil)
+	err = gob.NewEncoder(buf).Encode(v)
+	return buf.Bytes(), err
+}
+
+func gobUnmarshal(bs []byte, v interface{}) error {
+	buf := bytes.NewReader(bs)
+	return gob.NewDecoder(buf).Decode(v)
+}

services/docker-compose.prod.yml

@@ -1,9 +1,7 @@
 version: "3.7"
 services:
-  redis:
-    volumes:
-      - "/home/selfpass/data:/data"
-      - "/home/selfpass/redis.conf:/redis.conf"
   server:
     entrypoint:
       - server
+    volumes:
+      - "/home/selfpass/data:/home/selfpass/data"

services/docker-compose.yml

@@ -1,14 +1,5 @@
 version: "3.7"
 services:
-  redis:
-    image: "redis:5.0.5"
-    restart: on-failure
-    command:
-      - redis-server
-      - /redis.conf
-    volumes:
-      - "./redis.conf:/redis.conf"
-      - "./data:/data"
   server:
     build: .
     restart: on-failure
@@ -17,5 +8,5 @@ services:
       - -v
     ports:
       - "8080:8080"
-    depends_on:
+    volumes:
-      - redis
+      - "./data:/home/selfpass/data"

services/go.mod

@@ -3,7 +3,6 @@ module github.com/mitchell/selfpass/services
 go 1.12
 
 require (
-	github.com/etcd-io/bbolt v1.3.3
 	github.com/go-kit/kit v0.9.0
 	github.com/go-logfmt/logfmt v0.4.0 // indirect
 	github.com/go-stack/stack v1.8.0 // indirect
@@ -11,7 +10,7 @@ require (
 	github.com/mediocregopher/radix/v3 v3.3.0
 	github.com/mitchell/selfpass/protobuf/go v0.0.0-00010101000000-000000000000
 	github.com/spf13/pflag v1.0.3
-	go.etcd.io/bbolt v1.3.3 // indirect
+	go.etcd.io/bbolt v1.3.3
 	google.golang.org/grpc v1.22.0
 )
 

services/go.sum

@@ -3,8 +3,6 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/etcd-io/bbolt v1.3.3 h1:gSJmxrs37LgTqR/oyJBWok6k6SvXEUerFTbltIhXkBM=
-github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw=
 github.com/go-kit/kit v0.9.0 h1:wDJmvq38kDhkVxi50ni9ykkdUr1PKgqKOoi01fa0Mdk=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA=

services/migrations/201907191_redis_to_bolt.go

@@ -6,9 +6,9 @@ import (
 	"fmt"
 	"sync"
 
-	"github.com/etcd-io/bbolt"
 	"github.com/mediocregopher/radix/v3"
 	"github.com/spf13/pflag"
+	"go.etcd.io/bbolt"
 
 	"github.com/mitchell/selfpass/services/credentials/types"
 	"github.com/mitchell/selfpass/services/migrations/migration"

sp/commands/commands.go

@@ -81,6 +81,8 @@ receive:
 		keyIDMap[key] = md.ID
 	}
 
+	sort.Strings(keys)
+
 	prompt = &survey.Select{
 		Message:  "Primary user key (and tag):",
 		Options:  keys,

sp/commands/list.go

@@ -65,6 +65,10 @@ includes almost all the information but the most sensitive.`,
 			var source string
 			check(survey.AskOne(prompt, &source, nil))
 
+			sort.Slice(mds[source], func(i, j int) bool {
+				return mds[source][i].Primary < mds[source][j].Primary
+			})
+
 			for _, md := range mds[source] {
 				fmt.Println(md)
 			}

sp/go.sum

@@ -225,6 +225,7 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/yunify/qingstor-sdk-go v2.2.15+incompatible/go.mod h1:w6wqLDQ5bBTzxGJ55581UrSwLrsTAsdo9N6yX/8d9RY=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk=
 go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
 go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=