Swap main database to bolt; refactor sp list sorting

services/Makefile

@@ -31,7 +31,6 @@ machine-create-google:
 	    --google-username selfpass \
 	    --google-zone us-west1-c \
 	    selfpass01
-	$(MAKE) machine-put-redis.conf
 	$(MAKE) machine-put-data
 	$(MAKE) machine-install-stackdriver-agent
 	$(MAKE) machine-add-grpc-server-tag
@@ -42,14 +41,11 @@ machine-rm:
 machine-ssh:
 	docker-machine ssh selfpass01
 
-machine-put-redis.conf:
-	docker-machine scp ./redis.conf selfpass01:redis.conf
-
 machine-put-data:
-	docker-machine scp ./data/appendonly.aof selfpass01:data/
+	docker-machine scp ./data/bolt.db selfpass01:data
 
 machine-get-data:
-	docker-machine scp selfpass01:data/appendonly.aof ./data/
+	docker-machine scp selfpass01:data/bolt.db ./data
 
 machine-add-grpc-server-tag:
 	gcloud compute instances add-tags selfpass01 \

services/cmd/main.go

@@ -57,7 +57,7 @@ func main() {
 		},
 	})
 
-	db, err := repositories.NewRedisConn("tcp", "redis:6379", 2)
+	db, err := repositories.OpenBoltDB("/home/selfpass/data/bolt.db", 0600, nil)
 	check(err)
 
 	var svc types.Service

services/credentials/repositories/bolt.go

@@ -0,0 +1,139 @@
+package repositories
+
+import (
+	"bytes"
+	"context"
+	"encoding/gob"
+	"os"
+	"sync"
+
+	"go.etcd.io/bbolt"
+
+	"github.com/mitchell/selfpass/services/credentials/types"
+)
+
+func OpenBoltDB(file string, mode os.FileMode, opts *bbolt.Options) (out BoltDB, err error) {
+	db, err := bbolt.Open(file, mode, opts)
+	if err != nil {
+		return out, err
+	}
+
+	return BoltDB{bolt: db}, nil
+}
+
+type BoltDB struct {
+	bolt *bbolt.DB
+}
+
+func (db BoltDB) GetAllMetadata(ctx context.Context, sourceHost string, errch chan<- error) (output <-chan types.Metadata) {
+	mdch := make(chan types.Metadata, 1)
+
+	go func() {
+		defer close(mdch)
+
+		err := db.bolt.View(func(tx *bbolt.Tx) error {
+			bkt := tx.Bucket([]byte(credentialsBkt))
+			if bkt == nil {
+				return nil
+			}
+
+			var wg sync.WaitGroup
+			err := bkt.ForEach(func(_, value []byte) error {
+				wg.Add(1)
+
+				go func(value []byte) {
+					defer wg.Done()
+
+					var cred types.Credential
+
+					err := gobUnmarshal(value, &cred)
+					if err != nil {
+						errch <- err
+						return
+					}
+
+					if sourceHost == "" || sourceHost == cred.SourceHost {
+						mdch <- cred.Metadata
+					}
+				}(value)
+
+				return nil
+			})
+			if err != nil {
+				return err
+			}
+
+			wg.Wait()
+
+			return nil
+		})
+		if err != nil {
+			errch <- err
+			return
+		}
+	}()
+
+	return mdch
+}
+
+func (db BoltDB) Get(ctx context.Context, id string) (output types.Credential, err error) {
+	err = db.bolt.View(func(tx *bbolt.Tx) error {
+		bkt := tx.Bucket([]byte(credentialsBkt))
+		if bkt == nil {
+			return nil
+		}
+
+		value := bkt.Get([]byte(id))
+		if value == nil {
+			return nil
+		}
+
+		return gobUnmarshal(value, &output)
+	})
+
+	return output, err
+}
+
+func (db BoltDB) Put(ctx context.Context, c types.Credential) (err error) {
+	err = db.bolt.Update(func(tx *bbolt.Tx) error {
+		bkt, err := tx.CreateBucketIfNotExists([]byte(credentialsBkt))
+		if err != nil {
+			return err
+		}
+
+		value, err := gobMarshal(c)
+		if err != nil {
+			return err
+		}
+
+		return bkt.Put([]byte(c.ID), value)
+	})
+
+	return err
+}
+
+func (db BoltDB) Delete(ctx context.Context, id string) (err error) {
+	err = db.bolt.Update(func(tx *bbolt.Tx) error {
+		bkt := tx.Bucket([]byte(credentialsBkt))
+		if bkt == nil {
+			return nil
+		}
+
+		return bkt.Delete([]byte(id))
+	})
+
+	return err
+}
+
+const credentialsBkt = "credentials"
+
+func gobMarshal(v interface{}) (bs []byte, err error) {
+	buf := bytes.NewBuffer(nil)
+	err = gob.NewEncoder(buf).Encode(v)
+	return buf.Bytes(), err
+}
+
+func gobUnmarshal(bs []byte, v interface{}) error {
+	buf := bytes.NewReader(bs)
+	return gob.NewDecoder(buf).Decode(v)
+}

services/docker-compose.prod.yml

@@ -1,9 +1,7 @@
 version: "3.7"
 services:
-  redis:
-    volumes:
-      - "/home/selfpass/data:/data"
-      - "/home/selfpass/redis.conf:/redis.conf"
   server:
     entrypoint:
       - server
+    volumes:
+      - "/home/selfpass/data:/home/selfpass/data"

services/docker-compose.yml

@@ -1,14 +1,5 @@
 version: "3.7"
 services:
-  redis:
-    image: "redis:5.0.5"
-    restart: on-failure
-    command:
-      - redis-server
-      - /redis.conf
-    volumes:
-      - "./redis.conf:/redis.conf"
-      - "./data:/data"
   server:
     build: .
     restart: on-failure
@@ -17,5 +8,5 @@ services:
       - -v
     ports:
       - "8080:8080"
-    depends_on:
-      - redis
+    volumes:
+      - "./data:/home/selfpass/data"

services/go.mod

@@ -3,7 +3,6 @@ module github.com/mitchell/selfpass/services
 go 1.12
 
 require (
-	github.com/etcd-io/bbolt v1.3.3
 	github.com/go-kit/kit v0.9.0
 	github.com/go-logfmt/logfmt v0.4.0 // indirect
 	github.com/go-stack/stack v1.8.0 // indirect
@@ -11,7 +10,7 @@ require (
 	github.com/mediocregopher/radix/v3 v3.3.0
 	github.com/mitchell/selfpass/protobuf/go v0.0.0-00010101000000-000000000000
 	github.com/spf13/pflag v1.0.3
-	go.etcd.io/bbolt v1.3.3 // indirect
+	go.etcd.io/bbolt v1.3.3
 	google.golang.org/grpc v1.22.0
 )
 

services/go.sum

@@ -3,8 +3,6 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/etcd-io/bbolt v1.3.3 h1:gSJmxrs37LgTqR/oyJBWok6k6SvXEUerFTbltIhXkBM=
-github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw=
 github.com/go-kit/kit v0.9.0 h1:wDJmvq38kDhkVxi50ni9ykkdUr1PKgqKOoi01fa0Mdk=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA=

services/migrations/201907191_redis_to_bolt.go

@@ -6,9 +6,9 @@ import (
 	"fmt"
 	"sync"
 
-	"github.com/etcd-io/bbolt"
 	"github.com/mediocregopher/radix/v3"
 	"github.com/spf13/pflag"
+	"go.etcd.io/bbolt"
 
 	"github.com/mitchell/selfpass/services/credentials/types"
 	"github.com/mitchell/selfpass/services/migrations/migration"