Add 2 migrations for ids and boltdb; minor refactor to creds service

services/credentials/endpoints/endpoints.go

@@ -4,6 +4,7 @@ import (
 	"context"
 
 	"github.com/go-kit/kit/endpoint"
+
 	"github.com/mitchell/selfpass/services/credentials/types"
 )
 
@@ -48,10 +49,6 @@ func MakeUpdateEndpoint(svc types.Service) endpoint.Endpoint {
 	}
 }
 
-type DumpResponse struct {
-	Contents []byte
-}
-
 type IDRequest struct {
 	ID string
 }

services/credentials/middleware/logger.go

@@ -5,6 +5,7 @@ import (
 	"time"
 
 	"github.com/go-kit/kit/log"
+
 	"github.com/mitchell/selfpass/services/credentials/types"
 )
 
@@ -97,18 +98,3 @@ func (svc ServiceLogger) Delete(ctx context.Context, id string) (err error) {
 	err = svc.next.Delete(ctx, id)
 	return err
 }
-
-func (svc ServiceLogger) DumpDB(ctx context.Context) (output []byte, err error) {
-	defer func(begin time.Time) {
-		_ = svc.l.Log(
-			"service", "Credentials",
-			"method", "Dump",
-			"output", output,
-			"err", err,
-			"took", time.Since(begin),
-		)
-	}(time.Now())
-
-	output, err = svc.next.DumpDB(ctx)
-	return output, err
-}

services/credentials/repositories/redis.go

@@ -4,6 +4,7 @@ import (
 	"context"
 
 	"github.com/mediocregopher/radix/v3"
+
 	"github.com/mitchell/selfpass/services/credentials/types"
 )
 
@@ -23,7 +24,7 @@ func (conn RedisConn) GetAllMetadata(ctx context.Context, sourceHost string, err
 		defer close(mdch)
 
 		var key string
-		scr := radix.NewScanner(conn.p, radix.ScanOpts{Command: scan, Pattern: types.TypePrefixCred + dash + sourceHost + star})
+		scr := radix.NewScanner(conn.p, radix.ScanAllKeys)
 
 		for scr.Next(&key) {
 			select {
@@ -39,8 +40,10 @@ func (conn RedisConn) GetAllMetadata(ctx context.Context, sourceHost string, err
 				return
 			}
 
+			if sourceHost == "" || sourceHost == md.SourceHost {
 				mdch <- md
 			}
+		}
 	}()
 
 	return mdch
@@ -61,20 +64,7 @@ func (conn RedisConn) Delete(ctx context.Context, id string) (err error) {
 	return err
 }
 
-func (conn RedisConn) DumpDB(ctx context.Context) (bs []byte, err error) {
-	bs = []byte{}
-
-	if err := conn.p.Do(radix.Cmd(&bs, "DUMP")); err != nil {
-		return nil, err
-	}
-
-	return bs, nil
-}
-
 const (
-	dash    = "-"
-	star    = "*"
-	scan    = "SCAN"
 	hGetAll = "HGETALL"
 	hMSet   = "HMSET"
 	del     = "DEL"

services/credentials/service/service.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/base64"
 	"fmt"
+	"math/rand"
 	"time"
 
 	"github.com/mitchell/selfpass/services/credentials/types"
@@ -37,10 +38,12 @@ func (svc Credentials) Create(ctx context.Context, ci types.CredentialInput) (ou
 		return output, err
 	}
 
+	now := time.Now()
+
 	var c types.Credential
-	c.ID = generateID(ci)
+	c.ID = generateID()
-	c.CreatedAt = time.Now()
+	c.CreatedAt = now
-	c.UpdatedAt = time.Now()
+	c.UpdatedAt = now
 	c.Primary = ci.Primary
 	c.LoginURL = ci.LoginURL
 	c.SourceHost = ci.SourceHost
@@ -78,15 +81,19 @@ func validateCredentialInput(c types.CredentialInput) (err error) {
 	return err
 }
 
-func generateID(ci types.CredentialInput) string {
+func generateID() string {
-	idFormat := types.TypePrefixCred + "-%s-%s"
+	const idLen = 8
+	const alphanumerics = "abcdefghijkmnopqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ0123456789"
+	const alphaLen = len(alphanumerics)
 
-	if ci.Tag != "" {
+	rand.Seed(time.Now().UnixNano())
-		idFormat += "-%s"
+	id := make([]byte, idLen)
-		return fmt.Sprintf(idFormat, ci.SourceHost, ci.Primary, ci.Tag)
+
+	for index := range id {
+		id[index] = alphanumerics[rand.Int63()%int64(alphaLen)]
 	}
 
-	return fmt.Sprintf(idFormat, ci.SourceHost, ci.Primary)
+	return fmt.Sprintf("%s-%s", types.KeyCredential, string(id))
 }
 
 func (svc Credentials) Update(ctx context.Context, id string, ci types.CredentialInput) (output types.Credential, err error) {
@@ -103,7 +110,6 @@ func (svc Credentials) Update(ctx context.Context, id string, ci types.Credentia
 		return output, err
 	}
 
-	c.ID = generateID(ci)
 	c.UpdatedAt = time.Now()
 	c.Primary = ci.Primary
 	c.LoginURL = ci.LoginURL
@@ -114,12 +120,6 @@ func (svc Credentials) Update(ctx context.Context, id string, ci types.Credentia
 	c.Username = ci.Username
 	c.Tag = ci.Tag
 
-	if c.ID != id {
-		if err = svc.repo.Delete(ctx, id); err != nil {
-			return output, err
-		}
-	}
-
 	return c, svc.repo.Put(ctx, c)
 }
 
@@ -129,7 +129,3 @@ func (svc Credentials) Delete(ctx context.Context, id string) (err error) {
 	}
 	return svc.repo.Delete(ctx, id)
 }
-
-func (svc Credentials) DumpDB(ctx context.Context) (bs []byte, err error) {
-	return svc.repo.DumpDB(ctx)
-}

services/credentials/types/credential.go

@@ -5,7 +5,7 @@ import (
 	"time"
 )
 
-const TypePrefixCred = "cred"
+const KeyCredential = "cred"
 
 type Credential struct {
 	Metadata

services/credentials/types/interfaces.go

@@ -8,7 +8,6 @@ type Service interface {
 	Create(ctx context.Context, ci CredentialInput) (output Credential, err error)
 	Update(ctx context.Context, id string, ci CredentialInput) (output Credential, err error)
 	Delete(ctx context.Context, id string) (err error)
-	DumpDB(ctx context.Context) (bs []byte, err error)
 }
 
 type CredentialsRepo interface {
@@ -16,7 +15,6 @@ type CredentialsRepo interface {
 	Get(ctx context.Context, id string) (output Credential, err error)
 	Put(ctx context.Context, c Credential) (err error)
 	Delete(ctx context.Context, id string) (err error)
-	DumpDB(ctx context.Context) (bs []byte, err error)
 }
 
 type CredentialsClientInit func(ctx context.Context, target, ca, cert, key string) (c CredentialsClient, err error)

services/gen_certs_go.sh

@@ -4,7 +4,7 @@ cert=$(cat ./certs/server.pem)
 key=$(cat ./certs/server-key.pem)
 
 cat << EOM
-// Code generated by inject-certs.sh, DO NOT EDIT.
+// Code generated by gen_certs_go.sh, DO NOT EDIT.
 package main
 
 const ca = \`${ca}\`

services/go.mod

@@ -3,12 +3,15 @@ module github.com/mitchell/selfpass/services
 go 1.12
 
 require (
+	github.com/etcd-io/bbolt v1.3.3
 	github.com/go-kit/kit v0.9.0
 	github.com/go-logfmt/logfmt v0.4.0 // indirect
 	github.com/go-stack/stack v1.8.0 // indirect
 	github.com/golang/protobuf v1.3.2
 	github.com/mediocregopher/radix/v3 v3.3.0
 	github.com/mitchell/selfpass/protobuf/go v0.0.0-00010101000000-000000000000
+	github.com/spf13/pflag v1.0.3
+	go.etcd.io/bbolt v1.3.3 // indirect
 	google.golang.org/grpc v1.22.0
 )
 

services/go.sum

@@ -3,6 +3,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/etcd-io/bbolt v1.3.3 h1:gSJmxrs37LgTqR/oyJBWok6k6SvXEUerFTbltIhXkBM=
+github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw=
 github.com/go-kit/kit v0.9.0 h1:wDJmvq38kDhkVxi50ni9ykkdUr1PKgqKOoi01fa0Mdk=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA=
@@ -25,8 +27,12 @@ github.com/mediocregopher/radix/v3 v3.3.0 h1:oacPXPKHJg0hcngVVrdtTnfGJiS+PtwoQwT
 github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
+github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk=
+go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628=

services/migrations/201907190_redis_new_id.go

@@ -0,0 +1,71 @@
+package main
+
+import (
+	"fmt"
+	"math/rand"
+	"time"
+
+	"github.com/mediocregopher/radix/v3"
+	"github.com/spf13/pflag"
+
+	"github.com/mitchell/selfpass/services/credentials/types"
+	"github.com/mitchell/selfpass/services/migrations/migration"
+)
+
+func main() {
+	redisHost := pflag.StringP("redis-host", "r", "127.0.0.1:6379", "specify the redis host to target")
+	help := pflag.BoolP("help", "h", false, "see help")
+	pflag.Parse()
+
+	if *help {
+		pflag.PrintDefaults()
+		return
+	}
+
+	pool, err := radix.NewPool("tcp", *redisHost, 10)
+	migration.Check(err)
+
+	fmt.Println("Beginning migration...")
+
+	var pipeCmds []radix.CmdAction
+	var creds []*types.Credential
+	scanner := radix.NewScanner(pool, radix.ScanAllKeys)
+
+	for key := ""; scanner.Next(&key); {
+		var cred types.Credential
+		pipeCmds = append(pipeCmds, radix.Cmd(&cred, "HGETALL", key))
+		pipeCmds = append(pipeCmds, radix.Cmd(nil, "DEL", key))
+		creds = append(creds, &cred)
+	}
+
+	migration.Check(pool.Do(radix.Pipeline(pipeCmds...)))
+	pipeCmds = nil
+
+	for _, cred := range creds {
+		tcred := *cred
+		tcred.ID = generateID()
+
+		fmt.Printf("Migrating %s to %s.\n", cred.ID, tcred.ID)
+
+		pipeCmds = append(pipeCmds, radix.FlatCmd(nil, "HMSET", tcred.ID, tcred))
+	}
+
+	migration.Check(pool.Do(radix.Pipeline(pipeCmds...)))
+
+	fmt.Println("Done migrating.")
+}
+
+func generateID() string {
+	const idLen = 8
+	const alphanumerics = "abcdefghijkmnopqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ0123456789"
+	const alphaLen = len(alphanumerics)
+
+	rand.Seed(time.Now().UnixNano())
+	id := make([]byte, idLen)
+
+	for index := range id {
+		id[index] = alphanumerics[rand.Int63()%int64(alphaLen)]
+	}
+
+	return fmt.Sprintf("%s-%s", types.KeyCredential, string(id))
+}

services/migrations/201907191_redis_to_bolt.go

@@ -0,0 +1,77 @@
+package main
+
+import (
+	"bytes"
+	"encoding/gob"
+	"fmt"
+	"sync"
+
+	"github.com/etcd-io/bbolt"
+	"github.com/mediocregopher/radix/v3"
+	"github.com/spf13/pflag"
+
+	"github.com/mitchell/selfpass/services/credentials/types"
+	"github.com/mitchell/selfpass/services/migrations/migration"
+)
+
+const keyCredentials = "credentials"
+
+func main() {
+	redisHost := pflag.StringP("redis-host", "r", "127.0.0.1:6379", "specify the redis host")
+	boltFile := pflag.StringP("bolt-file", "b", "./data/bolt.db", "specify the bolt DB file")
+	help := pflag.BoolP("help", "h", false, "see help")
+	pflag.Parse()
+
+	if *help {
+		pflag.PrintDefaults()
+		return
+	}
+
+	pool, err := radix.NewPool("tcp", *redisHost, 10)
+	migration.Check(err)
+
+	db, err := bbolt.Open(*boltFile, 0600, nil)
+	migration.Check(err)
+
+	defer func() { migration.Check(db.Close()); migration.Check(pool.Close()) }()
+
+	fmt.Println("Beginning migration...")
+
+	var wg sync.WaitGroup
+	scanner := radix.NewScanner(pool, radix.ScanOpts{Command: "SCAN"})
+
+	for key := ""; scanner.Next(&key); {
+		wg.Add(1)
+		go func(key string) {
+			defer wg.Done()
+
+			var cred types.Credential
+			migration.Check(pool.Do(radix.Cmd(&cred, "HGETALL", key)))
+
+			fmt.Printf("Migrating %s.\n", cred.ID)
+
+			migration.Check(db.Batch(func(tx *bbolt.Tx) error {
+				credBkt, err := tx.CreateBucketIfNotExists([]byte(keyCredentials))
+				if err != nil {
+					return err
+				}
+
+				buf := bytes.NewBuffer(nil)
+				err = gob.NewEncoder(buf).Encode(cred)
+				if err != nil {
+					return err
+				}
+
+				if err = credBkt.Put([]byte(cred.ID), buf.Bytes()); err != nil {
+					return err
+				}
+
+				return nil
+			}))
+		}(key)
+	}
+
+	wg.Wait()
+
+	fmt.Println("Done migrating.")
+}

services/migrations/migration/migration.go

@@ -0,0 +1,19 @@
+package migration
+
+import (
+	"fmt"
+	"os"
+	"runtime"
+)
+
+func Check(err error) {
+	if err != nil {
+		_, _, line, ok := runtime.Caller(1)
+		if ok {
+			fmt.Printf("%v: %s\n", line, err)
+			os.Exit(1)
+		}
+
+		fmt.Println(err)
+	}
+}

sp/go.sum

@@ -48,6 +48,7 @@ github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5m
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
 github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g=
+github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
@@ -224,6 +225,7 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/yunify/qingstor-sdk-go v2.2.15+incompatible/go.mod h1:w6wqLDQ5bBTzxGJ55581UrSwLrsTAsdo9N6yX/8d9RY=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
 go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=