From a67be14fcb03e03e73b8a0152f4f47b94dd1bbba Mon Sep 17 00:00:00 2001 From: mitchell Date: Sat, 20 Jul 2019 04:12:58 -0400 Subject: [PATCH] Add 2 migrations for ids and boltdb; minor refactor to creds service --- services/credentials/endpoints/endpoints.go | 5 +- services/credentials/middleware/logger.go | 16 +--- services/credentials/repositories/redis.go | 20 ++--- services/credentials/service/service.go | 36 ++++----- services/credentials/types/credential.go | 2 +- services/credentials/types/interfaces.go | 2 - services/gen_certs_go.sh | 2 +- services/go.mod | 3 + services/go.sum | 6 ++ services/migrations/201907190_redis_new_id.go | 71 +++++++++++++++++ .../migrations/201907191_redis_to_bolt.go | 77 +++++++++++++++++++ services/migrations/migration/migration.go | 19 +++++ sp/go.sum | 2 + 13 files changed, 203 insertions(+), 58 deletions(-) create mode 100644 services/migrations/201907190_redis_new_id.go create mode 100644 services/migrations/201907191_redis_to_bolt.go create mode 100644 services/migrations/migration/migration.go diff --git a/services/credentials/endpoints/endpoints.go b/services/credentials/endpoints/endpoints.go index 6a77441..ce5878a 100644 --- a/services/credentials/endpoints/endpoints.go +++ b/services/credentials/endpoints/endpoints.go @@ -4,6 +4,7 @@ import ( "context" "github.com/go-kit/kit/endpoint" + "github.com/mitchell/selfpass/services/credentials/types" ) @@ -48,10 +49,6 @@ func MakeUpdateEndpoint(svc types.Service) endpoint.Endpoint { } } -type DumpResponse struct { - Contents []byte -} - type IDRequest struct { ID string } diff --git a/services/credentials/middleware/logger.go b/services/credentials/middleware/logger.go index 07471bd..d2af6d0 100644 --- a/services/credentials/middleware/logger.go +++ b/services/credentials/middleware/logger.go @@ -5,6 +5,7 @@ import ( "time" "github.com/go-kit/kit/log" + "github.com/mitchell/selfpass/services/credentials/types" ) @@ -97,18 +98,3 @@ func (svc ServiceLogger) Delete(ctx context.Context, id string) (err error) { err = svc.next.Delete(ctx, id) return err } - -func (svc ServiceLogger) DumpDB(ctx context.Context) (output []byte, err error) { - defer func(begin time.Time) { - _ = svc.l.Log( - "service", "Credentials", - "method", "Dump", - "output", output, - "err", err, - "took", time.Since(begin), - ) - }(time.Now()) - - output, err = svc.next.DumpDB(ctx) - return output, err -} diff --git a/services/credentials/repositories/redis.go b/services/credentials/repositories/redis.go index 9370e2c..32edc23 100644 --- a/services/credentials/repositories/redis.go +++ b/services/credentials/repositories/redis.go @@ -4,6 +4,7 @@ import ( "context" "github.com/mediocregopher/radix/v3" + "github.com/mitchell/selfpass/services/credentials/types" ) @@ -23,7 +24,7 @@ func (conn RedisConn) GetAllMetadata(ctx context.Context, sourceHost string, err defer close(mdch) var key string - scr := radix.NewScanner(conn.p, radix.ScanOpts{Command: scan, Pattern: types.TypePrefixCred + dash + sourceHost + star}) + scr := radix.NewScanner(conn.p, radix.ScanAllKeys) for scr.Next(&key) { select { @@ -39,7 +40,9 @@ func (conn RedisConn) GetAllMetadata(ctx context.Context, sourceHost string, err return } - mdch <- md + if sourceHost == "" || sourceHost == md.SourceHost { + mdch <- md + } } }() @@ -61,20 +64,7 @@ func (conn RedisConn) Delete(ctx context.Context, id string) (err error) { return err } -func (conn RedisConn) DumpDB(ctx context.Context) (bs []byte, err error) { - bs = []byte{} - - if err := conn.p.Do(radix.Cmd(&bs, "DUMP")); err != nil { - return nil, err - } - - return bs, nil -} - const ( - dash = "-" - star = "*" - scan = "SCAN" hGetAll = "HGETALL" hMSet = "HMSET" del = "DEL" diff --git a/services/credentials/service/service.go b/services/credentials/service/service.go index 6445fc6..3f3fae4 100644 --- a/services/credentials/service/service.go +++ b/services/credentials/service/service.go @@ -4,6 +4,7 @@ import ( "context" "encoding/base64" "fmt" + "math/rand" "time" "github.com/mitchell/selfpass/services/credentials/types" @@ -37,10 +38,12 @@ func (svc Credentials) Create(ctx context.Context, ci types.CredentialInput) (ou return output, err } + now := time.Now() + var c types.Credential - c.ID = generateID(ci) - c.CreatedAt = time.Now() - c.UpdatedAt = time.Now() + c.ID = generateID() + c.CreatedAt = now + c.UpdatedAt = now c.Primary = ci.Primary c.LoginURL = ci.LoginURL c.SourceHost = ci.SourceHost @@ -78,15 +81,19 @@ func validateCredentialInput(c types.CredentialInput) (err error) { return err } -func generateID(ci types.CredentialInput) string { - idFormat := types.TypePrefixCred + "-%s-%s" +func generateID() string { + const idLen = 8 + const alphanumerics = "abcdefghijkmnopqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ0123456789" + const alphaLen = len(alphanumerics) - if ci.Tag != "" { - idFormat += "-%s" - return fmt.Sprintf(idFormat, ci.SourceHost, ci.Primary, ci.Tag) + rand.Seed(time.Now().UnixNano()) + id := make([]byte, idLen) + + for index := range id { + id[index] = alphanumerics[rand.Int63()%int64(alphaLen)] } - return fmt.Sprintf(idFormat, ci.SourceHost, ci.Primary) + return fmt.Sprintf("%s-%s", types.KeyCredential, string(id)) } func (svc Credentials) Update(ctx context.Context, id string, ci types.CredentialInput) (output types.Credential, err error) { @@ -103,7 +110,6 @@ func (svc Credentials) Update(ctx context.Context, id string, ci types.Credentia return output, err } - c.ID = generateID(ci) c.UpdatedAt = time.Now() c.Primary = ci.Primary c.LoginURL = ci.LoginURL @@ -114,12 +120,6 @@ func (svc Credentials) Update(ctx context.Context, id string, ci types.Credentia c.Username = ci.Username c.Tag = ci.Tag - if c.ID != id { - if err = svc.repo.Delete(ctx, id); err != nil { - return output, err - } - } - return c, svc.repo.Put(ctx, c) } @@ -129,7 +129,3 @@ func (svc Credentials) Delete(ctx context.Context, id string) (err error) { } return svc.repo.Delete(ctx, id) } - -func (svc Credentials) DumpDB(ctx context.Context) (bs []byte, err error) { - return svc.repo.DumpDB(ctx) -} diff --git a/services/credentials/types/credential.go b/services/credentials/types/credential.go index 26a403f..42d59f3 100644 --- a/services/credentials/types/credential.go +++ b/services/credentials/types/credential.go @@ -5,7 +5,7 @@ import ( "time" ) -const TypePrefixCred = "cred" +const KeyCredential = "cred" type Credential struct { Metadata diff --git a/services/credentials/types/interfaces.go b/services/credentials/types/interfaces.go index 580b398..8c42428 100644 --- a/services/credentials/types/interfaces.go +++ b/services/credentials/types/interfaces.go @@ -8,7 +8,6 @@ type Service interface { Create(ctx context.Context, ci CredentialInput) (output Credential, err error) Update(ctx context.Context, id string, ci CredentialInput) (output Credential, err error) Delete(ctx context.Context, id string) (err error) - DumpDB(ctx context.Context) (bs []byte, err error) } type CredentialsRepo interface { @@ -16,7 +15,6 @@ type CredentialsRepo interface { Get(ctx context.Context, id string) (output Credential, err error) Put(ctx context.Context, c Credential) (err error) Delete(ctx context.Context, id string) (err error) - DumpDB(ctx context.Context) (bs []byte, err error) } type CredentialsClientInit func(ctx context.Context, target, ca, cert, key string) (c CredentialsClient, err error) diff --git a/services/gen_certs_go.sh b/services/gen_certs_go.sh index 64fe814..b80704e 100755 --- a/services/gen_certs_go.sh +++ b/services/gen_certs_go.sh @@ -4,7 +4,7 @@ cert=$(cat ./certs/server.pem) key=$(cat ./certs/server-key.pem) cat << EOM -// Code generated by inject-certs.sh, DO NOT EDIT. +// Code generated by gen_certs_go.sh, DO NOT EDIT. package main const ca = \`${ca}\` diff --git a/services/go.mod b/services/go.mod index 93ba000..c3f7a38 100644 --- a/services/go.mod +++ b/services/go.mod @@ -3,12 +3,15 @@ module github.com/mitchell/selfpass/services go 1.12 require ( + github.com/etcd-io/bbolt v1.3.3 github.com/go-kit/kit v0.9.0 github.com/go-logfmt/logfmt v0.4.0 // indirect github.com/go-stack/stack v1.8.0 // indirect github.com/golang/protobuf v1.3.2 github.com/mediocregopher/radix/v3 v3.3.0 github.com/mitchell/selfpass/protobuf/go v0.0.0-00010101000000-000000000000 + github.com/spf13/pflag v1.0.3 + go.etcd.io/bbolt v1.3.3 // indirect google.golang.org/grpc v1.22.0 ) diff --git a/services/go.sum b/services/go.sum index 9413d06..a366a3e 100644 --- a/services/go.sum +++ b/services/go.sum @@ -3,6 +3,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/etcd-io/bbolt v1.3.3 h1:gSJmxrs37LgTqR/oyJBWok6k6SvXEUerFTbltIhXkBM= +github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= github.com/go-kit/kit v0.9.0 h1:wDJmvq38kDhkVxi50ni9ykkdUr1PKgqKOoi01fa0Mdk= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA= @@ -25,8 +27,12 @@ github.com/mediocregopher/radix/v3 v3.3.0 h1:oacPXPKHJg0hcngVVrdtTnfGJiS+PtwoQwT github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= +go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= diff --git a/services/migrations/201907190_redis_new_id.go b/services/migrations/201907190_redis_new_id.go new file mode 100644 index 0000000..00b275f --- /dev/null +++ b/services/migrations/201907190_redis_new_id.go @@ -0,0 +1,71 @@ +package main + +import ( + "fmt" + "math/rand" + "time" + + "github.com/mediocregopher/radix/v3" + "github.com/spf13/pflag" + + "github.com/mitchell/selfpass/services/credentials/types" + "github.com/mitchell/selfpass/services/migrations/migration" +) + +func main() { + redisHost := pflag.StringP("redis-host", "r", "127.0.0.1:6379", "specify the redis host to target") + help := pflag.BoolP("help", "h", false, "see help") + pflag.Parse() + + if *help { + pflag.PrintDefaults() + return + } + + pool, err := radix.NewPool("tcp", *redisHost, 10) + migration.Check(err) + + fmt.Println("Beginning migration...") + + var pipeCmds []radix.CmdAction + var creds []*types.Credential + scanner := radix.NewScanner(pool, radix.ScanAllKeys) + + for key := ""; scanner.Next(&key); { + var cred types.Credential + pipeCmds = append(pipeCmds, radix.Cmd(&cred, "HGETALL", key)) + pipeCmds = append(pipeCmds, radix.Cmd(nil, "DEL", key)) + creds = append(creds, &cred) + } + + migration.Check(pool.Do(radix.Pipeline(pipeCmds...))) + pipeCmds = nil + + for _, cred := range creds { + tcred := *cred + tcred.ID = generateID() + + fmt.Printf("Migrating %s to %s.\n", cred.ID, tcred.ID) + + pipeCmds = append(pipeCmds, radix.FlatCmd(nil, "HMSET", tcred.ID, tcred)) + } + + migration.Check(pool.Do(radix.Pipeline(pipeCmds...))) + + fmt.Println("Done migrating.") +} + +func generateID() string { + const idLen = 8 + const alphanumerics = "abcdefghijkmnopqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ0123456789" + const alphaLen = len(alphanumerics) + + rand.Seed(time.Now().UnixNano()) + id := make([]byte, idLen) + + for index := range id { + id[index] = alphanumerics[rand.Int63()%int64(alphaLen)] + } + + return fmt.Sprintf("%s-%s", types.KeyCredential, string(id)) +} diff --git a/services/migrations/201907191_redis_to_bolt.go b/services/migrations/201907191_redis_to_bolt.go new file mode 100644 index 0000000..2669297 --- /dev/null +++ b/services/migrations/201907191_redis_to_bolt.go @@ -0,0 +1,77 @@ +package main + +import ( + "bytes" + "encoding/gob" + "fmt" + "sync" + + "github.com/etcd-io/bbolt" + "github.com/mediocregopher/radix/v3" + "github.com/spf13/pflag" + + "github.com/mitchell/selfpass/services/credentials/types" + "github.com/mitchell/selfpass/services/migrations/migration" +) + +const keyCredentials = "credentials" + +func main() { + redisHost := pflag.StringP("redis-host", "r", "127.0.0.1:6379", "specify the redis host") + boltFile := pflag.StringP("bolt-file", "b", "./data/bolt.db", "specify the bolt DB file") + help := pflag.BoolP("help", "h", false, "see help") + pflag.Parse() + + if *help { + pflag.PrintDefaults() + return + } + + pool, err := radix.NewPool("tcp", *redisHost, 10) + migration.Check(err) + + db, err := bbolt.Open(*boltFile, 0600, nil) + migration.Check(err) + + defer func() { migration.Check(db.Close()); migration.Check(pool.Close()) }() + + fmt.Println("Beginning migration...") + + var wg sync.WaitGroup + scanner := radix.NewScanner(pool, radix.ScanOpts{Command: "SCAN"}) + + for key := ""; scanner.Next(&key); { + wg.Add(1) + go func(key string) { + defer wg.Done() + + var cred types.Credential + migration.Check(pool.Do(radix.Cmd(&cred, "HGETALL", key))) + + fmt.Printf("Migrating %s.\n", cred.ID) + + migration.Check(db.Batch(func(tx *bbolt.Tx) error { + credBkt, err := tx.CreateBucketIfNotExists([]byte(keyCredentials)) + if err != nil { + return err + } + + buf := bytes.NewBuffer(nil) + err = gob.NewEncoder(buf).Encode(cred) + if err != nil { + return err + } + + if err = credBkt.Put([]byte(cred.ID), buf.Bytes()); err != nil { + return err + } + + return nil + })) + }(key) + } + + wg.Wait() + + fmt.Println("Done migrating.") +} diff --git a/services/migrations/migration/migration.go b/services/migrations/migration/migration.go new file mode 100644 index 0000000..e6ca0a7 --- /dev/null +++ b/services/migrations/migration/migration.go @@ -0,0 +1,19 @@ +package migration + +import ( + "fmt" + "os" + "runtime" +) + +func Check(err error) { + if err != nil { + _, _, line, ok := runtime.Caller(1) + if ok { + fmt.Printf("%v: %s\n", line, err) + os.Exit(1) + } + + fmt.Println(err) + } +} diff --git a/sp/go.sum b/sp/go.sum index fd02425..1b04be9 100644 --- a/sp/go.sum +++ b/sp/go.sum @@ -48,6 +48,7 @@ github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5m github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= +github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -224,6 +225,7 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yunify/qingstor-sdk-go v2.2.15+incompatible/go.mod h1:w6wqLDQ5bBTzxGJ55581UrSwLrsTAsdo9N6yX/8d9RY= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=