diff --git a/README.md b/README.md index 7e4b682..162e072 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,8 @@ remotely through encrypted transportation. All of which is deployable locally or platforms such as GCP and AWS. It is still currently in development. However, the server is already capable of serving a gRPC -based API using mutual TLS encryption, backed by Redis and Docker. It is also capable of being -deployed in a semi-automated fashion locally and to GCP thanks to Docker. +based API using mutual TLS encryption, backed by BoltDB. It is also capable +of being deployed in a semi-automated fashion locally and to GCP thanks to Docker. **Server Roadmap** @@ -64,6 +64,6 @@ using GUIs, with all the same safety and encryption as the CLI. - Go-Kit: services - gRPC/Protobuf: all - Cobra Commander & Viper Config: sp -- Redis: services +- BoltDB/Redis: services - Docker: services - Debian: docker images & machines diff --git a/services/Makefile b/services/Makefile index 9e7ba76..6e7698a 100644 --- a/services/Makefile +++ b/services/Makefile @@ -31,7 +31,6 @@ machine-create-google: --google-username selfpass \ --google-zone us-west1-c \ selfpass01 - $(MAKE) machine-put-redis.conf $(MAKE) machine-put-data $(MAKE) machine-install-stackdriver-agent $(MAKE) machine-add-grpc-server-tag @@ -42,14 +41,11 @@ machine-rm: machine-ssh: docker-machine ssh selfpass01 -machine-put-redis.conf: - docker-machine scp ./redis.conf selfpass01:redis.conf - machine-put-data: - docker-machine scp ./data/appendonly.aof selfpass01:data/ + docker-machine scp ./data/bolt.db selfpass01:data machine-get-data: - docker-machine scp selfpass01:data/appendonly.aof ./data/ + docker-machine scp selfpass01:data/bolt.db ./data machine-add-grpc-server-tag: gcloud compute instances add-tags selfpass01 \ diff --git a/services/cmd/main.go b/services/cmd/main.go index cb95c15..a632e12 100644 --- a/services/cmd/main.go +++ b/services/cmd/main.go @@ -57,7 +57,7 @@ func main() { }, }) - db, err := repositories.NewRedisConn("tcp", "redis:6379", 2) + db, err := repositories.OpenBoltDB("/home/selfpass/data/bolt.db", 0600, nil) check(err) var svc types.Service diff --git a/services/credentials/repositories/bolt.go b/services/credentials/repositories/bolt.go new file mode 100644 index 0000000..66dc230 --- /dev/null +++ b/services/credentials/repositories/bolt.go @@ -0,0 +1,139 @@ +package repositories + +import ( + "bytes" + "context" + "encoding/gob" + "os" + "sync" + + "go.etcd.io/bbolt" + + "github.com/mitchell/selfpass/services/credentials/types" +) + +func OpenBoltDB(file string, mode os.FileMode, opts *bbolt.Options) (out BoltDB, err error) { + db, err := bbolt.Open(file, mode, opts) + if err != nil { + return out, err + } + + return BoltDB{bolt: db}, nil +} + +type BoltDB struct { + bolt *bbolt.DB +} + +func (db BoltDB) GetAllMetadata(ctx context.Context, sourceHost string, errch chan<- error) (output <-chan types.Metadata) { + mdch := make(chan types.Metadata, 1) + + go func() { + defer close(mdch) + + err := db.bolt.View(func(tx *bbolt.Tx) error { + bkt := tx.Bucket([]byte(credentialsBkt)) + if bkt == nil { + return nil + } + + var wg sync.WaitGroup + err := bkt.ForEach(func(_, value []byte) error { + wg.Add(1) + + go func(value []byte) { + defer wg.Done() + + var cred types.Credential + + err := gobUnmarshal(value, &cred) + if err != nil { + errch <- err + return + } + + if sourceHost == "" || sourceHost == cred.SourceHost { + mdch <- cred.Metadata + } + }(value) + + return nil + }) + if err != nil { + return err + } + + wg.Wait() + + return nil + }) + if err != nil { + errch <- err + return + } + }() + + return mdch +} + +func (db BoltDB) Get(ctx context.Context, id string) (output types.Credential, err error) { + err = db.bolt.View(func(tx *bbolt.Tx) error { + bkt := tx.Bucket([]byte(credentialsBkt)) + if bkt == nil { + return nil + } + + value := bkt.Get([]byte(id)) + if value == nil { + return nil + } + + return gobUnmarshal(value, &output) + }) + + return output, err +} + +func (db BoltDB) Put(ctx context.Context, c types.Credential) (err error) { + err = db.bolt.Update(func(tx *bbolt.Tx) error { + bkt, err := tx.CreateBucketIfNotExists([]byte(credentialsBkt)) + if err != nil { + return err + } + + value, err := gobMarshal(c) + if err != nil { + return err + } + + return bkt.Put([]byte(c.ID), value) + }) + + return err +} + +func (db BoltDB) Delete(ctx context.Context, id string) (err error) { + err = db.bolt.Update(func(tx *bbolt.Tx) error { + bkt := tx.Bucket([]byte(credentialsBkt)) + if bkt == nil { + return nil + } + + return bkt.Delete([]byte(id)) + }) + + return err +} + +const credentialsBkt = "credentials" + +func gobMarshal(v interface{}) (bs []byte, err error) { + buf := bytes.NewBuffer(nil) + err = gob.NewEncoder(buf).Encode(v) + return buf.Bytes(), err +} + +func gobUnmarshal(bs []byte, v interface{}) error { + buf := bytes.NewReader(bs) + return gob.NewDecoder(buf).Decode(v) +} diff --git a/services/docker-compose.prod.yml b/services/docker-compose.prod.yml index 5b13ecb..2881b68 100644 --- a/services/docker-compose.prod.yml +++ b/services/docker-compose.prod.yml @@ -1,9 +1,7 @@ version: "3.7" services: - redis: - volumes: - - "/home/selfpass/data:/data" - - "/home/selfpass/redis.conf:/redis.conf" server: entrypoint: - server + volumes: + - "/home/selfpass/data:/home/selfpass/data" diff --git a/services/docker-compose.yml b/services/docker-compose.yml index ee8fe79..beb5f6a 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -1,14 +1,5 @@ version: "3.7" services: - redis: - image: "redis:5.0.5" - restart: on-failure - command: - - redis-server - - /redis.conf - volumes: - - "./redis.conf:/redis.conf" - - "./data:/data" server: build: . restart: on-failure @@ -17,5 +8,5 @@ services: - -v ports: - "8080:8080" - depends_on: - - redis + volumes: + - "./data:/home/selfpass/data" diff --git a/services/go.mod b/services/go.mod index c3f7a38..8ad4239 100644 --- a/services/go.mod +++ b/services/go.mod @@ -3,7 +3,6 @@ module github.com/mitchell/selfpass/services go 1.12 require ( - github.com/etcd-io/bbolt v1.3.3 github.com/go-kit/kit v0.9.0 github.com/go-logfmt/logfmt v0.4.0 // indirect github.com/go-stack/stack v1.8.0 // indirect @@ -11,7 +10,7 @@ require ( github.com/mediocregopher/radix/v3 v3.3.0 github.com/mitchell/selfpass/protobuf/go v0.0.0-00010101000000-000000000000 github.com/spf13/pflag v1.0.3 - go.etcd.io/bbolt v1.3.3 // indirect + go.etcd.io/bbolt v1.3.3 google.golang.org/grpc v1.22.0 ) diff --git a/services/go.sum b/services/go.sum index a366a3e..8a09532 100644 --- a/services/go.sum +++ b/services/go.sum @@ -3,8 +3,6 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/etcd-io/bbolt v1.3.3 h1:gSJmxrs37LgTqR/oyJBWok6k6SvXEUerFTbltIhXkBM= -github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= github.com/go-kit/kit v0.9.0 h1:wDJmvq38kDhkVxi50ni9ykkdUr1PKgqKOoi01fa0Mdk= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA= diff --git a/services/migrations/201907191_redis_to_bolt.go b/services/migrations/201907191_redis_to_bolt.go index 2669297..a3e3416 100644 --- a/services/migrations/201907191_redis_to_bolt.go +++ b/services/migrations/201907191_redis_to_bolt.go @@ -6,9 +6,9 @@ import ( "fmt" "sync" - "github.com/etcd-io/bbolt" "github.com/mediocregopher/radix/v3" "github.com/spf13/pflag" + "go.etcd.io/bbolt" "github.com/mitchell/selfpass/services/credentials/types" "github.com/mitchell/selfpass/services/migrations/migration" diff --git a/sp/commands/commands.go b/sp/commands/commands.go index 6ecfaad..af85553 100644 --- a/sp/commands/commands.go +++ b/sp/commands/commands.go @@ -81,6 +81,8 @@ receive: keyIDMap[key] = md.ID } + sort.Strings(keys) + prompt = &survey.Select{ Message: "Primary user key (and tag):", Options: keys, diff --git a/sp/commands/list.go b/sp/commands/list.go index ebb05ea..0b059ea 100644 --- a/sp/commands/list.go +++ b/sp/commands/list.go @@ -65,6 +65,10 @@ includes almost all the information but the most sensitive.`, var source string check(survey.AskOne(prompt, &source, nil)) + sort.Slice(mds[source], func(i, j int) bool { + return mds[source][i].Primary < mds[source][j].Primary + }) + for _, md := range mds[source] { fmt.Println(md) } diff --git a/sp/go.sum b/sp/go.sum index 1b04be9..2146303 100644 --- a/sp/go.sum +++ b/sp/go.sum @@ -225,6 +225,7 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yunify/qingstor-sdk-go v2.2.15+incompatible/go.mod h1:w6wqLDQ5bBTzxGJ55581UrSwLrsTAsdo9N6yX/8d9RY= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=