diff --git a/README.md b/README.md index fd424d15..675c8ce4 100644 --- a/README.md +++ b/README.md @@ -106,7 +106,7 @@ func main() { sbot, err := sbot.New() check(err) - publish, err := multilogs.OpenPublishLog(sbot.RootLog, sbot.UserFeeds, *sbot.KeyPair) + publish, err := multilogs.OpenPublishLog(sbot.ReceiveLog, sbot.UserFeeds, *sbot.KeyPair) check(err) alice, err := refs.ParseFeedRef("@alicesKeyInActualBase64Bytes.ed25519") diff --git a/client/client.go b/client/client.go index 9e058ec4..1c38f5d1 100644 --- a/client/client.go +++ b/client/client.go @@ -336,7 +336,7 @@ func (c Client) MessagesByType(opts message.MessagesByTypeArgs) (luigi.Source, e } func (c Client) Tangles(o message.TanglesArgs) (luigi.Source, error) { - src, err := c.Source(c.rootCtx, o.MarshalType, muxrpc.Method{"tangles"}, o) + src, err := c.Source(c.rootCtx, o.MarshalType, muxrpc.Method{"tangles", "replies"}, o) return src, errors.Wrap(err, "ssbClient/tangles: failed to create stream") } diff --git a/client/client_test.go b/client/client_test.go index d21fc70f..9e77c82d 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -23,8 +23,6 @@ import ( "go.cryptoscope.co/ssb/internal/testutils" "go.cryptoscope.co/ssb/message" "go.cryptoscope.co/ssb/network" - "go.cryptoscope.co/ssb/plugins2" - "go.cryptoscope.co/ssb/plugins2/tangles" "go.cryptoscope.co/ssb/sbot" refs "go.mindeco.de/ssb-refs" ) @@ -58,7 +56,10 @@ func TestUnixSock(t *testing.T) { var msgs []*refs.MessageRef const msgCount = 15 for i := 0; i < msgCount; i++ { - ref, err := c.Publish(struct{ I int }{i}) + ref, err := c.Publish(struct { + Type string `json:"type"` + Test int + }{"test", i}) r.NoError(err) r.NotNil(ref) msgs = append(msgs, ref) @@ -88,7 +89,7 @@ func TestUnixSock(t *testing.T) { msg, ok := v.(refs.Message) r.True(ok, "%d: wrong type: %T", i, v) - r.True(msg.Key().Equal(*msgs[i]), "wrong message %d", i) + r.True(msg.Key().Equal(msgs[i]), "wrong message %d", i) i++ } r.Equal(msgCount, i, "did not get all messages") @@ -319,7 +320,10 @@ func LotsOfStatusCalls(newPair mkPair) func(t *testing.T) { for i := 25; i > 0; i-- { time.Sleep(500 * time.Millisecond) - ref, err := c.Publish(struct{ Test int }{i}) + ref, err := c.Publish(struct { + Type string `json:"type"` + Test int + }{"test", i}) r.NoError(err, "publish %d errored", i) r.NotNil(ref) @@ -339,7 +343,7 @@ func LotsOfStatusCalls(newPair mkPair) func(t *testing.T) { a.GreaterOrEqual(statusCalls, uint32(1000), "expected more status calls") - v, err := srv.RootLog.Seq().Value() + v, err := srv.ReceiveLog.Seq().Value() r.NoError(err) r.EqualValues(24, v) @@ -348,6 +352,12 @@ func LotsOfStatusCalls(newPair mkPair) func(t *testing.T) { } } +type testMsg struct { + Type string `json:"type"` + Foo string + Bar int +} + func TestPublish(t *testing.T) { // defer leakcheck.Check(t) r, a := require.New(t), assert.New(t) @@ -381,25 +391,21 @@ func TestPublish(t *testing.T) { // end test boilerplate // no messages yet - seqv, err := srv.RootLog.Seq().Value() + seqv, err := srv.ReceiveLog.Seq().Value() r.NoError(err, "failed to get root log sequence") r.Equal(margaret.SeqEmpty, seqv) - type testMsg struct { - Foo string - Bar int - } - msg := testMsg{"hello", 23} + msg := testMsg{"test", "hello", 23} ref, err := c.Publish(msg) r.NoError(err, "failed to call publish") r.NotNil(ref) // get stored message from the log - seqv, err = srv.RootLog.Seq().Value() + seqv, err = srv.ReceiveLog.Seq().Value() r.NoError(err, "failed to get root log sequence") wantSeq := margaret.BaseSeq(0) a.Equal(wantSeq, seqv) - msgv, err := srv.RootLog.Get(wantSeq) + msgv, err := srv.ReceiveLog.Get(wantSeq) r.NoError(err) newMsg, ok := msgv.(refs.Message) r.True(ok) @@ -442,7 +448,6 @@ func TestTangles(t *testing.T) { sbot.WithInfo(srvLog), sbot.WithRepoPath(srvRepo), sbot.WithListenAddr(":0"), - sbot.LateOption(sbot.MountPlugin(&tangles.Plugin{}, plugins2.AuthMaster)), ) r.NoError(err, "sbot srv init failed") @@ -464,26 +469,27 @@ func TestTangles(t *testing.T) { // end test boilerplate type testMsg struct { + Type string `json:"type"` Foo string Bar int Root *refs.MessageRef `json:"root,omitempty"` } - msg := testMsg{"hello", 23, nil} + msg := testMsg{"test", "hello", 23, nil} rootRef, err := c.Publish(msg) r.NoError(err, "failed to call publish") r.NotNil(rootRef) - rep1 := testMsg{"reply", 1, rootRef} + rep1 := testMsg{"test", "reply", 1, rootRef} rep1Ref, err := c.Publish(rep1) r.NoError(err, "failed to call publish") r.NotNil(rep1Ref) - rep2 := testMsg{"reply", 2, rootRef} + rep2 := testMsg{"test", "reply", 2, rootRef} rep2Ref, err := c.Publish(rep2) r.NoError(err, "failed to call publish") r.NotNil(rep2Ref) opts := message.TanglesArgs{} - opts.Root = *rootRef + opts.Root = rootRef opts.Limit = 2 opts.Keys = true opts.MarshalType = refs.KeyValueRaw{} diff --git a/client/dont_break_test.go b/client/dont_break_test.go index 1ce8181e..a0ccd99d 100644 --- a/client/dont_break_test.go +++ b/client/dont_break_test.go @@ -67,7 +67,10 @@ func TestAskForSomethingWeird(t *testing.T) { var msgs []*refs.MessageRef const msgCount = 15 for i := 0; i < msgCount; i++ { - ref, err := c.Publish(struct{ I int }{i}) + ref, err := c.Publish(struct { + Type string `json:"type"` + Test int + }{"test", i}) r.NoError(err) r.NotNil(ref) msgs = append(msgs, ref) @@ -117,7 +120,7 @@ func TestAskForSomethingWeird(t *testing.T) { msg, ok := v.(refs.Message) r.True(ok, "%d: wrong type: %T", i, v) - r.True(msg.Key().Equal(*msgs[i]), "wrong message %d", i) + r.True(msg.Key().Equal(msgs[i]), "wrong message %d", i) i++ } r.Equal(msgCount, i, "did not get all messages") diff --git a/client/encoding_test.go b/client/encoding_test.go index 89335416..476a1f8a 100644 --- a/client/encoding_test.go +++ b/client/encoding_test.go @@ -59,18 +59,13 @@ func TestEncodeHistStreamAsJSON(t *testing.T) { // end test boilerplate // no messages yet - seqv, err := srv.RootLog.Seq().Value() + seqv, err := srv.ReceiveLog.Seq().Value() r.NoError(err, "failed to get root log sequence") r.Equal(margaret.SeqEmpty, seqv) - type testMsg struct { - Foo string - Bar int - } var wantRefs []string for i := 0; i < 10; i++ { - - msg := testMsg{"hello", 23} + msg := testMsg{"test", "hello", 23} ref, err := c.Publish(msg) r.NoError(err, "failed to call publish") r.NotNil(ref) @@ -78,7 +73,7 @@ func TestEncodeHistStreamAsJSON(t *testing.T) { wantRefs = append(wantRefs, ref.Ref()) } - seqv, err = srv.RootLog.Seq().Value() + seqv, err = srv.ReceiveLog.Seq().Value() r.NoError(err, "failed to get root log sequence") r.EqualValues(9, seqv) diff --git a/client/private_test.go b/client/private_test.go new file mode 100644 index 00000000..c175dba0 --- /dev/null +++ b/client/private_test.go @@ -0,0 +1,92 @@ +package client_test + +import ( + "context" + "os" + "path/filepath" + "testing" + + "go.cryptoscope.co/luigi" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.cryptoscope.co/ssb/client" + "go.cryptoscope.co/ssb/internal/testutils" + "go.cryptoscope.co/ssb/message" + "go.cryptoscope.co/ssb/sbot" +) + +func TestAutomaticUnboxing(t *testing.T) { + r, a := require.New(t), assert.New(t) + + srvRepo := filepath.Join("testrun", t.Name(), "serv") + os.RemoveAll(srvRepo) + srvLog := testutils.NewRelativeTimeLogger(nil) + + srv, err := sbot.New( + sbot.WithInfo(srvLog), + sbot.WithRepoPath(srvRepo), + sbot.WithListenAddr(":0"), + sbot.LateOption(sbot.WithUNIXSocket()), + ) + r.NoError(err, "sbot srv init failed") + + srv.PublishLog.Publish(map[string]string{"type": "test", "hello": "world"}) + + c, err := client.NewUnix(filepath.Join(srvRepo, "socket")) + r.NoError(err, "failed to make client connection") + // end test boilerplate + + ref, err := c.Whoami() + r.NoError(err, "failed to call whoami") + r.NotNil(ref) + a.Equal(srv.KeyPair.Id.Ref(), ref.Ref()) + + groupID, root, err := srv.Groups.Create("client test group") + r.NoError(err, "failed to create group") + + hello1, err := srv.Groups.PublishPostTo(groupID, "hello 1!") + r.NoError(err, "failed to post hello1") + + hello2, err := srv.Groups.PublishPostTo(groupID, "hello 2!") + r.NoError(err, "failed to post hello2") + + args := message.MessagesByTypeArgs{Type: "post"} + args.Private = false + src, err := c.MessagesByType(args) + r.NoError(err, "failed to create source for public messages") + testElementsInSource(t, src, 0) + + args.Private = true + src, err = c.MessagesByType(args) + r.NoError(err, "failed to create source for private messages") + testElementsInSource(t, src, 2) + + // TODO: check messages hello1 and hello2 are included in src + _ = hello1 + _ = hello2 + + targs := message.TanglesArgs{ + Root: root, + Name: "group", + } + targs.Private = false + src, err = c.Tangles(targs) + r.NoError(err, "failed to create source for tangled messages (public)") + testElementsInSource(t, src, 0) + + targs.Private = true + src, err = c.Tangles(targs) + r.NoError(err, "failed to create source for tangled messages (private)") + testElementsInSource(t, src, 3) // add-member + the two posts +} + +func testElementsInSource(t *testing.T, src luigi.Source, cnt int) { + ctx := context.Background() + r, a := require.New(t), assert.New(t) + var elems []interface{} + var snk = luigi.NewSliceSink(&elems) + err := luigi.Pump(ctx, snk, src) + r.NoError(err, "failed to get all elements from source (public)") + a.Len(elems, cnt) +} diff --git a/client/replicate_test.go b/client/replicate_test.go index 6efda60f..35307bc8 100644 --- a/client/replicate_test.go +++ b/client/replicate_test.go @@ -58,17 +58,18 @@ func TestReplicateUpTo(t *testing.T) { kp.Id.Algo = refs.RefAlgoFeedGabby } - publish, err := message.OpenPublishLog(srv.RootLog, uf, kp) + publish, err := message.OpenPublishLog(srv.ReceiveLog, uf, kp) r.NoError(err) testKeyPairs[kp.Id.Ref()] = i for n := i; n > 0; n-- { ref, err := publish.Publish(struct { + Type string `json:"type"` Test bool N int Hello string - }{true, n, kp.Id.Ref()}) + }{"test", true, n, kp.Id.Ref()}) r.NoError(err) t.Log(ref.Ref()) } diff --git a/client/streamtype_test.go b/client/streamtype_test.go index ffba1c5e..d09f9214 100644 --- a/client/streamtype_test.go +++ b/client/streamtype_test.go @@ -54,28 +54,23 @@ func TestReadStreamAsInterfaceMessage(t *testing.T) { // end test boilerplate // no messages yet - seqv, err := srv.RootLog.Seq().Value() + seqv, err := srv.ReceiveLog.Seq().Value() r.NoError(err, "failed to get root log sequence") r.Equal(margaret.SeqEmpty, seqv) - type testMsg struct { - Foo string - Bar int - } var wantRefs []string for i := 0; i < 10; i++ { - - msg := testMsg{"hello", 23} + msg := testMsg{"test", "hello", 23} ref, err := c.Publish(msg) r.NoError(err, "failed to call publish") r.NotNil(ref) // get stored message from the log - seqv, err = srv.RootLog.Seq().Value() + seqv, err = srv.ReceiveLog.Seq().Value() r.NoError(err, "failed to get root log sequence") wantSeq := margaret.BaseSeq(i) a.Equal(wantSeq, seqv) - msgv, err := srv.RootLog.Get(wantSeq) + msgv, err := srv.ReceiveLog.Get(wantSeq) r.NoError(err) newMsg, ok := msgv.(refs.Message) r.True(ok) diff --git a/cmd/go-sbot/crashrecovery_test.go b/cmd/go-sbot/crashrecovery_test.go index cf3fb205..6f3b7227 100644 --- a/cmd/go-sbot/crashrecovery_test.go +++ b/cmd/go-sbot/crashrecovery_test.go @@ -102,9 +102,10 @@ func TestRecoverFromCrash(t *testing.T) { } ref, err := c.Publish(struct { + Type string `json:"type"` Test string Try, I int - }{"working!", try, i}) + }{"test", "working!", try, i}) r.NoError(err) t.Logf("%d:connection established (i:%d) %s", try, i, ref.Ref()) diff --git a/cmd/go-sbot/main.go b/cmd/go-sbot/main.go index 70c7b114..0bf62af8 100644 --- a/cmd/go-sbot/main.go +++ b/cmd/go-sbot/main.go @@ -31,6 +31,7 @@ import ( "go.cryptoscope.co/ssb" "go.cryptoscope.co/ssb/internal/ctxutils" + "go.cryptoscope.co/ssb/internal/storedrefs" "go.cryptoscope.co/ssb/internal/testutils" "go.cryptoscope.co/ssb/multilogs" mksbot "go.cryptoscope.co/ssb/sbot" @@ -346,7 +347,7 @@ func runSbot() error { } RepoStats.With("part", "feeds").Set(float64(len(feeds))) - rseq, err := sbot.RootLog.Seq().Value() + rseq, err := sbot.ReceiveLog.Seq().Value() if err != nil { return errors.Wrap(err, "could not get root log sequence number") } @@ -385,7 +386,7 @@ func runSbot() error { } for _, blocked := range lst { - isStored, err := multilog.Has(uf, blocked.StoredAddr()) + isStored, err := multilog.Has(uf, storedrefs.Feed(blocked)) if err != nil { return errors.Wrap(err, "blocked lookup in multilog") } @@ -414,7 +415,8 @@ func runSbot() error { time.Sleep(1 * time.Second) select { case <-ctx.Done(): - return nil + err := sbot.Close() + return err default: } } diff --git a/cmd/sbotcli/main.go b/cmd/sbotcli/main.go index b989ff75..2de94531 100644 --- a/cmd/sbotcli/main.go +++ b/cmd/sbotcli/main.go @@ -30,7 +30,6 @@ import ( "go.cryptoscope.co/secretstream" "go.cryptoscope.co/ssb" ssbClient "go.cryptoscope.co/ssb/client" - "go.cryptoscope.co/ssb/message" refs "go.mindeco.de/ssb-refs" "golang.org/x/crypto/ed25519" cli "gopkg.in/urfave/cli.v2" @@ -82,14 +81,17 @@ var app = cli.App{ blockCmd, friendsCmd, logStreamCmd, + sortedStreamCmd, typeStreamCmd, historyStreamCmd, + partialStreamCmd, replicateUptoCmd, + repliesStreamCmd, callCmd, connectCmd, queryCmd, - privateCmd, publishCmd, + groupsCmd, }, } @@ -111,7 +113,6 @@ func check(err error) { } func main() { - cli.VersionPrinter = func(c *cli.Context) { fmt.Printf("%s (rev: %s, built: %s)\n", c.App.Version, Version, Build) } @@ -127,7 +128,7 @@ func todo(ctx *cli.Context) error { func initClient(ctx *cli.Context) error { longctx = context.Background() - longctx, shutdownFunc = context.WithTimeout(longctx, 10*time.Second) + longctx, shutdownFunc = context.WithTimeout(longctx, 45*time.Second) signalc := make(chan os.Signal) signal.Notify(signalc, os.Interrupt, syscall.SIGTERM) go func() { @@ -184,28 +185,6 @@ func newClient(ctx *cli.Context) (*ssbClient.Client, error) { return client, nil } -func getStreamArgs(ctx *cli.Context) message.CreateHistArgs { - var ref *refs.FeedRef - if id := ctx.String("id"); id != "" { - var err error - ref, err = refs.ParseFeedRef(id) - if err != nil { - panic(err) - } - } - args := message.CreateHistArgs{ - ID: ref, - Seq: ctx.Int64("seq"), - AsJSON: ctx.Bool("asJSON"), - } - args.Limit = ctx.Int64("limit") - args.Reverse = ctx.Bool("reverse") - args.Live = ctx.Bool("live") - args.Keys = ctx.Bool("keys") - args.Values = ctx.Bool("values") - return args -} - var callCmd = &cli.Command{ Name: "call", Usage: "make an dump* async call", @@ -322,9 +301,87 @@ var queryCmd = &cli.Command{ Action: todo, //query, } -var privateCmd = &cli.Command{ - Name: "private", +var groupsCmd = &cli.Command{ + Name: "groups", + Usage: "group managment (create, invite, publishTo, etc.)", Subcommands: []*cli.Command{ - privateReadCmd, + groupsCreateCmd, + groupsInviteCmd, + groupsPublishToCmd, + groupsJoinCmd, }, } + +var groupsCreateCmd = &cli.Command{ + Name: "create", + Usage: "create a new empty group", + Action: func(ctx *cli.Context) error { + client, err := newClient(ctx) + if err != nil { + return err + } + + name := ctx.Args().First() + if name == "" { + return fmt.Errorf("group name can't be empty") + } + + var val interface{} + val, err = client.Async(longctx, val, muxrpc.Method{"groups", "create"}, struct { + Name string `json:"name"` + }{name}) + if err != nil { + return err + } + log.Log("event", "group created") + goon.Dump(val) + return nil + }, +} + +var groupsInviteCmd = &cli.Command{ + Name: "invite", + Usage: "add people to a group", + Action: todo, +} + +var groupsPublishToCmd = &cli.Command{ + Name: "publishTo", + Usage: "publish a handcrafted JSON blob to a group", + Action: func(ctx *cli.Context) error { + var content interface{} + err := json.NewDecoder(os.Stdin).Decode(&content) + if err != nil { + return errors.Wrapf(err, "publish/raw: invalid json input from stdin") + } + + groupID, err := refs.ParseMessageRef(ctx.Args().First()) + if err != nil { + return fmt.Errorf("groupID needs to be a valid message ref: %w", err) + } + + if groupID.Algo != refs.RefAlgoCloakedGroup { + return fmt.Errorf("groupID needs to be a cloaked message ref, not %s", groupID.Algo) + } + + client, err := newClient(ctx) + if err != nil { + return err + } + + var reply interface{} + v, err := client.Async(longctx, reply, muxrpc.Method{"groups", "publishTo"}, groupID.Ref(), content) + if err != nil { + return errors.Wrapf(err, "publish call failed.") + } + log.Log("event", "publishTo", "type", "raw") + goon.Dump(v) + return nil + }, +} + +var groupsJoinCmd = &cli.Command{ + Name: "join", + Usage: "manually join a group by adding the group key", + Action: todo, +} diff --git a/cmd/sbotcli/streams.go b/cmd/sbotcli/streams.go index a1dbfce6..cac43937 100644 --- a/cmd/sbotcli/streams.go +++ b/cmd/sbotcli/streams.go @@ -9,6 +9,8 @@ import ( "io" "os" + "go.cryptoscope.co/ssb/message" + "github.com/pkg/errors" "go.cryptoscope.co/luigi" "go.cryptoscope.co/muxrpc" @@ -19,15 +21,43 @@ import ( var streamFlags = []cli.Flag{ &cli.IntFlag{Name: "limit", Value: -1}, &cli.IntFlag{Name: "seq", Value: 0}, + &cli.IntFlag{Name: "gt"}, + &cli.IntFlag{Name: "lt"}, &cli.BoolFlag{Name: "reverse"}, &cli.BoolFlag{Name: "live"}, &cli.BoolFlag{Name: "keys", Value: false}, &cli.BoolFlag{Name: "values", Value: false}, + &cli.BoolFlag{Name: "private", Value: false}, +} + +func getStreamArgs(ctx *cli.Context) message.CreateHistArgs { + var ref *refs.FeedRef + if id := ctx.String("id"); id != "" { + var err error + ref, err = refs.ParseFeedRef(id) + if err != nil { + panic(err) + } + } + args := message.CreateHistArgs{ + ID: ref, + Seq: ctx.Int64("seq"), + AsJSON: ctx.Bool("asJSON"), + } + args.Limit = ctx.Int64("limit") + args.Gt = ctx.Int64("gt") + args.Lt = ctx.Int64("lt") + args.Reverse = ctx.Bool("reverse") + args.Live = ctx.Bool("live") + args.Keys = ctx.Bool("keys") + args.Values = ctx.Bool("values") + args.Private = ctx.Bool("private") + return args } type mapMsg map[string]interface{} -var typeStreamCmd = &cli.Command{ +var partialStreamCmd = &cli.Command{ Name: "partial", Flags: append(streamFlags, &cli.StringFlag{Name: "id"}, &cli.BoolFlag{Name: "asJSON"}), Action: func(ctx *cli.Context) error { @@ -111,8 +141,8 @@ var logStreamCmd = &cli.Command{ }, } -var privateReadCmd = &cli.Command{ - Name: "read", +var sortedStreamCmd = &cli.Command{ + Name: "sorted", Flags: streamFlags, Action: func(ctx *cli.Context) error { client, err := newClient(ctx) @@ -121,7 +151,57 @@ var privateReadCmd = &cli.Command{ } var args = getStreamArgs(ctx) - src, err := client.Source(longctx, mapMsg{}, muxrpc.Method{"private", "read"}, args) + src, err := client.Source(longctx, mapMsg{}, muxrpc.Method{"createFeedStream"}, args) + if err != nil { + return errors.Wrap(err, "source stream call failed") + } + err = luigi.Pump(longctx, jsonDrain(os.Stdout), src) + return errors.Wrap(err, "log failed") + }, +} + +var typeStreamCmd = &cli.Command{ + Name: "bytype", + Flags: streamFlags, + Action: func(ctx *cli.Context) error { + client, err := newClient(ctx) + if err != nil { + return err + } + var targs message.MessagesByTypeArgs + arg := getStreamArgs(ctx) + targs.CommonArgs = arg.CommonArgs + targs.StreamArgs = arg.StreamArgs + targs.Type = ctx.Args().First() + src, err := client.Source(longctx, mapMsg{}, muxrpc.Method{"messagesByType"}, targs) + if err != nil { + return errors.Wrap(err, "source stream call failed") + } + err = luigi.Pump(longctx, jsonDrain(os.Stdout), src) + return errors.Wrap(err, "byType failed") + }, +} + +var repliesStreamCmd = &cli.Command{ + Name: "replies", + Flags: append(streamFlags, &cli.StringFlag{Name: "tname", Usage: "tangle name (v2)"}), + Action: func(ctx *cli.Context) error { + client, err := newClient(ctx) + if err != nil { + return err + } + + var targs message.TanglesArgs + arg := getStreamArgs(ctx) + targs.CommonArgs = arg.CommonArgs + targs.StreamArgs = arg.StreamArgs + targs.Root, err = refs.ParseMessageRef(ctx.Args().First()) + if err != nil { + return err + } + targs.Name = ctx.String("tname") + + src, err := client.Source(longctx, mapMsg{}, muxrpc.Method{"tangles", "read"}, targs) if err != nil { return errors.Wrap(err, "source stream call failed") } @@ -138,7 +218,6 @@ var replicateUptoCmd = &cli.Command{ if err != nil { return err } - var args = getStreamArgs(ctx) src, err := client.Source(longctx, mapMsg{}, muxrpc.Method{"replicate", "upto"}, args) if err != nil { diff --git a/cmd/ssb-migrate-log/main.go b/cmd/ssb-migrate-log/main.go deleted file mode 100644 index f9c0157f..00000000 --- a/cmd/ssb-migrate-log/main.go +++ /dev/null @@ -1,58 +0,0 @@ -// SPDX-License-Identifier: MIT - -package main - -import ( - "fmt" - "os" - "runtime/debug" - - "github.com/cryptix/go/logging" - - "github.com/pkg/errors" - - "go.cryptoscope.co/ssb/repo" - "go.cryptoscope.co/ssb/repo/migrations" - "go.cryptoscope.co/ssb/sbot" -) - -func check(err error) { - if err != nil { - fail(err) - } -} - -func fail(err error) { - fmt.Fprintf(os.Stderr, "error: %s\n", err) - fmt.Fprintln(os.Stderr, "occurred at") - debug.PrintStack() - os.Exit(1) -} - -func main() { - logging.SetupLogging(nil) - logger := logging.Logger("migrate") - if len(os.Args) != 2 { - fmt.Fprintln(os.Stderr, "usage: migrate2 ") - os.Exit(1) - } - repoDir := os.Args[1] - - repo := repo.New(repoDir) - didUpgrade, err := migrations.UpgradeToMultiMessage(logger, repo) - check(errors.Wrap(err, "BotInit: repo migration failed")) - - if didUpgrade { - os.RemoveAll(repo.GetPath("indexes")) - os.RemoveAll(repo.GetPath("sublogs")) - - sbot, err := sbot.New( - sbot.WithInfo(logger), - sbot.WithRepoPath(repoDir), - sbot.DisableNetworkNode(), - sbot.DisableLiveIndexMode()) - check(errors.Wrap(err, "BotInit: failed to make reindexing sbot")) - err = sbot.Close() - check(err) - } -} diff --git a/cmd/ssb-roaridx-inspector/main.go b/cmd/ssb-roaridx-inspector/main.go index 9a39b249..967a4994 100644 --- a/cmd/ssb-roaridx-inspector/main.go +++ b/cmd/ssb-roaridx-inspector/main.go @@ -7,9 +7,10 @@ import ( "os" "github.com/cryptix/go/logging" + "github.com/pkg/errors" + "go.cryptoscope.co/librarian" "go.cryptoscope.co/margaret/multilog" - multimkv "go.cryptoscope.co/margaret/multilog/roaring/mkv" - refs "go.mindeco.de/ssb-refs" + multifs "go.cryptoscope.co/margaret/multilog/roaring/fs" ) var check = logging.CheckFatal @@ -24,34 +25,29 @@ func main() { dir := os.Args[1] - mlog, err := multimkv.NewMultiLog(dir) + mlog, err := multifs.NewMultiLog(dir) check(err) - /* - addrs, err := mlog.List() - check(errors.Wrap(err, "error listing multilog")) - log.Log("mlog", "opened", "list#", len(addrs)) - for i, a := range addrs { - var sr ssb.StorageRef - err := sr.Unmarshal([]byte(a)) - check(err) - - sublog, err := mlog.Get(a) - check(err) - seqv, err := sublog.Seq().Value() - check(err) - log.Log("i", i, "addr", sr.Ref(), "seq", seqv) - } - */ + addrs, err := mlog.List() + check(errors.Wrap(err, "error listing multilog")) + log.Log("mlog", "opened", "list#", len(addrs)) + for i, a := range addrs { + + sublog, err := mlog.Get(a) + check(err) + seqv, err := sublog.Seq().Value() + check(err) + log.Log("i", i, "addr", string(a), "seq", seqv) + } // check has if len(os.Args) > 2 { - ref, err := refs.ParseFeedRef(os.Args[2]) - check(err) - has, err := multilog.Has(mlog, ref.StoredAddr()) - log.Log("mlog", "has", "addr", ref.Ref(), "has?", has, "hasErr", err) + addr := librarian.Addr(os.Args[2]) + + has, err := multilog.Has(mlog, addr) + log.Log("mlog", "has", "addr", string(addr), "has?", has, "hasErr", err) - bmap, err := mlog.LoadInternalBitmap(ref.StoredAddr()) + bmap, err := mlog.LoadInternalBitmap(addr) check(err) fmt.Println(bmap.GetCardinality()) fmt.Println(bmap.String()) diff --git a/feedset.go b/feedset.go index 4bf39424..576fcb93 100644 --- a/feedset.go +++ b/feedset.go @@ -5,8 +5,11 @@ package ssb import ( "sync" + "go.mindeco.de/ssb-refs/tfk" + "github.com/pkg/errors" "go.cryptoscope.co/librarian" + "go.cryptoscope.co/ssb/internal/storedrefs" refs "go.mindeco.de/ssb-refs" ) @@ -24,33 +27,20 @@ func NewFeedSet(size int) *StrFeedSet { } } -func (fs *StrFeedSet) AddStored(r *refs.StorageRef) error { - fs.mu.Lock() - defer fs.mu.Unlock() - - b, err := r.Marshal() - if err != nil { - return errors.Wrap(err, "failed to marshal stored ref") - } - - fs.set[librarian.Addr(b)] = struct{}{} - return nil -} - func (fs *StrFeedSet) AddRef(ref *refs.FeedRef) error { fs.mu.Lock() defer fs.mu.Unlock() copied := ref.Copy() - fs.set[copied.StoredAddr()] = struct{}{} + fs.set[storedrefs.Feed(copied)] = struct{}{} return nil } func (fs *StrFeedSet) Delete(ref *refs.FeedRef) error { fs.mu.Lock() defer fs.mu.Unlock() - delete(fs.set, ref.StoredAddr()) + delete(fs.set, storedrefs.Feed(ref)) return nil } @@ -68,17 +58,13 @@ func (fs StrFeedSet) List() ([]*refs.FeedRef, error) { i := 0 for feed := range fs.set { - var sr refs.StorageRef - err := sr.Unmarshal([]byte(feed)) + var sr tfk.Feed + err := sr.UnmarshalBinary([]byte(feed)) if err != nil { return nil, errors.Wrap(err, "failed to decode map entry") } - got, err := sr.FeedRef() - if err != nil { - return nil, errors.Wrap(err, "failed to make ref from map entry") - } // log.Printf("dbg List(%d) %s", i, ref.Ref()) - lst[i] = got.Copy() + lst[i] = sr.Feed().Copy() i++ } return lst, nil @@ -87,6 +73,6 @@ func (fs StrFeedSet) List() ([]*refs.FeedRef, error) { func (fs StrFeedSet) Has(ref *refs.FeedRef) bool { fs.mu.Lock() defer fs.mu.Unlock() - _, has := fs.set[ref.StoredAddr()] + _, has := fs.set[storedrefs.Feed(ref)] return has } diff --git a/go.mod b/go.mod index 01f3e6ea..7a5c3a1b 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( go.cryptoscope.co/secretstream v1.2.2 go.mindeco.de/ssb-gabbygrove v0.1.7-0.20200618115102-169cb68d2398 go.mindeco.de/ssb-multiserver v0.0.0-20200615120544-ce4da9d56996 - go.mindeco.de/ssb-refs v0.0.0-20201012085128-c5a76b42eacc + go.mindeco.de/ssb-refs v0.1.1-0.20201123094321-75dc1fe9e1e4 golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e golang.org/x/text v0.3.3 diff --git a/go.sum b/go.sum index f48914e8..7f558c68 100644 --- a/go.sum +++ b/go.sum @@ -416,6 +416,12 @@ go.mindeco.de/ssb-refs v0.0.0-20200615115708-7b8437f55cef/go.mod h1:u3EqvzDljgzr go.mindeco.de/ssb-refs v0.0.0-20200615121534-e677c80c4097/go.mod h1:NYcuq8EOqR8/RgmMa7qA4VgrXXUp+5Y0rk8CTi++Xys= go.mindeco.de/ssb-refs v0.0.0-20201012085128-c5a76b42eacc h1:NmGAN17v9hDOSx4zIwAL42O9oX8m7Bk+n0lovlEqL5E= go.mindeco.de/ssb-refs v0.0.0-20201012085128-c5a76b42eacc/go.mod h1:NYcuq8EOqR8/RgmMa7qA4VgrXXUp+5Y0rk8CTi++Xys= +go.mindeco.de/ssb-refs v0.0.0-20201114101049-bcafb38ada11 h1:tlMHTgFsb0O8r5mgn4Ywwxvf9xW/wB/MEGVpS2WquzA= +go.mindeco.de/ssb-refs v0.0.0-20201114101049-bcafb38ada11/go.mod h1:OnBnV02ux4lLsZ39LID6yYLqSDp+dqTHb/3miYPkQFs= +go.mindeco.de/ssb-refs v0.1.0 h1:zRMzF1fnYuyPWA3D9WDLQb2VeJ029lZjRgWdcBQQf3U= +go.mindeco.de/ssb-refs v0.1.0/go.mod h1:OnBnV02ux4lLsZ39LID6yYLqSDp+dqTHb/3miYPkQFs= +go.mindeco.de/ssb-refs v0.1.1-0.20201123094321-75dc1fe9e1e4 h1:UvAaXFqUrU/y8lJDeLn4DZYVrVQaasaOkeY4DuUpDuY= +go.mindeco.de/ssb-refs v0.1.1-0.20201123094321-75dc1fe9e1e4/go.mod h1:OnBnV02ux4lLsZ39LID6yYLqSDp+dqTHb/3miYPkQFs= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= diff --git a/graph/builder.go b/graph/builder.go index c1863cf8..7796010f 100644 --- a/graph/builder.go +++ b/graph/builder.go @@ -8,12 +8,15 @@ import ( "math" "sync" + "go.mindeco.de/ssb-refs/tfk" + "github.com/dgraph-io/badger" kitlog "github.com/go-kit/kit/log" "github.com/pkg/errors" "go.cryptoscope.co/librarian" libbadger "go.cryptoscope.co/librarian/badger" "go.cryptoscope.co/margaret" + "go.cryptoscope.co/ssb/internal/storedrefs" refs "go.mindeco.de/ssb-refs" "gonum.org/v1/gonum/graph" "gonum.org/v1/gonum/graph/path" @@ -92,8 +95,8 @@ func (b *builder) indexUpdateFunc(ctx context.Context, seq margaret.Seq, val int return nil } - addr := abs.Author().StoredAddr() - addr += c.Contact.StoredAddr() + addr := storedrefs.Feed(abs.Author()) + addr += storedrefs.Feed(c.Contact) switch { case c.Following: err = idx.Set(ctx, addr, 1) @@ -130,7 +133,7 @@ func (b *builder) DeleteAuthor(who *refs.FeedRef) error { iter := txn.NewIterator(badger.DefaultIteratorOptions) defer iter.Close() - prefix := []byte(who.StoredAddr()) + prefix := []byte(storedrefs.Feed(who)) for iter.Seek(prefix); iter.ValidForPrefix(prefix); iter.Next() { it := iter.Item() @@ -169,33 +172,30 @@ func (b *builder) Build() (*Graph, error) { for iter.Rewind(); iter.Valid(); iter.Next() { it := iter.Item() k := it.Key() - if len(k) != 66 { + if len(k) != 68 { continue } - rawFrom := k[:33] - rawTo := k[33:] + rawFrom := k[:34] + rawTo := k[34:] if bytes.Equal(rawFrom, rawTo) { // contact self?! continue } - var to, from refs.StorageRef - if err := from.Unmarshal(rawFrom); err != nil { + var to, from tfk.Feed + if err := from.UnmarshalBinary(rawFrom); err != nil { return errors.Wrapf(err, "builder: couldnt idx key value (from)") } - if err := to.Unmarshal(rawTo); err != nil { + if err := to.UnmarshalBinary(rawTo); err != nil { return errors.Wrap(err, "builder: couldnt idx key value (to)") } bfrom := librarian.Addr(rawFrom) nFrom, has := dg.lookup[bfrom] if !has { - fromRef, err := from.FeedRef() - if err != nil { - return err - } + fromRef := from.Feed() nFrom = &contactNode{dg.NewNode(), fromRef.Copy(), ""} dg.AddNode(nFrom) @@ -205,10 +205,7 @@ func (b *builder) Build() (*Graph, error) { bto := librarian.Addr(rawTo) nTo, has := dg.lookup[bto] if !has { - toRef, err := to.FeedRef() - if err != nil { - return err - } + toRef := to.Feed() nTo = &contactNode{dg.NewNode(), toRef.Copy(), ""} dg.AddNode(nTo) dg.lookup[bto] = nTo @@ -260,7 +257,7 @@ type Lookup struct { } func (l Lookup) Dist(to *refs.FeedRef) ([]graph.Node, float64) { - bto := to.StoredAddr() + bto := storedrefs.Feed(to) nTo, has := l.lookup[bto] if !has { return nil, math.Inf(-1) @@ -277,7 +274,7 @@ func (b *builder) Follows(forRef *refs.FeedRef) (*ssb.StrFeedSet, error) { iter := txn.NewIterator(badger.DefaultIteratorOptions) defer iter.Close() - prefix := []byte(forRef.StoredAddr()) + prefix := []byte(storedrefs.Feed(forRef)) for iter.Seek(prefix); iter.ValidForPrefix(prefix); iter.Next() { it := iter.Item() k := it.Key() @@ -286,12 +283,12 @@ func (b *builder) Follows(forRef *refs.FeedRef) (*ssb.StrFeedSet, error) { if len(v) >= 1 && v[0] == '1' { // extract 2nd feed ref out of db key // TODO: use compact StoredAddr - var sr refs.StorageRef - err := sr.Unmarshal(k[33:]) + var sr tfk.Feed + err := sr.UnmarshalBinary(k[34:]) if err != nil { return errors.Wrapf(err, "follows(%s): invalid ref entry in db for feed", forRef.Ref()) } - if err := fs.AddStored(&sr); err != nil { + if err := fs.AddRef(sr.Feed()); err != nil { return errors.Wrapf(err, "follows(%s): couldn't add parsed ref feed", forRef.Ref()) } } diff --git a/graph/builder_test.go b/graph/builder_test.go index 0961fd5d..95863f3e 100644 --- a/graph/builder_test.go +++ b/graph/builder_test.go @@ -21,10 +21,8 @@ import ( "go.cryptoscope.co/ssb" "go.cryptoscope.co/ssb/internal/ctxutils" - "go.cryptoscope.co/ssb/internal/mutil" "go.cryptoscope.co/ssb/internal/testutils" "go.cryptoscope.co/ssb/multilogs" - "go.cryptoscope.co/ssb/plugins2/bytype" "go.cryptoscope.co/ssb/repo" ) @@ -80,7 +78,7 @@ func TestBadger(t *testing.T) { func makeTypedLog(t *testing.T) testStore { r := require.New(t) - info := testutils.NewRelativeTimeLogger(nil) + // info := testutils.NewRelativeTimeLogger(nil) tRepoPath, err := ioutil.TempDir("", "test_mlog") r.NoError(err) @@ -99,23 +97,25 @@ func makeTypedLog(t *testing.T) testStore { tc.root = tRootLog tc.userLogs = uf - mt, serveMT, err := repo.OpenMultiLog(tRepo, "byType", bytype.IndexUpdate) - r.NoError(err, "sbot: failed to open message type sublogs") - mtErrc := serveLog(ctx, "type logs", tRootLog, serveMT, true) - - contactLog, err := mt.Get(librarian.Addr("contact")) - r.NoError(err, "sbot: failed to open message contact sublog") + panic("TODO: plugin refactor") + /* + mt, serveMT, err := repo.OpenMultiLog(tRepo, "byType", bytype.IndexUpdate) + r.NoError(err, "sbot: failed to open message type sublogs") + mtErrc := serveLog(ctx, "type logs", tRootLog, serveMT, true) - directedContactLog := mutil.Indirect(tRootLog, contactLog) - tc.gbuilder, err = NewLogBuilder(info, directedContactLog) - r.NoError(err, "sbot: NewLogBuilder failed") + contactLog, err := mt.Get(librarian.Addr("contact")) + r.NoError(err, "sbot: failed to open message contact sublog") + directedContactLog := mutil.Indirect(tRootLog, contactLog) + tc.gbuilder, err = NewLogBuilder(info, directedContactLog) + r.NoError(err, "sbot: NewLogBuilder failed") + */ tc.close = func() { r.NoError(uf.Close()) - r.NoError(mt.Close()) + // r.NoError(mt.Close()) cancel() - for err := range mergedErrors(ufErrc, mtErrc) { + for err := range mergedErrors(ufErrc) { r.NoError(err, "from chan") } t.Log("closed scenary") diff --git a/graph/graph.go b/graph/graph.go index 4cca3166..88fafb55 100644 --- a/graph/graph.go +++ b/graph/graph.go @@ -9,6 +9,7 @@ import ( "go.cryptoscope.co/librarian" "go.cryptoscope.co/ssb" + "go.cryptoscope.co/ssb/internal/storedrefs" refs "go.mindeco.de/ssb-refs" "gonum.org/v1/gonum/graph" "gonum.org/v1/gonum/graph/path" @@ -33,11 +34,11 @@ func NewGraph() *Graph { func (g *Graph) getEdge(from, to *refs.FeedRef) (graph.WeightedEdge, bool) { g.Mutex.Lock() defer g.Mutex.Unlock() - nFrom, has := g.lookup[from.StoredAddr()] + nFrom, has := g.lookup[storedrefs.Feed(from)] if !has { return nil, false } - nTo, has := g.lookup[to.StoredAddr()] + nTo, has := g.lookup[storedrefs.Feed(to)] if !has { return nil, false } @@ -68,7 +69,7 @@ func (g *Graph) BlockedList(from *refs.FeedRef) *ssb.StrFeedSet { g.Mutex.Lock() defer g.Mutex.Unlock() blocked := ssb.NewFeedSet(0) - nFrom, has := g.lookup[from.StoredAddr()] + nFrom, has := g.lookup[storedrefs.Feed(from)] if !has { return blocked } @@ -89,7 +90,7 @@ func (g *Graph) BlockedList(from *refs.FeedRef) *ssb.StrFeedSet { func (g *Graph) MakeDijkstra(from *refs.FeedRef) (*Lookup, error) { g.Mutex.Lock() defer g.Mutex.Unlock() - nFrom, has := g.lookup[from.StoredAddr()] + nFrom, has := g.lookup[storedrefs.Feed(from)] if !has { return nil, ErrNoSuchFrom{Who: from} } diff --git a/graph/logbuilder.go b/graph/logbuilder.go index 6565c8b3..15a734f7 100644 --- a/graph/logbuilder.go +++ b/graph/logbuilder.go @@ -12,6 +12,7 @@ import ( "github.com/pkg/errors" "go.cryptoscope.co/luigi" "go.cryptoscope.co/margaret" + "go.cryptoscope.co/ssb/internal/storedrefs" refs "go.mindeco.de/ssb-refs" "gonum.org/v1/gonum/graph" "gonum.org/v1/gonum/graph/simple" @@ -129,35 +130,18 @@ func (b *logBuilder) buildGraph(ctx context.Context, v interface{}, err error) e return nil } - bfrom := author.StoredAddr() + bfrom := storedrefs.Feed(author) nFrom, has := b.current.lookup[bfrom] if !has { - // stupid copy - sr, err := refs.NewStorageRef(author) - if err != nil { - return errors.Wrap(err, "failed to create graph node for author") - } - fr, err := sr.FeedRef() - if err != nil { - return errors.Wrap(err, "failed to create graph node for author") - } - nFrom = &contactNode{dg.NewNode(), fr, ""} + nFrom = &contactNode{dg.NewNode(), author.Copy(), ""} dg.AddNode(nFrom) b.current.lookup[bfrom] = nFrom } - bto := contact.StoredAddr() + bto := storedrefs.Feed(contact) nTo, has := b.current.lookup[bto] if !has { - sr, err := refs.NewStorageRef(contact) - if err != nil { - return errors.Wrap(err, "failed to create graph node for contact") - } - fr, err := sr.FeedRef() - if err != nil { - return errors.Wrap(err, "failed to create graph node for author") - } - nTo = &contactNode{dg.NewNode(), fr, ""} + nTo = &contactNode{dg.NewNode(), contact.Copy(), ""} dg.AddNode(nTo) b.current.lookup[bto] = nTo } @@ -188,7 +172,7 @@ func (b *logBuilder) Follows(from *refs.FeedRef) (*ssb.StrFeedSet, error) { if err != nil { return nil, errors.Wrap(err, "follows: couldn't build graph") } - fb := from.StoredAddr() + fb := storedrefs.Feed(from) nFrom, has := g.lookup[fb] if !has { return nil, ErrNoSuchFrom{from} @@ -218,7 +202,7 @@ func (b *logBuilder) Hops(from *refs.FeedRef, max int) *ssb.StrFeedSet { } b.current.Lock() defer b.current.Unlock() - fb := from.StoredAddr() + fb := storedrefs.Feed(from) nFrom, has := g.lookup[fb] if !has { fs := ssb.NewFeedSet(1) diff --git a/graph/people_test.go b/graph/people_test.go index 4d5efeb6..26ec4350 100644 --- a/graph/people_test.go +++ b/graph/people_test.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.cryptoscope.co/ssb" + "go.cryptoscope.co/ssb/internal/storedrefs" refs "go.mindeco.de/ssb-refs" ) @@ -257,7 +258,7 @@ func (tc PeopleTestCase) run(mk func(t *testing.T) testStore) func(t *testing.T) r.NoError(err, "failed to build graph for debugging") g.Lock() for nick, pub := range state.peers { - newKey := pub.key.Id.StoredAddr() + newKey := storedrefs.Feed(pub.key.Id) node, ok := g.lookup[newKey] if !a.True(ok, "did not find peer %s in graph lookup table (len:%d)", nick, len(g.lookup)) { continue diff --git a/internal/muxmux/new_mux.go b/internal/muxmux/new_mux.go index 66a2b2ae..aebe338b 100644 --- a/internal/muxmux/new_mux.go +++ b/internal/muxmux/new_mux.go @@ -61,7 +61,7 @@ func (hm *HandlerMux) RegisterAsync(m muxrpc.Method, h AsyncHandler) { // RegisterSource registers a 'source' call for name method func (hm *HandlerMux) RegisterSource(m muxrpc.Method, h SourceHandler) { hm.handlers[m.String()] = sourceStub{ - logger: hm.logger, - h: h, + // logger: hm.logger, + h: h, } } diff --git a/internal/muxmux/source.go b/internal/muxmux/source.go index 1967fcf0..b002eff5 100644 --- a/internal/muxmux/source.go +++ b/internal/muxmux/source.go @@ -3,7 +3,6 @@ package muxmux import ( "context" - "github.com/go-kit/kit/log" "go.cryptoscope.co/luigi" "go.cryptoscope.co/muxrpc" ) @@ -16,24 +15,21 @@ func (sf SourceFunc) HandleSource(ctx context.Context, r *muxrpc.Request, snk lu // SourceHandler initiates a 'source' call, so the handler is supposed to send a stream of stuff to the peer. type SourceHandler interface { - HandleSource(context.Context, *muxrpc.Request, luigi.Sink) error + HandleSource(context.Context, *muxrpc.Request, luigi.Sink, muxrpc.Endpoint) error } -type sourceStub struct { - logger log.Logger +type sourceStub struct { h SourceHandler } func (hm sourceStub) HandleCall(ctx context.Context, req *muxrpc.Request, edp muxrpc.Endpoint) { // TODO: check call type - err := hm.h.HandleSource(ctx, req, req.Stream) + err := hm.h.HandleSource(ctx, req, req.Stream, edp) if err != nil { req.CloseWithError(err) return } - - // req.Stream.Close() } func (hm sourceStub) HandleConnect(ctx context.Context, edp muxrpc.Endpoint) {} diff --git a/internal/refactors/storedaddr.gotpl b/internal/refactors/storedaddr.gotpl new file mode 100644 index 00000000..1ffbcfc7 --- /dev/null +++ b/internal/refactors/storedaddr.gotpl @@ -0,0 +1,17 @@ +package p + +// +build ignore + +import ( + "go.cryptoscope.co/librarian" + "go.cryptoscope.co/ssb/internal/storedrefs" + refs "go.mindeco.de/ssb-refs" +) + +func before(r *refs.FeedRef) librarian.Addr { + return r.StoredAddr() +} + +func after(r *refs.FeedRef) librarian.Addr { + return storedrefs.Feed(r) +} diff --git a/internal/storedrefs/storedrefs.go b/internal/storedrefs/storedrefs.go new file mode 100644 index 00000000..43bad35a --- /dev/null +++ b/internal/storedrefs/storedrefs.go @@ -0,0 +1,23 @@ +// Package storedrefs provides methods to encode certain types as bytes, as used by the internal storage system. +package storedrefs + +import ( + "github.com/pkg/errors" + "go.cryptoscope.co/librarian" + refs "go.mindeco.de/ssb-refs" + "go.mindeco.de/ssb-refs/tfk" +) + +// Feed returns the key under which this ref is stored in the indexing system +func Feed(r *refs.FeedRef) librarian.Addr { + sr, err := tfk.FeedFromRef(r) + if err != nil { + panic(errors.Wrap(err, "failed to make stored ref")) + } + + b, err := sr.MarshalBinary() + if err != nil { + panic(errors.Wrap(err, "error while marshalling stored ref")) + } + return librarian.Addr(b) +} diff --git a/internal/testutils/streamMessageLog.go b/internal/testutils/streamMessageLog.go new file mode 100644 index 00000000..44392d57 --- /dev/null +++ b/internal/testutils/streamMessageLog.go @@ -0,0 +1,44 @@ +package testutils + +import ( + "context" + "encoding/hex" + "testing" + + "github.com/stretchr/testify/require" + "go.cryptoscope.co/luigi" + "go.cryptoscope.co/margaret" + refs "go.mindeco.de/ssb-refs" +) + +func StreamLog(t *testing.T, l margaret.Log) { + r := require.New(t) + src, err := l.Query() + r.NoError(err) + i := 0 + for { + v, err := src.Next(context.TODO()) + if luigi.IsEOS(err) { + break + } + + mm, ok := v.(refs.Message) + r.True(ok, "%T", v) + + t.Logf("log seq: %d - %s:%d (%s)", + i, + mm.Author().ShortRef(), + mm.Seq(), + mm.Key().ShortRef()) + + b := mm.ContentBytes() + if n := len(b); n > 128 { + t.Log("truncating", n, " to last 32 bytes") + b = b[len(b)-32:] + } + t.Logf("\n%s", hex.Dump(b)) + + i++ + } + +} diff --git a/internal/transform/qrymap.go b/internal/transform/qrymap.go index 35f40979..175d4a34 100644 --- a/internal/transform/qrymap.go +++ b/internal/transform/qrymap.go @@ -5,6 +5,7 @@ package transform import ( "context" "encoding/json" + "fmt" "github.com/cryptix/go/encodedTime" "github.com/pkg/errors" @@ -58,6 +59,8 @@ func NewKeyValueWrapper(output luigi.Sink, keyWrap bool) luigi.Sink { if !ok { return nil, errors.Errorf("kvwrap: wrong message type in seqWrapper - got %T", sv) } + default: + return nil, fmt.Errorf("failed to find message in empty interface(%T)", v) } if !keyWrap { diff --git a/message/legacy/stored.go b/message/legacy/stored.go index 44ffdbea..aac125df 100644 --- a/message/legacy/stored.go +++ b/message/legacy/stored.go @@ -14,24 +14,6 @@ import ( "go.cryptoscope.co/margaret" ) -// OldStoredMessage is only available to ease migration from old, pre-multimsg formats -type OldStoredMessage struct { - Author *refs.FeedRef // @... pubkey - Previous *refs.MessageRef // %... message hashsha - Key *refs.MessageRef // %... message hashsha - Sequence margaret.BaseSeq - Timestamp time.Time - Raw []byte // the original message for gossiping see ssb.EncodePreserveOrdering for why -} - -func (sm OldStoredMessage) String() string { - s := fmt.Sprintf("msg(%s:%d) %s", sm.Author.Ref(), sm.Sequence, sm.Key.Ref()) - b, _ := EncodePreserveOrder(sm.Raw) - s += "\n" - s += string(b) - return s -} - // really dislike the underlines but they are there to implement the message interface more easily type StoredMessage struct { @@ -45,12 +27,6 @@ type StoredMessage struct { // TODO: consider lazy decoding approach from gabbygrove to reduce storage overhead } -// could use this to unexport fields, would require lots of constructors though -// func (sm StoredMessage) MarshalBinary() ([]byte, error) { -// } -// func (sm *StoredMessage) UnmarshalBinary(data []byte) error { -// } - func (sm StoredMessage) String() string { s := fmt.Sprintf("msg(%s:%d) %s", sm.Author_.Ref(), sm.Sequence_, sm.Key_.Ref()) b, _ := EncodePreserveOrder(sm.Raw_) diff --git a/message/multimsg/multimsg_wrap.go b/message/multimsg/multimsg_wrap.go index 73350f1d..b01e5477 100644 --- a/message/multimsg/multimsg_wrap.go +++ b/message/multimsg/multimsg_wrap.go @@ -41,19 +41,6 @@ func (wl WrappedLog) Append(val interface{}) (margaret.Seq, error) { var mm MultiMessage - if osm, ok := val.(legacy.OldStoredMessage); ok { - mm.tipe = Legacy - mm.Message = &legacy.StoredMessage{ - Author_: osm.Author, - Previous_: osm.Previous, - Key_: osm.Key, - Sequence_: osm.Sequence, - Timestamp_: osm.Timestamp, - Raw_: osm.Raw, - } - return wl.AlterableLog.Append(mm) - } - abs, ok := val.(refs.Message) if !ok { return margaret.SeqEmpty, errors.Errorf("wrappedLog: not a refs.Message: %T", val) diff --git a/message/publish.go b/message/publish.go index 605f22cd..0b610b49 100644 --- a/message/publish.go +++ b/message/publish.go @@ -17,6 +17,7 @@ import ( refs "go.mindeco.de/ssb-refs" "go.cryptoscope.co/ssb" + "go.cryptoscope.co/ssb/internal/storedrefs" "go.cryptoscope.co/ssb/message/legacy" ) @@ -132,7 +133,7 @@ func (pl *publishLog) Append(val interface{}) (margaret.Seq, error) { // then it's pretty printed again (now with the signature inside the message) to construct it's SHA256 hash, // which is used to reference it (by replys and it's previous) func OpenPublishLog(rootLog margaret.Log, sublogs multilog.MultiLog, kp *ssb.KeyPair, opts ...PublishOption) (ssb.Publisher, error) { - authorLog, err := sublogs.Get(kp.Id.StoredAddr()) + authorLog, err := sublogs.Get(storedrefs.Feed(kp.Id)) if err != nil { return nil, errors.Wrap(err, "publish: failed to open sublog for author") } diff --git a/message/publish_formats_test.go b/message/publish_formats_test.go index b1e856ae..2a3bd518 100644 --- a/message/publish_formats_test.go +++ b/message/publish_formats_test.go @@ -15,6 +15,7 @@ import ( "go.cryptoscope.co/ssb" "go.cryptoscope.co/ssb/internal/asynctesting" + "go.cryptoscope.co/ssb/internal/storedrefs" "go.cryptoscope.co/ssb/message/legacy" "go.cryptoscope.co/ssb/message/multimsg" "go.cryptoscope.co/ssb/multilogs" @@ -59,7 +60,7 @@ func TestFormatsSimple(t *testing.T) { r.NoError(err) testAuthor.Id.Algo = tc.ff - authorLog, err := userFeeds.Get(testAuthor.Id.StoredAddr()) + authorLog, err := userFeeds.Get(storedrefs.Feed(testAuthor.Id)) r.NoError(err) w, err := OpenPublishLog(rl, userFeeds, testAuthor) diff --git a/message/publish_test.go b/message/publish_test.go index 14ef212b..8ba68765 100644 --- a/message/publish_test.go +++ b/message/publish_test.go @@ -15,6 +15,7 @@ import ( "go.cryptoscope.co/ssb" "go.cryptoscope.co/ssb/internal/asynctesting" + "go.cryptoscope.co/ssb/internal/storedrefs" "go.cryptoscope.co/ssb/multilogs" "go.cryptoscope.co/ssb/repo" refs "go.mindeco.de/ssb-refs" @@ -47,7 +48,7 @@ func TestSignMessages(t *testing.T) { testAuthor, err := ssb.NewKeyPair(staticRand) r.NoError(err) - authorLog, err := userFeeds.Get(testAuthor.Id.StoredAddr()) + authorLog, err := userFeeds.Get(storedrefs.Feed(testAuthor.Id)) r.NoError(err) w, err := OpenPublishLog(rl, userFeeds, testAuthor) diff --git a/message/requests.go b/message/requests.go index 397433ff..dbc8f435 100644 --- a/message/requests.go +++ b/message/requests.go @@ -19,7 +19,7 @@ func NewCreateHistArgsFromMap(argMap map[string]interface{}) (*CreateHistArgs, e var qry CreateHistArgs for k, v := range argMap { switch k = strings.ToLower(k); k { - case "live", "keys", "values", "reverse", "asjson": + case "live", "keys", "values", "reverse", "asjson", "private": b, ok := v.(bool) if !ok { return nil, errors.Errorf("ssb/message: not a bool for %s", k) @@ -35,6 +35,8 @@ func NewCreateHistArgsFromMap(argMap map[string]interface{}) (*CreateHistArgs, e qry.Reverse = b case "asjson": qry.AsJSON = b + case "private": + qry.Private = b } case "type", "id": @@ -49,12 +51,8 @@ func NewCreateHistArgsFromMap(argMap map[string]interface{}) (*CreateHistArgs, e if err != nil { return nil, errors.Wrapf(err, "ssb/message: not a feed ref") } - - // TODO: - // case "type": - // qry.Type = val } - case "seq", "limit": + case "seq", "limit", "gt", "lt": n, ok := v.(float64) if !ok { return nil, errors.Errorf("ssb/message: not a float64(%T) for %s", v, k) @@ -64,6 +62,10 @@ func NewCreateHistArgsFromMap(argMap map[string]interface{}) (*CreateHistArgs, e qry.Seq = int64(n) case "limit": qry.Limit = int64(n) + case "gt": + qry.Gt = int64(n) + case "lt": + qry.Lt = int64(n) } } } @@ -80,21 +82,24 @@ type CommonArgs struct { Values bool `json:"values,omitempty"` Live bool `json:"live,omitempty"` - // Raw ??? - Raw bool `json:"raw"` - Seqs bool `json:"seqs"` - Cache bool `json:"cache"` + // flume-stub experiments + // Raw bool `json:"raw"` + // Seqs bool `json:"seqs"` + // Cache bool `json:"cache"` // this field is used to tell muxrpc into wich type the messages should be marshaled into. // for instance, it could be json.RawMessage or a map or a struct // TODO: find a nice way to have a default here MarshalType interface{} `json:"-"` + + Private bool `json:"private,omitempty"` } type StreamArgs struct { Limit int64 `json:"limit,omitempty"` - Gt int64 `json:"gt"` + Gt int64 `json:"gt,omitempty"` + Lt int64 `json:"lt,omitempty"` Reverse bool `json:"reverse,omitempty"` } @@ -128,5 +133,10 @@ type MessagesByTypeArgs struct { type TanglesArgs struct { CommonArgs StreamArgs - Root refs.MessageRef `json:"root"` + + Root *refs.MessageRef `json:"root"` + + // indicate the v2 subtangle (group, ...) + // empty string for v1 tangle + Name string `json:"name"` } diff --git a/multilogs/combined.go b/multilogs/combined.go index 68a410da..ca0ebc6c 100644 --- a/multilogs/combined.go +++ b/multilogs/combined.go @@ -10,12 +10,14 @@ import ( "os" "path/filepath" "sync" + "time" "github.com/keks/persist" "github.com/pkg/errors" "go.cryptoscope.co/librarian" "go.cryptoscope.co/margaret" "go.cryptoscope.co/margaret/multilog/roaring" + "go.cryptoscope.co/ssb/internal/storedrefs" "go.cryptoscope.co/ssb/message/multimsg" "go.cryptoscope.co/ssb/private" "go.cryptoscope.co/ssb/repo" @@ -27,8 +29,9 @@ import ( // Compared to the "old" fatbot approach of just having 4 independant indexes, // this one updates all 4 of them, resulting in less read-overhead // while also being able to index private massages by tangle and type. -func NewCombinedIndex(repoPath string, box *private.Manager, self *refs.FeedRef, u, p, bt, tan *roaring.MultiLog) (librarian.SinkIndex, error) { - statePath := repo.New(repoPath).GetPath(repo.PrefixMultiLog, "combined-state.json") +func NewCombinedIndex(repoPath string, box *private.Manager, self *refs.FeedRef, res *repo.SequenceResolver, u, p, bt, tan *roaring.MultiLog) (*combinedIndex, error) { + r := repo.New(repoPath) + statePath := r.GetPath(repo.PrefixMultiLog, "combined-state.json") mode := os.O_RDWR | os.O_EXCL if _, err := os.Stat(statePath); os.IsNotExist(err) { mode |= os.O_CREATE @@ -48,12 +51,16 @@ func NewCombinedIndex(repoPath string, box *private.Manager, self *refs.FeedRef, byType: bt, tangles: tan, + seqresolver: res, + file: idxStateFile, l: &sync.Mutex{}, } return idx, nil } +var _ librarian.SinkIndex = (*combinedIndex)(nil) + type combinedIndex struct { self *refs.FeedRef boxer *private.Manager @@ -63,6 +70,8 @@ type combinedIndex struct { byType *roaring.MultiLog tangles *roaring.MultiLog + seqresolver *repo.SequenceResolver + file *os.File l *sync.Mutex } @@ -88,6 +97,10 @@ func (slog *combinedIndex) Pour(ctx context.Context, swv interface{}) error { if isNulled, ok := v.(error); ok { if margaret.IsErrNulled(isNulled) { + err = slog.seqresolver.Append(seq.Seq(), 0, time.Now(), time.Now()) + if err != nil { + return errors.Wrap(err, "error updating sequence resolver (nulled message)") + } return nil } return isNulled @@ -98,9 +111,14 @@ func (slog *combinedIndex) Pour(ctx context.Context, swv interface{}) error { return errors.Errorf("error casting message. got type %T", v) } + err = slog.seqresolver.Append(seq.Seq(), abstractMsg.Seq(), abstractMsg.Claimed(), abstractMsg.Received()) + if err != nil { + return errors.Wrap(err, "error updating sequence resolver") + } + author := abstractMsg.Author() - authorLog, err := slog.users.Get(author.StoredAddr()) + authorLog, err := slog.users.Get(storedrefs.Feed(author)) if err != nil { return errors.Wrap(err, "error opening sublog") } @@ -116,14 +134,16 @@ func (slog *combinedIndex) Pour(ctx context.Context, swv interface{}) error { cleartext, err := slog.tryDecrypt(abstractMsg, seq) if err != nil { if err == errSkip { + // yes it's a boxed message but we can't read it (yet) return nil } + // something went horribly wrong return err } content = cleartext } - // by type:... + // by type:... and tangles (v1 & v2) var jsonContent struct { Type string Root *refs.MessageRef @@ -149,7 +169,7 @@ func (slog *combinedIndex) Pour(ctx context.Context, swv interface{}) error { return fmt.Errorf("ssb: untyped message") } - typedLog, err := slog.byType.Get(librarian.Addr(typeStr)) + typedLog, err := slog.byType.Get(librarian.Addr("string:" + typeStr)) if err != nil { return errors.Wrap(err, "error opening sublog") } @@ -173,6 +193,9 @@ func (slog *combinedIndex) Pour(ctx context.Context, swv interface{}) error { } for tname, tip := range jsonContent.Tangles { + if tname == "" { + continue + } addr := librarian.Addr(append([]byte("v2:"+tname+":"), tip.Root.Hash...)) tangleLog, err := slog.tangles.Get(addr) if err != nil { @@ -188,7 +211,9 @@ func (slog *combinedIndex) Pour(ctx context.Context, swv interface{}) error { } // Close does nothing. -func (slog *combinedIndex) Close() error { return nil } +func (slog *combinedIndex) Close() error { + return nil +} // QuerySpec returns the query spec that queries the next needed messages from the log func (slog *combinedIndex) QuerySpec() margaret.QuerySpec { @@ -205,6 +230,11 @@ func (slog *combinedIndex) QuerySpec() margaret.QuerySpec { seq = margaret.SeqEmpty } + if resN := slog.seqresolver.Seq() - 1; resN != seq.Seq() { + err := fmt.Errorf("combined idx (has:%d, will: %d)", resN, seq.Seq()) + return margaret.ErrorQuerySpec(err) + } + return margaret.MergeQuerySpec( margaret.Gt(seq), margaret.SeqWrap(true), @@ -226,28 +256,49 @@ func (slog *combinedIndex) tryDecrypt(msg refs.Message, rxSeq margaret.Seq) ([]b var ( cleartext []byte idxAddr librarian.Addr - retErr error ) + + /* as a help for re-indexing, keep track of all box1 and box2 messages. + + later we can get the set of messages we might need to re-index by + 1) taking meta:box2 + 2) ANDing it with the one of the author (intersection) + 3) subtracting all the messages we _can_ read (private:box2:$ourFeed) + */ + if box1 != nil { + idxAddr = librarian.Addr("meta:box1") + } else { + idxAddr = librarian.Addr("meta:box2") + } + + boxTyped, err := slog.private.Get(idxAddr) + if err != nil { + return nil, err + } + if _, err := boxTyped.Append(rxSeq.Seq()); err != nil { + return nil, errors.Wrapf(err, "private: error marking type:box") + } + + // try decrypt and pass on the clear text if box1 != nil { content, err := slog.boxer.DecryptBox1(box1) if err != nil { - idxAddr = librarian.Addr("notForUs:box1") - retErr = errSkip - } else { - idxAddr = librarian.Addr("box1:") + slog.self.StoredAddr() - cleartext = content + return nil, errSkip } + + idxAddr = librarian.Addr("box1:") + storedrefs.Feed(slog.self) + cleartext = content } else if box2 != nil { content, err := slog.boxer.DecryptBox2(box2, msg.Author(), msg.Previous()) if err != nil { - idxAddr = librarian.Addr("notForUs:box2") - retErr = errSkip - } else { - // instead by group root? could be PM... hmm - // would be nice to keep multi-keypair support here but might need rething of the gorups manager - idxAddr = librarian.Addr("box2:") + slog.self.StoredAddr() - cleartext = content + return nil, errSkip } + + // instead by group root? could be PM... hmm + // would be nice to keep multi-keypair support here + // but might need to rethink the group manager + idxAddr = librarian.Addr("box2:") + storedrefs.Feed(slog.self) + cleartext = content } else { return nil, fmt.Errorf("tryDecrypt: not skipped but also not valid content") } @@ -260,11 +311,11 @@ func (slog *combinedIndex) tryDecrypt(msg refs.Message, rxSeq margaret.Seq) ([]b return nil, errors.Wrapf(err, "private/readidx: error appending PM") } - return cleartext, retErr + return cleartext, nil } var ( - errSkip = fmt.Errorf("ssb: skip - not for us") + errSkip = fmt.Errorf("ssb: skip - not for us") errSkipBox1 = fmt.Errorf("ssb: skip box1 message") errSkipBox2 = fmt.Errorf("ssb: skip box2 message") ) diff --git a/multilogs/private_readidx.go b/multilogs/private_readidx.go index 89c660fc..8b8877e7 100644 --- a/multilogs/private_readidx.go +++ b/multilogs/private_readidx.go @@ -17,6 +17,7 @@ import ( refs "go.mindeco.de/ssb-refs" "go.cryptoscope.co/ssb" + "go.cryptoscope.co/ssb/internal/storedrefs" "go.cryptoscope.co/ssb/message/multimsg" "go.cryptoscope.co/ssb/private/box" "go.cryptoscope.co/ssb/repo" @@ -117,7 +118,7 @@ func (pr Private) update(ctx context.Context, seq margaret.Seq, val interface{}, if _, err := pr.boxer.Decrypt(kp, boxedContent); err != nil { continue } - userPrivs, err := mlog.Get(kp.Id.StoredAddr()) + userPrivs, err := mlog.Get(storedrefs.Feed(kp.Id)) if err != nil { return errors.Wrapf(err, "private/readidx: error opening priv sublog for %s", kp.Id.Ref()) } diff --git a/multilogs/userfeeds.go b/multilogs/userfeeds.go index 52d6a670..a480189a 100644 --- a/multilogs/userfeeds.go +++ b/multilogs/userfeeds.go @@ -4,20 +4,24 @@ package multilogs import ( "context" + "fmt" "github.com/pkg/errors" + "go.cryptoscope.co/librarian" "go.cryptoscope.co/margaret" "go.cryptoscope.co/margaret/multilog" + "go.cryptoscope.co/margaret/multilog/roaring" + "go.cryptoscope.co/ssb/internal/storedrefs" + "go.cryptoscope.co/ssb/repo" refs "go.mindeco.de/ssb-refs" ) const IndexNameFeeds = "userFeeds" -/* deprecated func OpenUserFeeds(r repo.Interface) (*roaring.MultiLog, librarian.SinkIndex, error) { + fmt.Println("warning: OpenUserFeeds is deprecated for NewCombinedIndex") return repo.OpenFileSystemMultiLog(r, IndexNameFeeds, UserFeedsUpdate) } -*/ func UserFeedsUpdate(ctx context.Context, seq margaret.Seq, value interface{}, mlog multilog.MultiLog) error { if nulled, ok := value.(error); ok { @@ -37,7 +41,7 @@ func UserFeedsUpdate(ctx context.Context, seq margaret.Seq, value interface{}, m return errors.Errorf("nil author on message?! %v (%d)", value, seq.Seq()) } - authorLog, err := mlog.Get(author.StoredAddr()) + authorLog, err := mlog.Get(storedrefs.Feed(author)) if err != nil { return errors.Wrap(err, "error opening sublog") } diff --git a/network/new.go b/network/new.go index e3a57ae1..c407c4ac 100644 --- a/network/new.go +++ b/network/new.go @@ -64,10 +64,13 @@ type node struct { log log.Logger - lisClose sync.Once + listening chan struct{} + + listenerLock sync.Mutex + lisClose sync.Once + lis net.Listener dialer netwrap.Dialer - l net.Listener localDiscovRx *Discoverer localDiscovTx *Advertiser secretServer *secretstream.Server @@ -77,8 +80,6 @@ type node struct { beforeCryptoConnWrappers []netwrap.ConnWrapper afterSecureConnWrappers []netwrap.ConnWrapper - listening chan struct{} - remotesLock sync.Mutex remotes map[string]muxrpc.Endpoint @@ -332,17 +333,22 @@ func (n *node) Serve(ctx context.Context, wrappers ...muxrpc.HandlerWrapper) err lisWrap := netwrap.NewListenerWrapper(n.secretServer.Addr(), append(n.opts.BefreCryptoWrappers, n.secretServer.ConnWrapper())...) var err error - n.l, err = netwrap.Listen(n.opts.ListenAddr, lisWrap) + n.listenerLock.Lock() + n.lis, err = netwrap.Listen(n.opts.ListenAddr, lisWrap) if err != nil { - + n.listenerLock.Unlock() return errors.Wrap(err, "error creating listener") } n.lisClose = sync.Once{} // reset once close(n.listening) + n.listenerLock.Unlock() - defer func() { + defer func() { // refresh listener to re-call n.lisClose.Do(func() { - n.l.Close() + n.listenerLock.Lock() + n.lis.Close() + n.lis = nil + n.listenerLock.Unlock() }) n.listening = make(chan struct{}) }() @@ -385,7 +391,13 @@ func (n *node) Serve(ctx context.Context, wrappers ...muxrpc.HandlerWrapper) err go func() { defer close(newConn) for { - conn, err := n.l.Accept() + n.listenerLock.Lock() + if n.lis == nil { + n.listenerLock.Unlock() + return + } + n.listenerLock.Unlock() + conn, err := n.lis.Accept() if err != nil { if strings.Contains(err.Error(), "use of closed network connection") { // yikes way of handling this @@ -462,7 +474,7 @@ func (n *node) Connect(ctx context.Context, addr net.Addr) error { func (n *node) GetListenAddr() net.Addr { _, ok := <-n.listening if !ok { - return n.l.Addr() + return n.lis.Addr() } level.Error(n.log).Log("msg", "listener not ready") return nil @@ -490,11 +502,12 @@ func (n *node) Close() error { return errors.Wrap(err, "ssb: failed to close http listener") } } - - if n.l != nil { + n.listenerLock.Lock() + defer n.listenerLock.Unlock() + if n.lis != nil { var closeErr error n.lisClose.Do(func() { - closeErr = n.l.Close() + closeErr = n.lis.Close() }) if closeErr != nil && !strings.Contains(errors.Cause(closeErr).Error(), "use of closed network connection") { return errors.Wrap(closeErr, "ssb: network node failed to close it's listener") diff --git a/plugins/blobs/get.go b/plugins/blobs/get.go index 008228fe..854695f7 100644 --- a/plugins/blobs/get.go +++ b/plugins/blobs/get.go @@ -6,7 +6,6 @@ import ( "context" "encoding/json" "io" - "time" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" @@ -64,7 +63,6 @@ func (h getHandler) HandleCall(ctx context.Context, req *muxrpc.Request, edp mux sz, err := h.bs.Size(wantedRef) if err != nil { req.Stream.CloseWithError(errors.New("do not have blob")) - checkAndLog(errLog, errors.Wrap(err, "error closing stream with error")) return } @@ -74,16 +72,13 @@ func (h getHandler) HandleCall(ctx context.Context, req *muxrpc.Request, edp mux } logger = log.With(logger, "blob", wantedRef.ShortRef()) - info := level.Info(logger) errLog = level.Error(logger) r, err := h.bs.Get(wantedRef) if err != nil { - err = req.Stream.CloseWithError(errors.New("do not have blob")) - checkAndLog(errLog, errors.Wrap(err, "error closing stream with error")) + req.Stream.CloseWithError(errors.New("do not have blob")) return } - start := time.Now() w := muxrpc.NewSinkWriter(req.Stream) _, err = io.Copy(w, r) @@ -91,7 +86,7 @@ func (h getHandler) HandleCall(ctx context.Context, req *muxrpc.Request, edp mux err = w.Close() checkAndLog(errLog, errors.Wrap(err, "error closing blob output")) - if err == nil { - info.Log("event", "transmission successfull", "took", time.Since(start)) - } + // if err == nil { + // info.Log("event", "transmission successfull", "took", time.Since(start)) + // } } diff --git a/plugins/friends/sources.go b/plugins/friends/sources.go index ea116eca..b3e4ff1b 100644 --- a/plugins/friends/sources.go +++ b/plugins/friends/sources.go @@ -20,7 +20,7 @@ type blocksSrc struct { builder graph.Builder } -func (h blocksSrc) HandleSource(ctx context.Context, req *muxrpc.Request, snk luigi.Sink) error { +func (h blocksSrc) HandleSource(ctx context.Context, req *muxrpc.Request, snk luigi.Sink, edp muxrpc.Endpoint) error { type argT struct { Who refs.FeedRef } @@ -68,7 +68,7 @@ type HopsArgs struct { Max uint `json:"max"` } -func (h hopsSrc) HandleSource(ctx context.Context, req *muxrpc.Request, snk luigi.Sink) error { +func (h hopsSrc) HandleSource(ctx context.Context, req *muxrpc.Request, snk luigi.Sink, edp muxrpc.Endpoint) error { var args []HopsArgs if err := json.Unmarshal(req.RawArgs, &args); err != nil { return fmt.Errorf("invalid argument on isFollowing call: %w", err) diff --git a/plugins/gossip/feed_manager.go b/plugins/gossip/feed_manager.go index a00bb767..deceed8f 100644 --- a/plugins/gossip/feed_manager.go +++ b/plugins/gossip/feed_manager.go @@ -20,6 +20,7 @@ import ( "go.cryptoscope.co/muxrpc" "go.cryptoscope.co/ssb" "go.cryptoscope.co/ssb/internal/mutil" + "go.cryptoscope.co/ssb/internal/storedrefs" "go.cryptoscope.co/ssb/internal/transform" "go.cryptoscope.co/ssb/message" refs "go.mindeco.de/ssb-refs" @@ -29,9 +30,9 @@ import ( type FeedManager struct { rootCtx context.Context - RootLog margaret.Log - UserFeeds multilog.MultiLog - logger logging.Interface + ReceiveLog margaret.Log + UserFeeds multilog.MultiLog + logger logging.Interface liveFeeds map[string]*multiSink liveFeedsMut sync.Mutex @@ -45,20 +46,20 @@ type FeedManager struct { // Feeds. func NewFeedManager( ctx context.Context, - rootLog margaret.Log, + rxlog margaret.Log, userFeeds multilog.MultiLog, info logging.Interface, sysGauge metrics.Gauge, sysCtr metrics.Counter, ) *FeedManager { fm := &FeedManager{ - RootLog: rootLog, - UserFeeds: userFeeds, - logger: info, - rootCtx: ctx, - sysCtr: sysCtr, - sysGauge: sysGauge, - liveFeeds: make(map[string]*multiSink), + ReceiveLog: rxlog, + UserFeeds: userFeeds, + logger: info, + rootCtx: ctx, + sysCtr: sysCtr, + sysGauge: sysGauge, + liveFeeds: make(map[string]*multiSink), } // QUESTION: How should the error case be handled? go fm.serveLiveFeeds() @@ -86,13 +87,13 @@ func (m *FeedManager) pour(ctx context.Context, val interface{}, err error) erro } func (m *FeedManager) serveLiveFeeds() { - seqv, err := m.RootLog.Seq().Value() + seqv, err := m.ReceiveLog.Seq().Value() if err != nil { err = errors.Wrap(err, "failed to get root log sequence") panic(err) } - src, err := m.RootLog.Query( + src, err := m.ReceiveLog.Query( margaret.Gt(seqv.(margaret.BaseSeq)), margaret.Live(true), margaret.SeqWrap(true), @@ -204,7 +205,7 @@ func (m *FeedManager) CreateStreamHistory( return errors.Errorf("bad request: missing id argument") } // check what we got - userLog, err := m.UserFeeds.Get(arg.ID.StoredAddr()) + userLog, err := m.UserFeeds.Get(storedrefs.Feed(arg.ID)) if err != nil { return errors.Wrapf(err, "failed to open sublog for user") } @@ -225,12 +226,25 @@ func (m *FeedManager) CreateStreamHistory( // Make query limit := nonliveLimit(arg, latest) - resolved := mutil.Indirect(m.RootLog, userLog) - src, err := resolved.Query( - margaret.Gte(margaret.BaseSeq(arg.Seq)), + qryArgs := []margaret.QuerySpec{ margaret.Limit(int(limit)), margaret.Reverse(arg.Reverse), - ) + } + + if arg.Seq > 0 { + qryArgs = append(qryArgs, margaret.Gte(margaret.BaseSeq(arg.Seq))) + } + + if arg.Lt > 0 { + qryArgs = append(qryArgs, margaret.Lt(margaret.BaseSeq(arg.Lt))) + } + + if arg.Gt > 0 { + qryArgs = append(qryArgs, margaret.Gt(margaret.BaseSeq(arg.Gt))) + } + + resolved := mutil.Indirect(m.ReceiveLog, userLog) + src, err := resolved.Query(qryArgs...) if err != nil { return errors.Wrapf(err, "invalid user log query") } diff --git a/plugins/gossip/feed_manager_test.go b/plugins/gossip/feed_manager_test.go index 54f03924..115bc0c7 100644 --- a/plugins/gossip/feed_manager_test.go +++ b/plugins/gossip/feed_manager_test.go @@ -20,6 +20,7 @@ import ( "go.cryptoscope.co/ssb" "go.cryptoscope.co/ssb/internal/asynctesting" "go.cryptoscope.co/ssb/internal/ctxutils" + "go.cryptoscope.co/ssb/internal/storedrefs" "go.cryptoscope.co/ssb/internal/testutils" "go.cryptoscope.co/ssb/message" "go.cryptoscope.co/ssb/multilogs" @@ -152,7 +153,7 @@ func TestCreateHistoryStream(t *testing.T) { create(t, userFeedLen, "prefill") t.Log("created prefil") - log, err := userFeeds.Get(keyPair.Id.StoredAddr()) + log, err := userFeeds.Get(storedrefs.Feed(keyPair.Id)) r.NoError(err) seqv, err := log.Seq().Value() r.NoError(err) diff --git a/plugins/gossip/fetch.go b/plugins/gossip/fetch.go index a36f7ac9..e635ca98 100644 --- a/plugins/gossip/fetch.go +++ b/plugins/gossip/fetch.go @@ -23,6 +23,7 @@ import ( "go.cryptoscope.co/ssb" "go.cryptoscope.co/ssb/internal/neterr" + "go.cryptoscope.co/ssb/internal/storedrefs" "go.cryptoscope.co/ssb/message" ) @@ -93,7 +94,7 @@ func (h *handler) makeWorker(work <-chan *refs.FeedRef, ctx context.Context, edp func isIn(list []librarian.Addr, a *refs.FeedRef) bool { for _, el := range list { - if bytes.Equal([]byte(a.StoredAddr()), []byte(el)) { + if bytes.Equal([]byte(storedrefs.Feed(a)), []byte(el)) { return true } } @@ -113,7 +114,7 @@ func (g *handler) fetchFeed( default: } // check our latest - frAddr := fr.StoredAddr() + frAddr := storedrefs.Feed(fr) addr := string(frAddr) g.activeLock.Lock() _, ok := g.activeFetch[addr] @@ -158,7 +159,7 @@ func (g *handler) fetchFeed( if err != nil { return errors.Wrapf(err, "failed to look up root seq for latest user sublog") } - msgV, err := g.RootLog.Get(rootLogValue.(margaret.Seq)) + msgV, err := g.ReceiveLog.Get(rootLogValue.(margaret.Seq)) if err != nil { return errors.Wrapf(err, "failed retreive stored message") } @@ -210,7 +211,7 @@ func (g *handler) fetchFeed( } return err } - _, err = g.RootLog.Append(val) + _, err = g.ReceiveLog.Append(val) return errors.Wrap(err, "failed to append verified message to rootLog") }) diff --git a/plugins/gossip/handler.go b/plugins/gossip/handler.go index 8c104f48..c36d4455 100644 --- a/plugins/gossip/handler.go +++ b/plugins/gossip/handler.go @@ -18,16 +18,17 @@ import ( "go.cryptoscope.co/margaret/multilog" "go.cryptoscope.co/muxrpc" "go.cryptoscope.co/ssb" + "go.cryptoscope.co/ssb/internal/storedrefs" "go.cryptoscope.co/ssb/message" refs "go.mindeco.de/ssb-refs" ) type handler struct { - Id *refs.FeedRef - RootLog margaret.Log - UserFeeds multilog.MultiLog - WantList ssb.ReplicationLister - Info logging.Interface + Id *refs.FeedRef + ReceiveLog margaret.Log + UserFeeds multilog.MultiLog + WantList ssb.ReplicationLister + Info logging.Interface hmacSec HMACSecret hopCount int @@ -59,7 +60,7 @@ func (g *handler) HandleConnect(ctx context.Context, e muxrpc.Endpoint) { start := time.Now() // re-sync _our_ feed if we don't have it yet (re-onboarding of an existing feed) - hasSelf, err := multilog.Has(g.UserFeeds, g.Id.StoredAddr()) + hasSelf, err := multilog.Has(g.UserFeeds, storedrefs.Feed(g.Id)) if err != nil { info.Log("handleConnect", "multilog.Has(self)", "err", err) return @@ -75,7 +76,7 @@ func (g *handler) HandleConnect(ctx context.Context, e muxrpc.Endpoint) { } if g.promisc { - hasCallee, err := multilog.Has(g.UserFeeds, remoteRef.StoredAddr()) + hasCallee, err := multilog.Has(g.UserFeeds, storedrefs.Feed(remoteRef)) if err != nil { info.Log("handleConnect", "multilog.Has(callee)", "err", err) return @@ -220,8 +221,7 @@ func (g *handler) HandleCall( // } else { // dbgLog.Log("msg", "feed access granted") } - // query.Limit = 50 - // spew.Dump(query) + err = g.feedManager.CreateStreamHistory(ctx, req.Stream, query) if err != nil { if luigi.IsEOS(err) { diff --git a/plugins/gossip/plugin.go b/plugins/gossip/plugin.go index cf12d9a3..3f738813 100644 --- a/plugins/gossip/plugin.go +++ b/plugins/gossip/plugin.go @@ -26,7 +26,7 @@ func New( ctx context.Context, log logging.Interface, id *refs.FeedRef, - rootLog margaret.Log, + rxlog margaret.Log, userFeeds multilog.MultiLog, fm *FeedManager, wantList ssb.ReplicationLister, @@ -35,7 +35,7 @@ func New( h := &handler{ Id: id, - RootLog: rootLog, + ReceiveLog: rxlog, UserFeeds: userFeeds, feedManager: fm, WantList: wantList, @@ -74,7 +74,7 @@ func NewHist( ctx context.Context, log logging.Interface, id *refs.FeedRef, - rootLog margaret.Log, + rxlog margaret.Log, userFeeds multilog.MultiLog, wantList ssb.ReplicationLister, fm *FeedManager, @@ -83,7 +83,7 @@ func NewHist( h := &handler{ Id: id, - RootLog: rootLog, + ReceiveLog: rxlog, UserFeeds: userFeeds, feedManager: fm, WantList: wantList, diff --git a/plugins/groups/handler.go b/plugins/groups/handler.go new file mode 100644 index 00000000..a10cb211 --- /dev/null +++ b/plugins/groups/handler.go @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: MIT + +// Package groups supplies muxprc handlers for group managment. +package groups + +import ( + "github.com/cryptix/go/logging" + "github.com/go-kit/kit/log/level" + "go.cryptoscope.co/muxrpc" + + "go.cryptoscope.co/ssb" + "go.cryptoscope.co/ssb/internal/muxmux" + "go.cryptoscope.co/ssb/private" +) + +/* + + create: 'async', + invite: 'async', + publishTo: 'async', +*/ + +var ( + _ ssb.Plugin = plugin{} // compile-time type check + method = muxrpc.Method{"groups"} +) + +func checkAndLog(log logging.Interface, err error) { + if err != nil { + level.Error(log).Log("err", err) + } +} + +func New(log logging.Interface, groups *private.Manager) ssb.Plugin { + rootHdlr := muxmux.New(log) + + rootHdlr.RegisterAsync(append(method, "create"), create{ + log: log, + groups: groups, + }) + + rootHdlr.RegisterAsync(append(method, "publishTo"), publishTo{ + log: log, + groups: groups, + }) + + /* + rootHdlr.RegisterAsync(muxrpc.Method{"friends", "isBlocking"}, isBlockingH{ + log: log, + builder: b, + self: self, + }) + + rootHdlr.RegisterSource(muxrpc.Method{"friends", "blocks"}, blocksSrc{ + log: log, + builder: b, + self: self, + }) + + rootHdlr.RegisterSource(muxrpc.Method{"friends", "hops"}, hopsSrc{ + log: log, + builder: b, + self: self, + }) + + rootHdlr.RegisterAsync(muxrpc.Method{"friends", "plotsvg"}, plotSVGHandler{ + log: log, + builder: b, + self: self, + }) + */ + + return plugin{ + h: &rootHdlr, + log: log, + } +} + +type plugin struct { + h muxrpc.Handler + log logging.Interface +} + +func (plugin) Name() string { return method[0] } +func (plugin) Method() muxrpc.Method { return method } +func (p plugin) Handler() muxrpc.Handler { return p.h } diff --git a/plugins/groups/is.go b/plugins/groups/is.go new file mode 100644 index 00000000..8d736cb8 --- /dev/null +++ b/plugins/groups/is.go @@ -0,0 +1,135 @@ +package groups + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + "go.cryptoscope.co/muxrpc" + "go.cryptoscope.co/ssb/private" + refs "go.mindeco.de/ssb-refs" +) + +type create struct { + log log.Logger + + groups *private.Manager +} + +func (h create) HandleAsync(ctx context.Context, req *muxrpc.Request) (interface{}, error) { + var args []struct { + Name string + } + if err := json.Unmarshal(req.RawArgs, &args); err != nil { + return nil, fmt.Errorf("invalid argument on groups.create call: %w", err) + } + + if len(args) != 1 { + return nil, fmt.Errorf("expected one arg {name}") + } + a := args[0] + + cloaked, root, err := h.groups.Create(a.Name) + if err != nil { + return nil, err + } + + level.Info(h.log).Log("event", "group created", "cloaked", cloaked.Ref()) + + return struct { + Group *refs.MessageRef `json:"group_id"` + Root *refs.MessageRef `json:"root"` + }{cloaked, root}, err +} + +type publishTo struct { + log log.Logger + + groups *private.Manager +} + +func (h publishTo) HandleAsync(ctx context.Context, req *muxrpc.Request) (interface{}, error) { + var args []json.RawMessage + if err := json.Unmarshal(req.RawArgs, &args); err != nil { + return nil, fmt.Errorf("invalid argument on publishTo call: %w", err) + } + if len(args) != 2 { + return nil, fmt.Errorf("expected two args [groupID, content]") + } + + var groupID refs.MessageRef + err := json.Unmarshal(args[0], &groupID) + if err != nil { + return nil, fmt.Errorf("groupID needs to be a valid message ref: %w", err) + } + + if groupID.Algo != refs.RefAlgoCloakedGroup { + return nil, fmt.Errorf("groupID needs to be a cloaked message ref, not %s", groupID.Algo) + } + + newMsg, err := h.groups.PublishTo(&groupID, args[1]) + if err != nil { + return nil, fmt.Errorf("failed to publish message to group") + } + + return newMsg.Ref(), nil +} + +/* +type isBlockingH struct { + self refs.FeedRef + + log log.Logger + + builder graph.Builder +} + +func (h isBlockingH) HandleAsync(ctx context.Context, req *muxrpc.Request) (interface{}, error) { + var args []sourceDestArg + if err := json.Unmarshal(req.RawArgs, &args); err != nil { + return nil, fmt.Errorf("invalid argument on isBlocking call: %w", err) + } + if len(args) != 1 { + return nil, fmt.Errorf("expected one arg {source, dest}") + } + a := args[0] + + g, err := h.builder.Build() + if err != nil { + return nil, err + } + + return g.Blocks(&a.Source, &a.Dest), nil +} + +type plotSVGHandler struct { + self refs.FeedRef + + log log.Logger + + builder graph.Builder +} + +func (h plotSVGHandler) HandleAsync(ctx context.Context, req *muxrpc.Request) (interface{}, error) { + g, err := h.builder.Build() + if err != nil { + return nil, err + } + + fname, err := ioutil.TempFile("", "graph-*.svg") + if err != nil { + return nil, err + } + + err = g.RenderSVG(fname) + if err != nil { + fname.Close() + os.Remove(fname.Name()) + return nil, err + } + + return fname.Name(), fname.Close() +} +*/ diff --git a/plugins/legacyinvites/guest.go b/plugins/legacyinvites/guest.go index 5cdfbd8e..030ace7f 100644 --- a/plugins/legacyinvites/guest.go +++ b/plugins/legacyinvites/guest.go @@ -7,6 +7,7 @@ import ( "github.com/pkg/errors" "go.cryptoscope.co/muxrpc" + "go.cryptoscope.co/ssb/internal/storedrefs" refs "go.mindeco.de/ssb-refs" "go.cryptoscope.co/ssb" @@ -52,7 +53,7 @@ func (h acceptHandler) HandleCall(ctx context.Context, req *muxrpc.Request, edp return } - kvKey := []byte(guestRef.StoredAddr()) + kvKey := []byte(storedrefs.Feed(guestRef)) has, err := h.service.kv.Get(nil, kvKey) if err != nil { diff --git a/plugins/legacyinvites/service.go b/plugins/legacyinvites/service.go index 79ba6ea6..6666c023 100644 --- a/plugins/legacyinvites/service.go +++ b/plugins/legacyinvites/service.go @@ -16,6 +16,7 @@ import ( "modernc.org/kv" "go.cryptoscope.co/ssb" + "go.cryptoscope.co/ssb/internal/storedrefs" "go.cryptoscope.co/ssb/invite" "go.cryptoscope.co/ssb/repo" ) @@ -49,7 +50,7 @@ func (s *Service) Authorize(to *refs.FeedRef) error { s.mu.Lock() defer s.mu.Unlock() - kvKey := []byte(to.StoredAddr()) + kvKey := []byte(storedrefs.Feed(to)) if err := s.kv.BeginTransaction(); err != nil { return err @@ -130,7 +131,7 @@ func (s Service) Create(uses uint, note string) (*invite.Token, error) { return nil, fmt.Errorf("invite/create: generate seeded keypair (%w)", err) } - has, err := s.kv.Get(nil, []byte(inviteKeyPair.Id.StoredAddr())) + has, err := s.kv.Get(nil, []byte(storedrefs.Feed(inviteKeyPair.Id))) if err != nil { s.kv.Rollback() return nil, fmt.Errorf("invite/create: failed to probe new key (%w)", err) @@ -152,7 +153,7 @@ func (s Service) Create(uses uint, note string) (*invite.Token, error) { return nil, fmt.Errorf("invite/create: failed to marshal state data (%w)", err) } - err = s.kv.Set([]byte(seedRef.StoredAddr()), data) + err = s.kv.Set([]byte(storedrefs.Feed(seedRef)), data) if err != nil { s.kv.Rollback() return nil, fmt.Errorf("invite/create: failed to store state data (%w)", err) diff --git a/plugins/partial/feed.go b/plugins/partial/feed.go index 28ef1efc..1ff91bbc 100644 --- a/plugins/partial/feed.go +++ b/plugins/partial/feed.go @@ -17,7 +17,7 @@ type getFeedHandler struct { fm *gossip.FeedManager } -func (h getFeedHandler) HandleSource(ctx context.Context, req *muxrpc.Request, snk luigi.Sink) error { +func (h getFeedHandler) HandleSource(ctx context.Context, req *muxrpc.Request, snk luigi.Sink, edp muxrpc.Endpoint) error { args := req.Args() if len(args) < 1 { @@ -52,7 +52,7 @@ type getFeedReverseHandler struct { fm *gossip.FeedManager } -func (h getFeedReverseHandler) HandleSource(ctx context.Context, req *muxrpc.Request, snk luigi.Sink) error { +func (h getFeedReverseHandler) HandleSource(ctx context.Context, req *muxrpc.Request, snk luigi.Sink, edp muxrpc.Endpoint) error { args := req.Args() if len(args) < 1 { diff --git a/plugins/partial/oftype.go b/plugins/partial/oftype.go index 6ccbcd59..7b7a3f67 100644 --- a/plugins/partial/oftype.go +++ b/plugins/partial/oftype.go @@ -11,6 +11,7 @@ import ( "go.cryptoscope.co/margaret" "go.cryptoscope.co/margaret/multilog/roaring" "go.cryptoscope.co/muxrpc" + "go.cryptoscope.co/ssb/internal/storedrefs" refs "go.mindeco.de/ssb-refs" ) @@ -21,7 +22,7 @@ type getMessagesOfTypeHandler struct { bytype *roaring.MultiLog } -func (h getMessagesOfTypeHandler) HandleSource(ctx context.Context, req *muxrpc.Request, snk luigi.Sink) error { +func (h getMessagesOfTypeHandler) HandleSource(ctx context.Context, req *muxrpc.Request, snk luigi.Sink, edp muxrpc.Endpoint) error { if len(req.Args()) < 1 { return errors.Errorf("invalid arguments") } @@ -69,7 +70,7 @@ func (h getMessagesOfTypeHandler) HandleSource(ctx context.Context, req *muxrpc. } - workSet, err := h.feeds.LoadInternalBitmap(feed.StoredAddr()) + workSet, err := h.feeds.LoadInternalBitmap(storedrefs.Feed(feed)) if err != nil { // TODO actual assert not found error. // errors.Errorf("failed to load feed %s bitmap: %s", feed.ShortRef(), err.Error()) @@ -78,7 +79,7 @@ func (h getMessagesOfTypeHandler) HandleSource(ctx context.Context, req *muxrpc. } - tipeSeqs, err := h.bytype.LoadInternalBitmap(librarian.Addr(tipe)) + tipeSeqs, err := h.bytype.LoadInternalBitmap(librarian.Addr("string:" + tipe)) if err != nil { // return errors.Errorf("failed to load msg type %s bitmap: %s", tipe, err.Error()) snk.Close() @@ -108,7 +109,7 @@ func (h getMessagesOfTypeHandler) HandleSource(ctx context.Context, req *muxrpc. kv.Value = *msg.ValueContent() b, err := json.Marshal(kv) if err != nil { - return errors.Errorf("failed to encode json: %w", err) + return fmt.Errorf("failed to encode json: %w", err) } err = snk.Pour(ctx, json.RawMessage(b)) if err != nil { diff --git a/plugins/rawread/logt.go b/plugins/rawread/logt.go index cdf250bf..101e1d12 100644 --- a/plugins/rawread/logt.go +++ b/plugins/rawread/logt.go @@ -5,118 +5,240 @@ package rawread import ( "context" "fmt" + "math" + "os" + bmap "github.com/RoaringBitmap/roaring" "github.com/davecgh/go-spew/spew" + "github.com/go-kit/kit/log" "github.com/pkg/errors" "go.cryptoscope.co/librarian" "go.cryptoscope.co/luigi" "go.cryptoscope.co/margaret" - "go.cryptoscope.co/margaret/multilog" + "go.cryptoscope.co/margaret/multilog/roaring" "go.cryptoscope.co/muxrpc" "go.cryptoscope.co/ssb" "go.cryptoscope.co/ssb/internal/mutil" + "go.cryptoscope.co/ssb/internal/muxmux" "go.cryptoscope.co/ssb/internal/transform" "go.cryptoscope.co/ssb/message" + "go.cryptoscope.co/ssb/private" + "go.cryptoscope.co/ssb/repo" ) -// messagesByType --help -// (logt) Retrieve messages with a given type, ordered by receive-time. -// logt --type {type} [--gt index] [--gte index] [--lt index] [--lte index] [--reverse] [--keys] [--values] [--limit n] -type logTplug struct { +type Plugin struct { + rxlog margaret.Log + types *roaring.MultiLog + + priv *roaring.MultiLog + isSelf ssb.Authorizer + unboxer *private.Manager + + res *repo.SequenceResolver + h muxrpc.Handler } -func NewByType(rootLog margaret.Log, typeLogs multilog.MultiLog) ssb.Plugin { - plug := &logTplug{} - plug.h = logThandler{ - root: rootLog, - types: typeLogs, - } - return plug -} +func NewByTypePlugin( + log log.Logger, + rootLog margaret.Log, + ml *roaring.MultiLog, + pl *roaring.MultiLog, + pm *private.Manager, + res *repo.SequenceResolver, + isSelf ssb.Authorizer, +) ssb.Plugin { + plug := &Plugin{ + rxlog: rootLog, + types: ml, -func (lt logTplug) Name() string { return "messagesByType" } + priv: pl, -func (logTplug) Method() muxrpc.Method { - return muxrpc.Method{"messagesByType"} -} -func (lt logTplug) Handler() muxrpc.Handler { - return lt.h -} + unboxer: pm, -type logThandler struct { - root margaret.Log - types multilog.MultiLog -} + res: res, -func (g logThandler) HandleConnect(ctx context.Context, e muxrpc.Endpoint) { + isSelf: isSelf, + } + + h := muxmux.New(log) + h.RegisterSource(muxrpc.Method{"messagesByType"}, plug) + + plug.h = &h + return plug } -func (g logThandler) HandleCall(ctx context.Context, req *muxrpc.Request, edp muxrpc.Endpoint) { - fmt.Println("byTypeCall:", req.Method.String()) - spew.Dump(req.Args()) - if len(req.Args()) < 1 { - req.CloseWithError(errors.Errorf("invalid arguments")) - return +func (lt Plugin) Name() string { return "msgTypes" } +func (Plugin) Method() muxrpc.Method { return muxrpc.Method{"messagesByType"} } +func (lt Plugin) Handler() muxrpc.Handler { return lt.h } + +func (g Plugin) HandleSource(ctx context.Context, req *muxrpc.Request, snk luigi.Sink, edp muxrpc.Endpoint) error { + args := req.Args() + if len(args) < 1 { + return errors.Errorf("invalid arguments") } - var ( - tipe librarian.Addr - qry message.CreateHistArgs - ) - switch v := req.Args()[0].(type) { - case string: - tipe = librarian.Addr(v) + var qry message.MessagesByTypeArgs + + switch v := args[0].(type) { + case map[string]interface{}: q, err := message.NewCreateHistArgsFromMap(v) if err != nil { - req.CloseWithError(errors.Wrap(err, "bad request")) - return + return errors.Wrap(err, "bad request") } - qry = *q - default: - req.CloseWithError(errors.Errorf("invalid argument type %T", req.Args()[0])) - return - } + qry.CommonArgs = q.CommonArgs + qry.StreamArgs = q.StreamArgs - if len(req.Args()) == 2 { - mv, ok := req.Args()[1].(map[string]interface{}) + var ok bool + qry.Type, ok = v["type"].(string) if !ok { - req.CloseWithError(errors.Errorf("bad request")) - return + return errors.Errorf("bad request - missing root") } - q, err := message.NewCreateHistArgsFromMap(mv) - if err != nil { - req.CloseWithError(errors.Wrap(err, "bad request")) - return - } - qry = *q - } else { + + case string: qry.Limit = -1 - // TODO: msg should be wrapped in obj with key and rxt + qry.Type = v qry.Keys = true - // only return message keys - qry.Values = true + default: + return errors.Errorf("invalid argument type %T", args[0]) } - tipeLog, err := g.types.Get(tipe) + + remote, err := ssb.GetFeedRefFromAddr(edp.Remote()) if err != nil { - req.CloseWithError(errors.Wrap(err, "logT: no multilog for tipe?")) - return + return errors.Wrap(err, "failed to establish remote") + } + + isSelf := g.isSelf.Authorize(remote) + if qry.Private && isSelf != nil { + return fmt.Errorf("not authroized") + } + + // create toJSON sink + snk = transform.NewKeyValueWrapper(req.Stream, qry.Keys) + + // wrap it into a counter for debugging + var cnt int + snk = newSinkCounter(&cnt, snk) + + idxAddr := librarian.Addr("string:" + qry.Type) + if qry.Live { + if qry.Private { + return fmt.Errorf("TODO: fix live && private") + } + typed, err := g.types.Get(idxAddr) + if err != nil { + return errors.Wrap(err, "failed to load typed log") + } + + src, err := mutil.Indirect(g.rxlog, typed).Query(margaret.Limit(int(qry.Limit)), margaret.Live(qry.Live)) + if err != nil { + return errors.Wrap(err, "logT: failed to qry tipe") + } + + // if qry.Private { TODO + // src = g.unboxedSrc(src) + // } + + err = luigi.Pump(ctx, snk, src) + if err != nil { + return errors.Wrap(err, "logT: failed to pump msgs") + } + + return snk.Close() } - resolved := mutil.Indirect(g.root, tipeLog) - src, err := resolved.Query(margaret.Limit(int(qry.Limit)), margaret.Live(qry.Live)) + /* TODO: i'm skipping a fairly big refactor here to find out what works first. + ideallly the live and not-live code would just be the same, somehow shoving it into Query(...). + Same goes for timestamp sorting and private. + Private is at least orthogonal, whereas sorting and live don't go well together. + */ + + // not live + typed, err := g.types.LoadInternalBitmap(idxAddr) if err != nil { - req.CloseWithError(errors.Wrap(err, "logT: failed to qry tipe")) - return + return errors.Wrap(err, "failed to load typed log") + } + + if qry.Private { + snk = g.unboxer.WrappedUnboxingSink(snk) + } else { + // filter all boxed messages from the stream + box1, err := g.priv.LoadInternalBitmap(librarian.Addr("meta:box1")) + if err != nil { + // TODO: compare not found + // return errors.Wrap(err, "failed to load bmap for box1") + box1 = bmap.New() + } + + box2, err := g.priv.LoadInternalBitmap(librarian.Addr("meta:box2")) + if err != nil { + // TODO: compare not found + // return errors.Wrap(err, "failed to load bmap for box2") + box2 = bmap.New() + } + + box1.Or(box2) // all the boxed messages + + // remove all the boxed ones from the type we are looking up + typed.AndNot(box1) + } + + // TODO: set _all_ correctly if gt=0 && lt=0 + if qry.Lt == 0 { + qry.Lt = math.MaxInt64 } - err = luigi.Pump(ctx, transform.NewKeyValueWrapper(req.Stream, qry.Keys), src) + spew.Dump(qry) + + var filter = func(ts int64) bool { + isGreater := ts > qry.Gt + isSmaller := ts < qry.Lt + return isGreater && isSmaller + } + + sort, err := g.res.SortAndFilterBitmap(typed, repo.SortByClaimed, filter, qry.Reverse) if err != nil { - req.CloseWithError(errors.Wrap(err, "logT: failed to pump msgs")) - return + return errors.Wrap(err, "failed to filter bitmap") + } + + for _, res := range sort { + v, err := g.rxlog.Get(margaret.BaseSeq(res.Seq)) + if err != nil { + fmt.Fprintln(os.Stderr, "messagesByType failed to get seq:", res.Seq, " with:", err) + continue + } + + // skip nulled + if verr, ok := v.(error); ok && margaret.IsErrNulled(verr) { + continue + } + + if err := snk.Pour(ctx, v); err != nil { + fmt.Fprintln(os.Stderr, "messagesByType failed send:", res.Seq, " with:", err) + break + } + + if qry.Limit >= 0 { + qry.Limit-- + if qry.Limit == 0 { + break + } + } } + fmt.Println("streamed", cnt, " for type:", qry.Type) + return snk.Close() +} - req.Stream.Close() +func newSinkCounter(counter *int, sink luigi.Sink) luigi.FuncSink { + return func(ctx context.Context, v interface{}, err error) error { + if err != nil { + fmt.Println("weird", err) + return err + } + + *counter++ + return sink.Pour(ctx, v) + } } diff --git a/plugins/rawread/sorted.go b/plugins/rawread/sorted.go new file mode 100644 index 00000000..71c5b30d --- /dev/null +++ b/plugins/rawread/sorted.go @@ -0,0 +1,117 @@ +package rawread + +import ( + "context" + "encoding/json" + "fmt" + "os" + "time" + + "github.com/go-kit/kit/log" + "github.com/pkg/errors" + "go.cryptoscope.co/luigi" + "go.cryptoscope.co/margaret" + "go.cryptoscope.co/muxrpc" + + "go.cryptoscope.co/ssb" + "go.cryptoscope.co/ssb/internal/muxmux" + "go.cryptoscope.co/ssb/internal/transform" + "go.cryptoscope.co/ssb/message" + "go.cryptoscope.co/ssb/repo" +) + +// ~> sbot createFeedStream --help +// (log) Fetch messages ordered by the time received. +// log [--live] [--gt ts] [--lt ts] [--reverse] [--keys] [--limit n] +type sortedPlug struct { + root margaret.Log + res *repo.SequenceResolver + + h muxrpc.Handler +} + +func NewSortedStream(log log.Logger, rootLog margaret.Log, res *repo.SequenceResolver) ssb.Plugin { + plug := &sortedPlug{ + root: rootLog, + res: res, + } + + h := muxmux.New(log) + + h.RegisterSource(muxrpc.Method{"createFeedStream"}, plug) + + plug.h = &h + + return plug +} + +func (lt sortedPlug) Name() string { return "createFeedStream" } + +func (sortedPlug) Method() muxrpc.Method { + return muxrpc.Method{"createFeedStream"} +} +func (lt sortedPlug) Handler() muxrpc.Handler { + return lt.h +} + +func (g sortedPlug) HandleSource(ctx context.Context, req *muxrpc.Request, snk luigi.Sink, edp muxrpc.Endpoint) error { + start := time.Now() + var qry message.CreateLogArgs + if len(req.Args()) == 1 { + var args []message.CreateLogArgs + err := json.Unmarshal(req.RawArgs, &args) + if err != nil { + return errors.Wrap(err, "bad request data") + } + if len(args) == 1 { + qry = args[0] + } + } else { + // Defaults for no arguments + qry.Keys = true + qry.Limit = -1 + } + + // empty query doesn't make much sense... + if qry.Limit == 0 { + qry.Limit = -1 + } + + // TODO: only return message keys + // qry.Values = true + + sortedSeqs, err := g.res.SortAndFilterAll(repo.SortByClaimed, func(ts int64) bool { + isGreater := ts > qry.Gt + isSmaller := ts < qry.Lt + return isGreater && isSmaller + }, true) + + if err != nil { + return err + } + fmt.Fprintln(os.Stderr, "createFeedStream sorted seqs:", len(sortedSeqs), "after:", time.Since(start)) + + toJSON := transform.NewKeyValueWrapper(req.Stream, qry.Keys) + + for _, res := range sortedSeqs { + v, err := g.root.Get(margaret.BaseSeq(res.Seq)) + if err != nil { + fmt.Fprintln(os.Stderr, "createFeedStream failed to get seq:", res.Seq, " with:", err, "after:", time.Since(start)) + continue + } + + if err := toJSON.Pour(ctx, v); err != nil { + fmt.Fprintln(os.Stderr, "createFeedStream failed send:", res.Seq, " with:", err, "after:", time.Since(start)) + break + } + + if qry.Limit >= 0 { + qry.Limit-- + if qry.Limit == 0 { + break + } + } + } + fmt.Fprintln(os.Stderr, "createFeedStream closed after:", time.Since(start)) + return snk.Close() +} diff --git a/plugins/tangles/plugin.go b/plugins/tangles/plugin.go new file mode 100644 index 00000000..24d5c98a --- /dev/null +++ b/plugins/tangles/plugin.go @@ -0,0 +1,220 @@ +// SPDX-License-Identifier: MIT + +package tangles + +import ( + "context" + "encoding/json" + "fmt" + "os" + + bmap "github.com/RoaringBitmap/roaring" + "github.com/davecgh/go-spew/spew" + "github.com/go-kit/kit/log" + "github.com/pkg/errors" + "go.cryptoscope.co/librarian" + "go.cryptoscope.co/luigi" + "go.cryptoscope.co/margaret" + "go.cryptoscope.co/margaret/multilog/roaring" + "go.cryptoscope.co/muxrpc" + + "go.cryptoscope.co/ssb" + "go.cryptoscope.co/ssb/internal/mutil" + "go.cryptoscope.co/ssb/internal/muxmux" + "go.cryptoscope.co/ssb/internal/transform" + "go.cryptoscope.co/ssb/message" + "go.cryptoscope.co/ssb/private" + refs "go.mindeco.de/ssb-refs" +) + +type Plugin struct { + h muxrpc.Handler +} + +func NewPlugin(rxlog margaret.Log, tangles, private *roaring.MultiLog, unboxer *private.Manager, isSelf ssb.Authorizer) *Plugin { + mux := muxmux.New(log.NewNopLogger()) + + mux.RegisterSource(muxrpc.Method{"tangles", "replies"}, tangleHandler{ + rxlog: rxlog, + tangles: tangles, + + // private utils + private: private, + unboxer: unboxer, + isSelf: isSelf, + }) + + /* TODO: heads + mux.RegisterAsync(muxrpc.Method{"tangles", "heads"}, headsHandler{ + rxlog: rxlog, + tangles: threads, + + // private utils + private: private, + unboxer: unboxer, + isSelf: isSelf, + }) + */ + + return &Plugin{ + h: &mux, + } +} + +func (lt Plugin) Name() string { return "tangles" } +func (Plugin) Method() muxrpc.Method { return muxrpc.Method{"tangles"} } +func (lt Plugin) Handler() muxrpc.Handler { return lt.h } + +type tangleHandler struct { + rxlog margaret.Log + tangles *roaring.MultiLog + private *roaring.MultiLog + + isSelf ssb.Authorizer + unboxer *private.Manager +} + +func (g tangleHandler) HandleSource(ctx context.Context, req *muxrpc.Request, snk luigi.Sink, edp muxrpc.Endpoint) error { + if len(req.Args()) < 1 { + return errors.Errorf("invalid arguments") + } + + var qryarr []message.TanglesArgs + var qry message.TanglesArgs + + err := json.Unmarshal(req.RawArgs, &qryarr) + if err != nil { + if req.RawArgs[0] != '"' { + return errors.Wrap(err, "bad request - invalid root") + } + + var ref refs.MessageRef + err := json.Unmarshal(req.RawArgs, &ref) + if err != nil { + return errors.Wrap(err, "bad request - invalid root (string?)") + } + qry.Root = &ref + qry.Limit = -1 + qry.Keys = true + } else { + if n := len(qryarr); n != 1 { + return fmt.Errorf("expected 1 argument but got %d", n) + } + qry = qryarr[0] + // defaults?! + } + + if qry.Limit == 0 { + qry.Limit = -1 + } + + remote, err := ssb.GetFeedRefFromAddr(edp.Remote()) + if err != nil { + return errors.Wrap(err, "failed to determain remote") + } + + isSelf := g.isSelf.Authorize(remote) + if qry.Private && isSelf != nil { + return fmt.Errorf("not authroized") + } + + fmt.Println("query for:", qry.Root.Ref()) + spew.Dump(qry) + + // create toJSON sink + snk = transform.NewKeyValueWrapper(req.Stream, qry.Keys) + + // lookup address depending if we have a name for the tangle or not + addr := librarian.Addr(append([]byte("v1:"), qry.Root.Hash...)) + if qry.Name != "" { + addr = librarian.Addr("v2:"+qry.Name+":") + librarian.Addr(qry.Root.Hash) + } + + // TODO: needs same kind of refactor that messagesByType needs + + if qry.Live { + if qry.Private { + return fmt.Errorf("TODO: fix live && private") + } + threadLog, err := g.tangles.Get(addr) + if err != nil { + return errors.Wrap(err, "failed to load thread") + } + + src, err := mutil.Indirect(g.rxlog, threadLog).Query(margaret.Limit(int(qry.Limit)), margaret.Live(qry.Live), margaret.Reverse(qry.Reverse)) + if err != nil { + return errors.Wrap(err, "tangle: failed to create query") + } + + err = luigi.Pump(ctx, snk, src) + if err != nil { + return errors.Wrap(err, "tangle: failed to pump msgs") + } + + return snk.Close() + } + + // not live + threadBmap, err := g.tangles.LoadInternalBitmap(addr) + if err != nil { + // TODO: check err == persist: not found + return snk.Close() + return errors.Wrap(err, "failed to load thread log") + } + + if qry.Private { + snk = g.unboxer.WrappedUnboxingSink(snk) + } else { + // filter all boxed messages from the stream + box1, err := g.private.LoadInternalBitmap(librarian.Addr("meta:box1")) + if err != nil { + // TODO: compare not found + // return errors.Wrap(err, "failed to load bmap for box1") + box1 = bmap.New() + } + + box2, err := g.private.LoadInternalBitmap(librarian.Addr("meta:box2")) + if err != nil { + // TODO: compare not found + // return errors.Wrap(err, "failed to load bmap for box2") + box2 = bmap.New() + } + + box1.Or(box2) // all the boxed messages + + // remove all the boxed ones from the type we are looking up + threadBmap.AndNot(box1) + } + + // TODO: sort by previous + + it := threadBmap.Iterator() + + for it.HasNext() { + seq := margaret.BaseSeq(it.Next()) + v, err := g.rxlog.Get(seq) + if err != nil { + fmt.Fprintln(os.Stderr, "tangles failed to get seq:", seq, " with:", err) + continue + } + + // skip nulled + if verr, ok := v.(error); ok && margaret.IsErrNulled(verr) { + continue + } + + if err := snk.Pour(ctx, v); err != nil { + fmt.Fprintln(os.Stderr, "tangles failed send:", seq, " with:", err) + break + } + + if qry.Limit >= 0 { + qry.Limit-- + if qry.Limit == 0 { + break + } + } + } + + return snk.Close() +} diff --git a/plugins2/tangles/tangles_test.go b/plugins/tangles/tangles_test.go similarity index 96% rename from plugins2/tangles/tangles_test.go rename to plugins/tangles/tangles_test.go index 944fecdc..3c0ed617 100644 --- a/plugins2/tangles/tangles_test.go +++ b/plugins/tangles/tangles_test.go @@ -2,17 +2,20 @@ // +build ignore +// TODO: mutli-author refactor + package tangles import ( - "context" - "io/ioutil" + "crypto/rand" + "os" + "path/filepath" "testing" + "github.com/go-kit/kit/log" "github.com/stretchr/testify/require" "go.cryptoscope.co/ssb" - "go.cryptoscope.co/ssb/internal/ctxutils" - "go.cryptoscope.co/ssb/message" + "go.cryptoscope.co/ssb/indexes" "go.cryptoscope.co/ssb/repo" ) diff --git a/plugins2/bytype/index.go b/plugins2/bytype/index.go deleted file mode 100644 index 02ef2cb6..00000000 --- a/plugins2/bytype/index.go +++ /dev/null @@ -1,64 +0,0 @@ -// SPDX-License-Identifier: MIT - -package bytype - -import ( - "context" - "encoding/json" - - "github.com/pkg/errors" - "go.cryptoscope.co/librarian" - "go.cryptoscope.co/margaret" - "go.cryptoscope.co/margaret/multilog" - refs "go.mindeco.de/ssb-refs" -) - -func (plug *Plugin) UseMultiLog(ml multilog.MultiLog) { - plug.h.types = ml -} - -/* legacy -func (plug *Plugin) MakeMultiLog(r repo.Interface) (multilog.MultiLog, librarian.SinkIndex, error) { - mlog, serve, err := repo.OpenFileSystemMultiLog(r, plug.Name(), IndexUpdate) - plug.h.types = mlog - return mlog, serve, err -} -*/ - -func IndexUpdate(ctx context.Context, seq margaret.Seq, msgv interface{}, mlog multilog.MultiLog) error { - if nulled, ok := msgv.(error); ok { - if margaret.IsErrNulled(nulled) { - return nil - } - return nulled - } - msg, ok := msgv.(refs.Message) - if !ok { - err := errors.Errorf("error casting message. got type %T", msgv) - return err - } - - var typedMsg struct { - Content struct { - Type string - } - } - - content := msg.ValueContentJSON() - err := json.Unmarshal(content, &typedMsg) - typeStr := typedMsg.Content.Type - // TODO: maybe check error with more detail - i.e. only drop type errors - if err != nil || typeStr == "" { - // TODO: special case boxed messages - // fmt.Println(string(content)) - return nil - } - - typedLog, err := mlog.Get(librarian.Addr(typeStr)) - if err != nil { - return errors.Wrap(err, "error opening sublog") - } - - _, err = typedLog.Append(seq) - return errors.Wrapf(err, "error appending message of type %q", typeStr) -} diff --git a/plugins2/bytype/msgtypes_test.go b/plugins2/bytype/msgtypes_test.go deleted file mode 100644 index 42af1b63..00000000 --- a/plugins2/bytype/msgtypes_test.go +++ /dev/null @@ -1,156 +0,0 @@ -// SPDX-License-Identifier: MIT - -package bytype - -import ( - "context" - "io/ioutil" - "testing" - - "github.com/stretchr/testify/require" - - "go.cryptoscope.co/ssb" - "go.cryptoscope.co/ssb/internal/asynctesting" - "go.cryptoscope.co/ssb/internal/ctxutils" - "go.cryptoscope.co/ssb/message" - "go.cryptoscope.co/ssb/multilogs" - "go.cryptoscope.co/ssb/repo" -) - -func TestMessageTypes(t *testing.T) { - // > test boilerplate - // TODO: abstract serving and error channel handling - // Meta TODO: close handling and status of indexing - r := require.New(t) - - tRepoPath, err := ioutil.TempDir("", t.Name()) - r.NoError(err) - - ctx, cancel := ctxutils.WithError(context.Background(), ssb.ErrShuttingDown) - - tRepo := repo.New(tRepoPath) - tRootLog, err := repo.OpenLog(tRepo) - r.NoError(err) - - uf, serveUF, err := multilogs.OpenUserFeeds(tRepo) - r.NoError(err) - ufErrc := asynctesting.ServeLog(ctx, "user feeds", tRootLog, serveUF, true) - - mt, serveMT, err := repo.OpenMultiLog(tRepo, "byType", IndexUpdate) - r.NoError(err) - mtErrc := asynctesting.ServeLog(ctx, "message types", tRootLog, serveMT, true) - - alice, err := ssb.NewKeyPair(nil) - r.NoError(err) - alicePublish, err := message.OpenPublishLog(tRootLog, mt, alice) - r.NoError(err) - - bob, err := ssb.NewKeyPair(nil) - r.NoError(err) - bobPublish, err := message.OpenPublishLog(tRootLog, mt, bob) - r.NoError(err) - - claire, err := ssb.NewKeyPair(nil) - r.NoError(err) - clairePublish, err := message.OpenPublishLog(tRootLog, mt, claire) - r.NoError(err) - - // > create contacts - var tmsgs = []interface{}{ - map[string]interface{}{ - "type": "contact", - "contact": alice.Id.Ref(), - "following": true, - "test": "alice1", - }, - map[string]interface{}{ - "type": "post", - "text": "something", - "test": "alice2", - }, - "1923u1310310.nobox", - map[string]interface{}{ - "type": "about", - "about": bob.Id.Ref(), - "name": "bob", - "test": "alice3", - }, - } - for i, msg := range tmsgs { - newSeq, err := alicePublish.Append(msg) - r.NoError(err, "failed to publish test message %d", i) - r.NotNil(newSeq) - } - - asynctesting.CheckTypes(t, "contact", []string{"alice1"}, tRootLog, mt) - asynctesting.CheckTypes(t, "post", []string{"alice2"}, tRootLog, mt) - asynctesting.CheckTypes(t, "about", []string{"alice3"}, tRootLog, mt) - - var bobsMsgs = []interface{}{ - map[string]interface{}{ - "type": "contact", - "contact": bob.Id.Ref(), - "following": true, - "test": "bob1", - }, - "1923u1310310.nobox", - map[string]interface{}{ - "type": "about", - "about": bob.Id.Ref(), - "name": "bob", - "test": "bob2", - }, - map[string]interface{}{ - "type": "post", - "text": "hello, world", - "test": "bob3", - }, - } - for i, msg := range bobsMsgs { - newSeq, err := bobPublish.Append(msg) - r.NoError(err, "failed to publish test message %d", i) - r.NotNil(newSeq) - } - - asynctesting.CheckTypes(t, "contact", []string{"alice1", "bob1"}, tRootLog, mt) - asynctesting.CheckTypes(t, "about", []string{"alice3", "bob2"}, tRootLog, mt) - asynctesting.CheckTypes(t, "post", []string{"alice2", "bob3"}, tRootLog, mt) - - var clairesMsgs = []interface{}{ - "1923u1310310.nobox", - map[string]interface{}{ - "type": "post", - "text": "hello, world", - "test": "claire1", - }, - map[string]interface{}{ - "type": "contact", - "contact": claire.Id.Ref(), - "following": true, - "test": "claire2", - }, - map[string]interface{}{ - "type": "about", - "about": claire.Id.Ref(), - "name": "claire", - "test": "claire3", - }, - } - for i, msg := range clairesMsgs { - newSeq, err := clairePublish.Append(msg) - r.NoError(err, "failed to publish test message %d", i) - r.NotNil(newSeq) - } - - asynctesting.CheckTypes(t, "contact", []string{"alice1", "bob1", "claire2"}, tRootLog, mt) - asynctesting.CheckTypes(t, "about", []string{"alice3", "bob2", "claire3"}, tRootLog, mt) - asynctesting.CheckTypes(t, "post", []string{"alice2", "bob3", "claire1"}, tRootLog, mt) - - mt.Close() - uf.Close() - cancel() - - for err := range asynctesting.MergedErrors(mtErrc, ufErrc) { - r.NoError(err, "from chan") - } -} diff --git a/plugins2/bytype/plugin.go b/plugins2/bytype/plugin.go deleted file mode 100644 index f0026b34..00000000 --- a/plugins2/bytype/plugin.go +++ /dev/null @@ -1,121 +0,0 @@ -// SPDX-License-Identifier: MIT - -package bytype - -import ( - "context" - "fmt" - - "github.com/davecgh/go-spew/spew" - "github.com/pkg/errors" - "go.cryptoscope.co/librarian" - "go.cryptoscope.co/luigi" - "go.cryptoscope.co/margaret" - "go.cryptoscope.co/margaret/multilog" - "go.cryptoscope.co/muxrpc" - "go.cryptoscope.co/ssb/internal/mutil" - "go.cryptoscope.co/ssb/internal/transform" - "go.cryptoscope.co/ssb/message" - "go.cryptoscope.co/ssb/plugins2" -) - -type Plugin struct { - h handler -} - -var ( - _ plugins2.NeedsRootLog = (*Plugin)(nil) -) - -// TODO: return plugin spec similar to margaret qry spec? - -func (tp *Plugin) WantRootLog(rl margaret.Log) error { - tp.h.root = rl - return nil -} - -func (lt Plugin) Name() string { return "msgTypes" } -func (Plugin) Method() muxrpc.Method { return muxrpc.Method{"messagesByType"} } -func (lt Plugin) Handler() muxrpc.Handler { return lt.h } - -type handler struct { - root margaret.Log - types multilog.MultiLog -} - -func (g handler) HandleConnect(ctx context.Context, e muxrpc.Endpoint) {} - -func (g handler) HandleCall(ctx context.Context, req *muxrpc.Request, edp muxrpc.Endpoint) { - args := req.Args() - if len(args) < 1 { - req.CloseWithError(errors.Errorf("invalid arguments")) - return - } - var qry message.MessagesByTypeArgs - - switch v := args[0].(type) { - - case map[string]interface{}: - q, err := message.NewCreateHistArgsFromMap(v) - if err != nil { - req.CloseWithError(errors.Wrap(err, "bad request")) - return - } - qry.CommonArgs = q.CommonArgs - qry.StreamArgs = q.StreamArgs - - var ok bool - qry.Type, ok = v["type"].(string) - if !ok { - req.CloseWithError(errors.Errorf("bad request - missing root")) - return - } - - case string: - qry.Limit = -1 - qry.Type = v - qry.Keys = true - - default: - req.CloseWithError(errors.Errorf("invalid argument type %T", args[0])) - return - } - spew.Dump(qry) - typed, err := g.types.Get(librarian.Addr(qry.Type)) - if err != nil { - req.CloseWithError(errors.Wrap(err, "failed to load typed log")) - return - } - - src, err := mutil.Indirect(g.root, typed).Query(margaret.Limit(int(qry.Limit)), margaret.Live(qry.Live), margaret.Reverse(qry.Reverse)) - if err != nil { - req.CloseWithError(errors.Wrap(err, "logT: failed to qry tipe")) - return - } - - snk := transform.NewKeyValueWrapper(req.Stream, qry.Keys) - - var cnt int - - err = luigi.Pump(ctx, newSinkCounter(&cnt, snk), src) - if err != nil { - req.CloseWithError(errors.Wrap(err, "logT: failed to pump msgs")) - return - } - - fmt.Println("bytype", qry.Type, cnt) - - req.Stream.Close() -} - -func newSinkCounter(counter *int, sink luigi.Sink) luigi.FuncSink { - return func(ctx context.Context, v interface{}, err error) error { - if err != nil { - fmt.Println("weird", err) - return err - } - - *counter++ - return sink.Pour(ctx, v) - } -} diff --git a/plugins2/names/about.go b/plugins2/names/about.go index f4f5af39..1267df9a 100644 --- a/plugins2/names/about.go +++ b/plugins2/names/about.go @@ -16,7 +16,6 @@ import ( libbadger "go.cryptoscope.co/librarian/badger" "go.cryptoscope.co/margaret" - "go.cryptoscope.co/ssb" "go.cryptoscope.co/ssb/client" "go.cryptoscope.co/ssb/repo" refs "go.mindeco.de/ssb-refs" @@ -209,8 +208,6 @@ func (plug *Plugin) MakeSimpleIndex(r repo.Interface) (librarian.Index, libraria return nil, nil, errors.Wrap(err, "error getting about index") } - // TODO: hook serve to close db - plug.about = aboutStore{db} return idx, update, err @@ -228,14 +225,13 @@ func updateAboutMessage(ctx context.Context, seq margaret.Seq, msgv interface{}, var aboutMSG refs.About err := json.Unmarshal(msg.ContentBytes(), &aboutMSG) if err != nil { - // fmt.Println("msg", "skipped contact message", "reason", err) + // nothing to do with this message + // TODO: git repos and gathering use about messages for their names return nil - // debugging - if ssb.IsMessageUnusable(err) { - return nil - } } + // fmt.Println("msg", "decoded", "seq", seq.Seq()) + // about:from:field addr := aboutMSG.About.Ref() addr += ":" diff --git a/plugins2/names/about_test.go b/plugins2/names/about_test.go index b86e06c2..e7cd3219 100644 --- a/plugins2/names/about_test.go +++ b/plugins2/names/about_test.go @@ -1,5 +1,9 @@ // SPDX-License-Identifier: MIT +// TODO: move these into the sbot package + +// +build ignore + package names_test import ( @@ -61,7 +65,7 @@ func TestAboutNames(t *testing.T) { _, err = ali.PublishLog.Publish(newName) r.NoError(err) - src, err := ali.RootLog.Query() + src, err := ali.ReceiveLog.Query() r.NoError(err) var i = 0 for { diff --git a/plugins2/names/names_test.go b/plugins2/names/names_test.go index de8e3b7d..90d07cd3 100644 --- a/plugins2/names/names_test.go +++ b/plugins2/names/names_test.go @@ -1,5 +1,7 @@ // SPDX-License-Identifier: MIT +// +build ignore + package names import ( @@ -89,7 +91,7 @@ func XTestNames(t *testing.T) { r.EqualValues(seq, v.(margaret.Seq).Seq()) } - checkLogSeq(mainbot.RootLog, len(intros)-1) // got all the messages + checkLogSeq(mainbot.ReceiveLog, len(intros)-1) // got all the messages c, err := client.NewUnix(filepath.Join(tRepoPath, "socket")) r.NoError(err) diff --git a/plugins2/tangles/plugin.go b/plugins2/tangles/plugin.go deleted file mode 100644 index 5a6262d4..00000000 --- a/plugins2/tangles/plugin.go +++ /dev/null @@ -1,123 +0,0 @@ -// SPDX-License-Identifier: MIT - -package tangles - -import ( - "context" - - "github.com/pkg/errors" - "go.cryptoscope.co/librarian" - "go.cryptoscope.co/luigi" - "go.cryptoscope.co/margaret" - "go.cryptoscope.co/margaret/multilog" - "go.cryptoscope.co/muxrpc" - "go.cryptoscope.co/ssb/internal/mutil" - "go.cryptoscope.co/ssb/internal/transform" - "go.cryptoscope.co/ssb/message" - "go.cryptoscope.co/ssb/plugins2" - refs "go.mindeco.de/ssb-refs" -) - -type Plugin struct { - h tangleHandler -} - -var ( - _ plugins2.NeedsRootLog = (*Plugin)(nil) -) - -// TODO: return plugin spec similar to margaret qry spec? -func (tp *Plugin) UseMultiLog(ml multilog.MultiLog) { - tp.h.tangle = ml -} - -func (tp *Plugin) WantRootLog(rl margaret.Log) error { - tp.h.root = rl - return nil -} - -func (lt Plugin) Name() string { return "tangles" } -func (Plugin) Method() muxrpc.Method { return muxrpc.Method{"tangles"} } -func (lt Plugin) Handler() muxrpc.Handler { return lt.h } - -type tangleHandler struct { - root margaret.Log - tangle multilog.MultiLog -} - -func (g tangleHandler) HandleConnect(ctx context.Context, e muxrpc.Endpoint) {} - -func (g tangleHandler) HandleCall(ctx context.Context, req *muxrpc.Request, edp muxrpc.Endpoint) { - if len(req.Args()) < 1 { - req.CloseWithError(errors.Errorf("invalid arguments")) - return - } - var qry struct { - message.CreateHistArgs - Root *refs.MessageRef - } - - switch v := req.Args()[0].(type) { - case string: - var err error - qry.Root, err = refs.ParseMessageRef(v) - if err != nil { - req.CloseWithError(errors.Wrap(err, "bad request - invalid root")) - return - } - qry.Limit = -1 - qry.Keys = true - case map[string]interface{}: - q, err := message.NewCreateHistArgsFromMap(v) - if err != nil { - req.CloseWithError(errors.Wrap(err, "bad request")) - return - } - qry.CreateHistArgs = *q - qry.StreamArgs = q.StreamArgs - - root, ok := v["root"].(string) - if !ok { - req.CloseWithError(errors.Errorf("bad request - missing root")) - return - } - - qry.Root, err = refs.ParseMessageRef(root) - if err != nil { - req.CloseWithError(errors.Wrap(err, "bad request - invalid root")) - return - } - default: - req.CloseWithError(errors.Errorf("invalid argument type %T", req.Args()[0])) - return - } - - if qry.Live { - qry.Limit = -1 - } - - if qry.Limit == 0 { - qry.Limit = -1 - } - - addr := librarian.Addr(append([]byte("v1:"), qry.Root.Hash...)) - threadLog, err := g.tangle.Get(addr) - if err != nil { - req.CloseWithError(errors.Wrap(err, "failed to load thread")) - return - } - - src, err := mutil.Indirect(g.root, threadLog).Query(margaret.Limit(int(qry.Limit)), margaret.Live(qry.Live), margaret.Reverse(qry.Reverse)) - if err != nil { - req.CloseWithError(errors.Wrap(err, "logT: failed to qry tipe")) - return - } - - err = luigi.Pump(ctx, transform.NewKeyValueWrapper(req.Stream, qry.Keys), src) - if err != nil { - req.CloseWithError(errors.Wrap(err, "logT: failed to pump msgs")) - return - } - - req.Stream.Close() -} diff --git a/plugins2/tangles/tangles.go b/plugins2/tangles/tangles.go deleted file mode 100644 index cb8a2b77..00000000 --- a/plugins2/tangles/tangles.go +++ /dev/null @@ -1,50 +0,0 @@ -// SPDX-License-Identifier: MIT - -package tangles - -import ( - "context" - "encoding/json" - "fmt" - - "github.com/pkg/errors" - "go.cryptoscope.co/librarian" - "go.cryptoscope.co/margaret" - "go.cryptoscope.co/margaret/multilog" - refs "go.mindeco.de/ssb-refs" -) - -var legacy = func(ctx context.Context, seq margaret.Seq, msgv interface{}, mlog multilog.MultiLog) error { - if nulled, ok := msgv.(error); ok { - if margaret.IsErrNulled(nulled) { - return nil - } - return nulled - } - - msg, ok := msgv.(refs.Message) - if !ok { - err := errors.Errorf("error casting message. got type %T", msgv) - fmt.Println("tangleIDX failed:", err) - return err - } - - var value struct { - Root *refs.MessageRef - } - - err := json.Unmarshal(msg.ContentBytes(), &value) - // TODO: maybe check error with more detail - i.e. only drop type errors - if err != nil || value.Root == nil { - return nil - } - - tangleLog, err := mlog.Get(librarian.Addr(value.Root.Hash)) - if err != nil { - return errors.Wrap(err, "error opening sublog") - } - - _, err = tangleLog.Append(seq) - // log.Println(msg.Key.Ref(), value.Root.Ref(), seq) - return errors.Wrapf(err, "error appending root message %v", msg.Key()) -} diff --git a/private/box2/boxer.go b/private/box2/boxer.go index 73c77a82..a1ef5ee6 100644 --- a/private/box2/boxer.go +++ b/private/box2/boxer.go @@ -1,6 +1,8 @@ package box2 import ( + "bytes" + "encoding/base64" "encoding/binary" stderr "errors" "fmt" @@ -9,7 +11,7 @@ import ( "github.com/pkg/errors" "golang.org/x/crypto/nacl/secretbox" - "go.cryptoscope.co/ssb/keys" + "go.cryptoscope.co/ssb/private/keys" refs "go.mindeco.de/ssb-refs" "go.mindeco.de/ssb-refs/tfk" ) @@ -198,16 +200,28 @@ func deriveMessageKey(author *refs.FeedRef, prev *refs.MessageRef, candidates [] return slotKeys, info, nil } -// TODO: Maybe return entire decrypted message? -func (bxr *Boxer) Decrypt(ctxt []byte, author *refs.FeedRef, prev *refs.MessageRef, candidates []keys.Recipient) ([]byte, error) { +func (bxr *Boxer) GetReadKey(ctxt []byte, author *refs.FeedRef, prev *refs.MessageRef, candidates []keys.Recipient) ([]byte, error) { + _, readKey, _, err := bxr.getReadKey(ctxt, author, prev, candidates) + if err != nil { + return nil, err + } + return readKey[:], nil +} + +func (bxr *Boxer) getReadKey(ctxt []byte, author *refs.FeedRef, prev *refs.MessageRef, candidates []keys.Recipient) ( + []byte, + [KeySize]byte, + makeHKDFContextList, + error) { slotKeys, info, err := deriveMessageKey(author, prev, candidates) if err != nil { - return nil, errors.Wrap(err, "error constructing keying information") + err = errors.Wrap(err, "error constructing keying information") + return nil, [KeySize]byte{}, nil, err } var ( hdr = make([]byte, 16) msgKey, headerKey [KeySize]byte - readKey, bodyKey [KeySize]byte + readKey [KeySize]byte slot []byte ok bool i, j, k int @@ -228,11 +242,11 @@ OUTER: err = DeriveTo(readKey[:], msgKey[:], info([]byte("read_key"))...) if err != nil { - return nil, err + return nil, [KeySize]byte{}, nil, err } err = DeriveTo(headerKey[:], readKey[:], info([]byte("header_key"))...) if err != nil { - return nil, err + return nil, [KeySize]byte{}, nil, err } hdr, ok = secretbox.Open(hdr[:0], headerbox, &zero24, &headerKey) @@ -242,12 +256,26 @@ OUTER: } } if !ok { - return nil, ErrCouldNotDecrypt + err = ErrCouldNotDecrypt + return nil, [KeySize]byte{}, nil, err + } + + return hdr, readKey, info, nil +} + +// Decrypt takes the ciphertext, it's auther and the previous hash of the message and some canddiates to try to decrypt with. +// It returns the decrypted cleartext or an error. +func (bxr *Boxer) Decrypt(ctxt []byte, author *refs.FeedRef, prev *refs.MessageRef, candidates []keys.Recipient) ([]byte, error) { + // TODO + hdr, readKey, info, err := bxr.getReadKey(ctxt, author, prev, candidates) + if err != nil { + return nil, err } var ( bodyOffset = int(binary.LittleEndian.Uint16(hdr)) plain = make([]byte, 0, len(ctxt)-bodyOffset-secretbox.Overhead) + bodyKey [KeySize]byte ) // decrypt body @@ -255,7 +283,7 @@ OUTER: if err != nil { return nil, err } - plain, ok = secretbox.Open(plain, ctxt[bodyOffset:], &zero24, &bodyKey) + plain, ok := secretbox.Open(plain, ctxt[bodyOffset:], &zero24, &bodyKey) if !ok { return nil, ErrInvalid } @@ -264,6 +292,26 @@ OUTER: } // utils +// func (mgr *Manager) + +var box2Suffix = []byte(".box2\"") + +func GetCiphertextFromMessage(m refs.Message) ([]byte, error) { + content := m.ContentBytes() + + if !bytes.HasSuffix(content, box2Suffix) { + return nil, fmt.Errorf("message does not have .box2 suffix") + } + + n := base64.StdEncoding.DecodedLen(len(content)) + ctxt := make([]byte, n) + decn, err := base64.StdEncoding.Decode(ctxt, bytes.TrimSuffix(content, box2Suffix)[1:]) + if err != nil { + return nil, err + } + return ctxt[:decn], nil +} + func clear(buf []byte) { for i := range buf { buf[i] = 0 diff --git a/private/box2/boxer_test.go b/private/box2/boxer_test.go index 96f930cc..249b221f 100644 --- a/private/box2/boxer_test.go +++ b/private/box2/boxer_test.go @@ -9,7 +9,7 @@ import ( "github.com/stretchr/testify/require" "go.cryptoscope.co/ssb" - "go.cryptoscope.co/ssb/keys" + "go.cryptoscope.co/ssb/private/keys" ) func TestBoxer(t *testing.T) { diff --git a/private/box2/encoding.go b/private/box2/encoding.go index a4c11b4f..0242302e 100644 --- a/private/box2/encoding.go +++ b/private/box2/encoding.go @@ -49,16 +49,17 @@ func EncodeSLP(out []byte, list ...[]byte) []byte { func DeriveTo(out, key []byte, infos ...[]byte) error { if n := len(out); n != 32 { - return fmt.Errorf("box2: expected 32bytes as output argument, got %d", n) + return fmt.Errorf("box2: expected 32b as output argument, got %d", n) } - r := hkdf.Expand(sha256.New, key, EncodeSLP(nil, infos...)) + slp := EncodeSLP(nil, infos...) + r := hkdf.Expand(sha256.New, key, slp) nout, err := r.Read(out) if err != nil { return fmt.Errorf("box2: failed to derive key: %w", err) } if nout != 32 { - return fmt.Errorf("box2: expected to read 32bytes into output, got %d", nout) + return fmt.Errorf("box2: expected to read 32b into output, got %d", nout) } return nil diff --git a/private/box2/encoding_test.go b/private/box2/encoding_test.go index 7981e23f..5ddcd354 100644 --- a/private/box2/encoding_test.go +++ b/private/box2/encoding_test.go @@ -42,7 +42,7 @@ func TestEncodeList(t *testing.T) { for _, tc := range tcs { tc := tc t.Run(tc.name, func(t *testing.T) { - require.Equal(t, tc.out, encodeList(nil, tc.in)) + require.Equal(t, tc.out, EncodeSLP(nil, tc.in...)) }) } } diff --git a/private/box2/spec_box_test.go b/private/box2/spec_box_test.go index 37168ba9..46242235 100644 --- a/private/box2/spec_box_test.go +++ b/private/box2/spec_box_test.go @@ -7,7 +7,7 @@ import ( "go.mindeco.de/ssb-refs/tfk" "github.com/stretchr/testify/require" - "go.cryptoscope.co/ssb/keys" + "go.cryptoscope.co/ssb/private/keys" ) type boxSpecTest struct { diff --git a/private/box2/spec_cloakedid_test.go b/private/box2/spec_cloakedid_test.go index af5979a0..d4508400 100644 --- a/private/box2/spec_cloakedid_test.go +++ b/private/box2/spec_cloakedid_test.go @@ -4,7 +4,7 @@ import ( "testing" "github.com/stretchr/testify/require" - "go.cryptoscope.co/ssb/keys" + "go.cryptoscope.co/ssb/private/keys" ) type cloakedIDSpecTest struct { diff --git a/private/box2/spec_derive_test.go b/private/box2/spec_derive_test.go index ba76469b..41133865 100644 --- a/private/box2/spec_derive_test.go +++ b/private/box2/spec_derive_test.go @@ -4,7 +4,7 @@ import ( "testing" "github.com/stretchr/testify/require" - "go.cryptoscope.co/ssb/keys" + "go.cryptoscope.co/ssb/private/keys" "go.mindeco.de/ssb-refs/tfk" ) diff --git a/private/box2/spec_unbox_test.go b/private/box2/spec_unbox_test.go index b2b71ae4..e7419956 100644 --- a/private/box2/spec_unbox_test.go +++ b/private/box2/spec_unbox_test.go @@ -4,7 +4,7 @@ import ( "testing" "github.com/stretchr/testify/require" - "go.cryptoscope.co/ssb/keys" + "go.cryptoscope.co/ssb/private/keys" "go.mindeco.de/ssb-refs/tfk" ) diff --git a/private/box2/spec_unslot_test.go b/private/box2/spec_unslot_test.go index 450cc492..072c7f4d 100644 --- a/private/box2/spec_unslot_test.go +++ b/private/box2/spec_unslot_test.go @@ -4,7 +4,7 @@ import ( "testing" "github.com/stretchr/testify/require" - "go.cryptoscope.co/ssb/keys" + "go.cryptoscope.co/ssb/private/keys" "go.mindeco.de/ssb-refs/tfk" ) diff --git a/private/groups_test.go b/private/groups_test.go index 368d579d..164ad2d6 100644 --- a/private/groups_test.go +++ b/private/groups_test.go @@ -3,8 +3,6 @@ package private_test import ( "bytes" "context" - "encoding/base64" - "encoding/hex" "encoding/json" "fmt" "os" @@ -17,13 +15,12 @@ import ( "github.com/go-kit/kit/log/level" "github.com/stretchr/testify/require" "go.cryptoscope.co/librarian" - "go.cryptoscope.co/luigi" "go.cryptoscope.co/margaret" "go.cryptoscope.co/margaret/multilog/roaring" "golang.org/x/sync/errgroup" "go.cryptoscope.co/ssb" - "go.cryptoscope.co/ssb/indexes" + "go.cryptoscope.co/ssb/internal/storedrefs" "go.cryptoscope.co/ssb/private" "go.cryptoscope.co/ssb/sbot" refs "go.mindeco.de/ssb-refs" @@ -63,7 +60,6 @@ func TestGroupsFullCircle(t *testing.T) { sbot.WithRepoPath(filepath.Join(testRepo, "srh")), sbot.WithListenAddr(":0"), sbot.LateOption(sbot.WithUNIXSocket()), - sbot.LateOption(sbot.MountSimpleIndex("get", indexes.OpenGet)), // todo muxrpc plugin is hardcoded ) r.NoError(err) botgroup.Go(bs.Serve(srh)) @@ -79,26 +75,14 @@ func TestGroupsFullCircle(t *testing.T) { t.Log(cloaked.Ref(), "\nroot:", groupTangleRoot.Ref()) - // helper function, closured to wrap the r-helper suffix := []byte(".box2\"") - getCiphertext := func(m refs.Message) []byte { - content := m.ContentBytes() - - r.True(bytes.HasSuffix(content, suffix), "%q", content) - - n := base64.StdEncoding.DecodedLen(len(content)) - ctxt := make([]byte, n) - decn, err := base64.StdEncoding.Decode(ctxt, bytes.TrimSuffix(content, suffix)[1:]) - r.NoError(err) - return ctxt[:decn] - } // make sure this is an encrypted message msg, err := srh.Get(*groupTangleRoot) r.NoError(err) // can we decrypt it? - clear, err := srh.Groups.DecryptBox2(getCiphertext(msg), srh.KeyPair.Id, msg.Previous()) + clear, err := srh.Groups.DecryptBox2Message(msg) r.NoError(err) t.Log(string(clear)) @@ -120,7 +104,6 @@ func TestGroupsFullCircle(t *testing.T) { sbot.WithRepoPath(filepath.Join(testRepo, "tal")), sbot.WithListenAddr(":0"), sbot.LateOption(sbot.WithUNIXSocket()), - sbot.LateOption(sbot.MountSimpleIndex("get", indexes.OpenGet)), // todo muxrpc plugin is hardcoded ) r.NoError(err) botgroup.Go(bs.Serve(tal)) @@ -163,12 +146,12 @@ func TestGroupsFullCircle(t *testing.T) { // some length checks srhsFeeds, ok := srh.GetMultiLog("userFeeds") r.True(ok) - srhsCopyOfTal, err := srhsFeeds.Get(tal.KeyPair.Id.StoredAddr()) + srhsCopyOfTal, err := srhsFeeds.Get(storedrefs.Feed(tal.KeyPair.Id)) r.NoError(err) talsFeeds, ok := tal.GetMultiLog("userFeeds") r.True(ok) - talsCopyOfSrh, err := talsFeeds.Get(srh.KeyPair.Id.StoredAddr()) + talsCopyOfSrh, err := talsFeeds.Get(storedrefs.Feed(srh.KeyPair.Id)) r.NoError(err) // did we get the expected number of messages? @@ -192,7 +175,7 @@ func TestGroupsFullCircle(t *testing.T) { r.True(bytes.HasSuffix(content, suffix), "%q", content) t.Log(string(content)) - decr, err := tal.Groups.DecryptBox2(getCiphertext(addMsgCopy), addMsgCopy.Author(), addMsgCopy.Previous()) + decr, err := tal.Groups.DecryptBox2Message(addMsgCopy) r.NoError(err) t.Log(string(decr)) @@ -201,7 +184,7 @@ func TestGroupsFullCircle(t *testing.T) { r.NoError(err) t.Logf("%x", ga.GroupKey) - cloaked2, err := tal.Groups.Join(ga.GroupKey, ga.InitialMessage) + cloaked2, err := tal.Groups.Join(ga.GroupKey, ga.Root) r.NoError(err) r.Equal(cloaked.Hash, cloaked2.Hash, "cloaked ID not equal") @@ -224,9 +207,9 @@ func TestGroupsFullCircle(t *testing.T) { replyMsg, err := srh.Get(*reply) r.NoError(err) - replyContent, err := srh.Groups.DecryptBox2(getCiphertext(replyMsg), replyMsg.Author(), replyMsg.Previous()) + replyContent, err := srh.Groups.DecryptBox2Message(replyMsg) r.NoError(err) - t.Log(string(replyContent)) + t.Log("decrypted reply:", string(replyContent)) // indexed? chkCount := func(ml *roaring.MultiLog) func(tipe librarian.Addr, cnt int) { @@ -244,27 +227,40 @@ func TestGroupsFullCircle(t *testing.T) { } } - chkCount(srh.ByType)("test", 2) - chkCount(srh.ByType)("post", 2) + chkCount(srh.ByType)("string:test", 2) + chkCount(srh.ByType)("string:post", 2) - chkCount(tal.ByType)("test", 2) - chkCount(tal.ByType)("post", 1) // TODO: reindex + chkCount(tal.ByType)("string:test", 2) + chkCount(tal.ByType)("string:post", 1) // TODO: reindex - addr := librarian.Addr("box2:") + srh.KeyPair.Id.StoredAddr() + addr := librarian.Addr("box2:") + storedrefs.Feed(srh.KeyPair.Id) chkCount(srh.Private)(addr, 3) - addr = librarian.Addr("box2:") + tal.KeyPair.Id.StoredAddr() + addr = librarian.Addr("box2:") + storedrefs.Feed(tal.KeyPair.Id) chkCount(tal.Private)(addr, 2) // TODO: reindex - t.Log("srh") - streamLog(t, srh.RootLog) + /* + t.Log("srh") + testutils.StreamLog(t, srh.ReceiveLog) + t.Log("tal") + testutils.StreamLog(t, tal.ReceiveLog) + */ - t.Log("tal") - streamLog(t, tal.RootLog) + addr = librarian.Addr("meta:box2") + allBoxed, err := tal.Private.LoadInternalBitmap(addr) + r.NoError(err) + t.Log("all boxed:", allBoxed.String()) - stillBoxed, err := tal.Private.LoadInternalBitmap(librarian.Addr("notForUs:box2")) + addr = librarian.Addr("box2:") + storedrefs.Feed(tal.KeyPair.Id) + readable, err := tal.Private.LoadInternalBitmap(addr) r.NoError(err) - t.Log("stillBoxed:", stillBoxed.String()) + + allBoxed.AndNot(readable) + + if n := allBoxed.GetCardinality(); n > 0 { + t.Errorf("still have boxed messages that are not indexed: %d", n) + t.Log("still boxed:", allBoxed.String()) + } tal.Shutdown() srh.Shutdown() @@ -291,32 +287,3 @@ func (bs botServer) Serve(s *sbot.Sbot) func() error { return err } } - -func streamLog(t *testing.T, l margaret.Log) { - - r := require.New(t) - src, err := l.Query() - r.NoError(err) - i := 0 - for { - v, err := src.Next(context.TODO()) - if luigi.IsEOS(err) { - break - } - - mm, ok := v.(refs.Message) - r.True(ok, "%T", v) - - t.Log(i, mm.Key().ShortRef()) - t.Log(mm.Author().ShortRef(), mm.Seq()) - - b := mm.ContentBytes() - if len(b) > 128 { - b = b[len(b)-32:] - } - t.Logf("\n%s", hex.Dump(b)) - - i++ - } - -} diff --git a/keys/error.go b/private/keys/error.go similarity index 100% rename from keys/error.go rename to private/keys/error.go diff --git a/keys/key.go b/private/keys/key.go similarity index 100% rename from keys/key.go rename to private/keys/key.go diff --git a/keys/ops_db_test.go b/private/keys/ops_db_test.go similarity index 100% rename from keys/ops_db_test.go rename to private/keys/ops_db_test.go diff --git a/keys/ops_key_test.go b/private/keys/ops_key_test.go similarity index 100% rename from keys/ops_key_test.go rename to private/keys/ops_key_test.go diff --git a/keys/ops_store_test.go b/private/keys/ops_store_test.go similarity index 100% rename from keys/ops_store_test.go rename to private/keys/ops_store_test.go diff --git a/keys/store.go b/private/keys/store.go similarity index 100% rename from keys/store.go rename to private/keys/store.go diff --git a/keys/store_test.go b/private/keys/store_test.go similarity index 100% rename from keys/store_test.go rename to private/keys/store_test.go diff --git a/keys/types.go b/private/keys/types.go similarity index 100% rename from keys/types.go rename to private/keys/types.go diff --git a/private/manager.go b/private/manager.go index 252e9284..5c8e11d0 100644 --- a/private/manager.go +++ b/private/manager.go @@ -1,13 +1,19 @@ package private import ( + "bytes" + "context" "crypto/rand" "crypto/sha256" + "encoding/base64" "fmt" "io" "sort" + "github.com/cryptix/go/encodedTime" "github.com/pkg/errors" + "go.cryptoscope.co/luigi" + "go.cryptoscope.co/luigi/mfr" "go.cryptoscope.co/margaret" "go.cryptoscope.co/margaret/multilog" "golang.org/x/crypto/curve25519" @@ -15,15 +21,17 @@ import ( "go.cryptoscope.co/ssb" "go.cryptoscope.co/ssb/internal/extra25519" - "go.cryptoscope.co/ssb/keys" "go.cryptoscope.co/ssb/private/box" "go.cryptoscope.co/ssb/private/box2" + "go.cryptoscope.co/ssb/private/keys" refs "go.mindeco.de/ssb-refs" "go.mindeco.de/ssb-refs/tfk" ) +// Manager is in charge of storing and retriving keys with the help of keymgr, can de- and encrypt messages and publish them. type Manager struct { - receiveLog margaret.Log + receiveLog margaret.Log + receiveByRef ssb.Getter publog ssb.Publisher tangles multilog.MultiLog @@ -34,13 +42,17 @@ type Manager struct { rand io.Reader } -func NewManager(author *ssb.KeyPair, publishLog ssb.Publisher, km *keys.Store, tangles multilog.MultiLog) *Manager { +// NewManager creates a new Manager +func NewManager(author *ssb.KeyPair, publishLog ssb.Publisher, km *keys.Store, rxlog margaret.Log, getter ssb.Getter, tangles multilog.MultiLog) *Manager { return &Manager{ - tangles: tangles, + receiveLog: rxlog, + receiveByRef: getter, author: author, publog: publishLog, + tangles: tangles, + keymgr: km, rand: rand.Reader, } @@ -51,6 +63,7 @@ var ( infoContext = []byte("envelope-ssb-dm-v1/key") ) +// GetOrDeriveKeyFor derives an encryption key for 1:1 private messages with an other feed. func (mgr *Manager) GetOrDeriveKeyFor(other *refs.FeedRef) (keys.Recipients, error) { ourID := keys.ID(sortAndConcat(mgr.author.Id.ID, other.ID)) scheme := keys.SchemeDiffieStyleConvertedED25519 @@ -95,13 +108,14 @@ func (mgr *Manager) GetOrDeriveKeyFor(other *refs.FeedRef) (keys.Recipients, err var messageShared = make([]byte, 32) var bs = bytesSlice{ - append(myCurvePub[:], tfkMy...), - append(otherCurvePub[:], tfkOther...), + // PSEUDO TFK + // TODO: add proper type 3 for these curve keys + append(append([]byte{03, 00}, myCurvePub[:]...), tfkMy...), + append(append([]byte{03, 00}, otherCurvePub[:]...), tfkOther...), } sort.Sort(bs) slpInfo := box2.EncodeSLP(nil, infoContext, bs[0], bs[1]) - n, err := hkdf.New(sha256.New, keyInput[:], dmSalt, slpInfo).Read(messageShared) if err != nil { return nil, err @@ -129,12 +143,14 @@ func (mgr *Manager) GetOrDeriveKeyFor(other *refs.FeedRef) (keys.Recipients, err return ks, nil } +// EncryptBox1 creates box1 ciphertext that is readable by the recipients. func (mgr *Manager) EncryptBox1(content []byte, rcpts ...*refs.FeedRef) ([]byte, error) { bxr := box.NewBoxer(mgr.rand) ctxt, err := bxr.Encrypt(content, rcpts...) return ctxt, errors.Wrap(err, "error encrypting message (box1)") } +// EncryptBox2 creates box2 ciphertext func (mgr *Manager) EncryptBox2(content []byte, prev *refs.MessageRef, recpts []refs.Ref) ([]byte, error) { // first, look up keys @@ -166,8 +182,9 @@ func (mgr *Manager) EncryptBox2(content []byte, prev *refs.MessageRef, recpts [] bxr := box2.NewBoxer(mgr.rand) ctxt, err := bxr.Encrypt(content, mgr.author.Id, prev, allKeys) return ctxt, errors.Wrap(err, "error encrypting message (box1)") - } + +// DecryptBox1 does exactly what the name suggests, it returns the cleartext if mgr.author can read it func (mgr *Manager) DecryptBox1(ctxt []byte) ([]byte, error) { // TODO: key managment (single author manager) @@ -190,15 +207,12 @@ func (mgr *Manager) DecryptBox1(ctxt []byte) ([]byte, error) { // copy(keyPair.Pair.Secret[:], ) // copy(keyPair.Pair.Public[:], mgr.author.ID) - // try decrypt - if mgr.rand == nil { - panic("what?!") - } bxr := box.NewBoxer(mgr.rand) plain, err := bxr.Decrypt(mgr.author, []byte(ctxt)) - return plain, errors.Wrap(err, "could not decrypt") + return plain, err } +// DecryptBox2 decrypts box2 messages, using the keys that were previously stored/received. func (mgr *Manager) DecryptBox2(ctxt []byte, author *refs.FeedRef, prev *refs.MessageRef) ([]byte, error) { // assumes 1:1 pm // fetch feed2feed shared key @@ -219,6 +233,74 @@ func (mgr *Manager) DecryptBox2(ctxt []byte, author *refs.FeedRef, prev *refs.Me // try decrypt bxr := box2.NewBoxer(mgr.rand) plain, err := bxr.Decrypt([]byte(ctxt), author, prev, allKeys) - return plain, errors.Wrap(err, "could not decrypt") + return plain, err +} + +func (mgr *Manager) DecryptMessage(m refs.Message) ([]byte, error) { + + if ctxt, err := mgr.DecryptBox2Message(m); err == nil { + return ctxt, nil + } + + if ctxt, err := mgr.DecryptBox1Message(m); err == nil { + return ctxt, nil + } + + return nil, fmt.Errorf("private: not a boxed message") +} + +func (mgr *Manager) DecryptBox1Message(m refs.Message) ([]byte, error) { + ciphtext := m.ContentBytes() + + box1Suffix := []byte(".box\"") + if !bytes.HasSuffix(ciphtext, box1Suffix) { + return nil, fmt.Errorf("private: not a box1 message") + } + + b64data := bytes.TrimSuffix(ciphtext[1:], []byte(".box\"")) + boxedData := make([]byte, base64.StdEncoding.DecodedLen(len(ciphtext)-6)) + n, err := base64.StdEncoding.Decode(boxedData, b64data) + if err != nil { + return nil, err + } + + return mgr.DecryptBox1(boxedData[:n]) +} + +func (mgr *Manager) DecryptBox2Message(m refs.Message) ([]byte, error) { + ctxt, err := box2.GetCiphertextFromMessage(m) + if err != nil { + return nil, err + } + + return mgr.DecryptBox2(ctxt, m.Author(), m.Previous()) +} + +func (mgr *Manager) WrappedUnboxingSink(snk luigi.Sink) luigi.Sink { + return mfr.SinkMap(snk, func(_ context.Context, v interface{}) (interface{}, error) { + msg, ok := v.(refs.Message) + if !ok { + return nil, fmt.Errorf("failed to find message in empty interface(%T)", v) + } + + cleartxt, err := mgr.DecryptMessage(msg) + if err != nil { + return v, nil + } + + var rv refs.KeyValueRaw + rv.Key_ = msg.Key() + rv.Value.Author = *msg.Author() + rv.Value.Previous = msg.Previous() + rv.Value.Sequence = margaret.BaseSeq(msg.Seq()) + rv.Value.Timestamp = encodedTime.NewMillisecs(msg.Claimed().Unix()) + rv.Value.Signature = "reboxed" + + rv.Value.Content = cleartxt + + rv.Meta = make(map[string]interface{}) + rv.Meta["private"] = true + return rv, nil + }) } diff --git a/private/manager_groups.go b/private/manager_groups.go index d72be4f4..9574e331 100644 --- a/private/manager_groups.go +++ b/private/manager_groups.go @@ -6,10 +6,12 @@ import ( "encoding/json" "fmt" + "go.mindeco.de/ssb-refs/tfk" + "go.cryptoscope.co/margaret" "go.cryptoscope.co/ssb/private/box2" - "go.cryptoscope.co/ssb/keys" + "go.cryptoscope.co/ssb/private/keys" refs "go.mindeco.de/ssb-refs" ) @@ -60,6 +62,9 @@ func (mgr *Manager) Create(name string) (*refs.MessageRef, *refs.MessageRef, err return cloakedID, publicRoot, nil } +// Join is called with a groupKey and the tangle root for the group. +// It adds the key to the keystore so that messages to this group can be decrypted. +// It returns the cloaked message reference or an error. func (mgr *Manager) Join(groupKey []byte, root *refs.MessageRef) (*refs.MessageRef, error) { var r keys.Recipient r.Scheme = keys.SchemeLargeSymmetricGroup @@ -85,7 +90,36 @@ func (mgr *Manager) deriveCloakedAndStoreNewKey(k keys.Recipient) (*refs.Message cloakedID.Algo = refs.RefAlgoCloakedGroup cloakedID.Hash = make([]byte, 32) - err := box2.DeriveTo(cloakedID.Hash, k.Key, []byte("cloaked_msg_id"), k.Metadata.GroupRoot.Hash) + if k.Key == nil { + return nil, fmt.Errorf("deriveCloaked: nil recipient key") + } + + if k.Metadata.GroupRoot == nil { + return nil, fmt.Errorf("deriveCloaked: groupRoot nil") + } + + // TODO: might find a way without this 2nd roundtrip of getting the message. + initMsg, err := mgr.receiveByRef.Get(*k.Metadata.GroupRoot) + if err != nil { + return nil, err + } + + ctxt, err := box2.GetCiphertextFromMessage(initMsg) + if err != nil { + return nil, err + } + + readKey, err := box2.NewBoxer(mgr.rand).GetReadKey(ctxt, initMsg.Author(), initMsg.Previous(), keys.Recipients{k}) + if err != nil { + return nil, err + } + + rootAsTFK, err := tfk.Encode(k.Metadata.GroupRoot) + if err != nil { + return nil, err + } + + err = box2.DeriveTo(cloakedID.Hash, readKey, []byte("cloaked_msg_id"), rootAsTFK) if err != nil { return nil, err } @@ -103,49 +137,23 @@ func (mgr *Manager) deriveCloakedAndStoreNewKey(k keys.Recipient) (*refs.Message return &cloakedID, nil } -/* -{ - type: 'group/add-member', - version: 'v1', - groupKey: '3YUat1ylIUVGaCjotAvof09DhyFxE8iGbF6QxLlCWWc=', - initialMsg: '%THxjTGPuXvvxnbnAV7xVuVXdhDcmoNtDDN0j3UTxcd8=.sha256', - text: 'welcome keks!', // optional - recps: [ - '%vof09Dhy3YUat1ylIUVGaCjotAFxE8iGbF6QxLlCWWc=.cloaked', // group_id - '@YXkE3TikkY4GFMX3lzXUllRkNTbj5E+604AkaO1xbz8=.ed25519' // feed_id (for new person) - ], - tangles: { - group: { - root: '%THxjTGPuXvvxnbnAV7xVuVXdhDcmoNtDDN0j3UTxcd8=.sha256', - previous: [ - '%Sp294oBk7OJxizvPOlm6Sqk3fFJA2EQFiyJ1MS/BZ9E=.sha256' - ] - }, - members: { - root: '%THxjTGPuXvvxnbnAV7xVuVXdhDcmoNtDDN0j3UTxcd8=.sha256', - previous: [ - '%lm6Sqk3fFJA2EQFiyJ1MSASDASDASDASDASDAS/BZ9E=.sha256', - '%Sp294oBk7OJxizvPOlm6Sqk3fFJA2EQFiyJ1MS/BZ9E=.sha256' - ] - } - } -} -*/ - +// GroupAddMember is a JSON serialization helper. +// See https://github.com/ssbc/private-group-spec/tree/master/group/add-member for more. type GroupAddMember struct { Type string `json:"type"` Text string `json:"text"` Version string `json:"version"` - GroupKey keys.Base64String `json:"groupKey"` - InitialMessage *refs.MessageRef `json:"initialMsg"` + GroupKey keys.Base64String `json:"groupKey"` + Root *refs.MessageRef `json:"root"` // initial message Recps []string `json:"recps"` Tangles refs.Tangles `json:"tangles"` } +// AddMember creates, encrypts and publishes a GroupAddMember message. func (mgr *Manager) AddMember(groupID *refs.MessageRef, r *refs.FeedRef, welcome string) (*refs.MessageRef, error) { if groupID.Algo != refs.RefAlgoCloakedGroup { return nil, fmt.Errorf("not a group") @@ -174,7 +182,7 @@ func (mgr *Manager) AddMember(groupID *refs.MessageRef, r *refs.FeedRef, welcome ga.GroupKey = keys.Base64String(gskey[0].Key) groupRoot := gskey[0].Metadata.GroupRoot - ga.InitialMessage = groupRoot + ga.Root = groupRoot ga.Recps = []string{groupID.Ref(), r.Ref()} @@ -191,18 +199,41 @@ func (mgr *Manager) AddMember(groupID *refs.MessageRef, r *refs.FeedRef, welcome return mgr.encryptAndPublish(jsonContent, gskey) } +// PublishTo encrypts and publishes a json blob as content to a group. func (mgr *Manager) PublishTo(groupID *refs.MessageRef, content []byte) (*refs.MessageRef, error) { if groupID.Algo != refs.RefAlgoCloakedGroup { return nil, fmt.Errorf("not a group") } - r, err := mgr.keymgr.GetKeys(keys.SchemeLargeSymmetricGroup, groupID.Hash) + rs, err := mgr.keymgr.GetKeys(keys.SchemeLargeSymmetricGroup, groupID.Hash) + if err != nil { + return nil, err + } + if nr := len(rs); nr != 1 { + return nil, fmt.Errorf("expected 1 key for group, got %d", nr) + } + r := rs[0] + + // assign group tangle + var decodedContent map[string]interface{} + err = json.Unmarshal(content, &decodedContent) + if err != nil { + return nil, err + } + + var groupState = map[string]refs.TanglePoint{} + groupState["group"] = mgr.getTangleState(r.Metadata.GroupRoot, "group") + decodedContent["tangles"] = groupState + + updatedContent, err := json.Marshal(decodedContent) if err != nil { return nil, err } - return mgr.encryptAndPublish(content, r) + return mgr.encryptAndPublish(updatedContent, rs) } +// PublishPostTo publishes a new post to a group. +// TODO: reply root? func (mgr *Manager) PublishPostTo(groupID *refs.MessageRef, text string) (*refs.MessageRef, error) { if groupID.Algo != refs.RefAlgoCloakedGroup { return nil, fmt.Errorf("not a group") @@ -236,6 +267,10 @@ func (mgr *Manager) PublishPostTo(groupID *refs.MessageRef, text string) (*refs. // TODO: protect against race of changing previous func (mgr *Manager) encryptAndPublish(c []byte, recps keys.Recipients) (*refs.MessageRef, error) { + if !json.Valid(c) { + return nil, fmt.Errorf("box2 manager: passed content is not valid JSON") + } + prev, err := mgr.getPrevious() if err != nil { return nil, err diff --git a/private/manager_test.go b/private/manager_test.go index b5e2386d..2ef2c6ec 100644 --- a/private/manager_test.go +++ b/private/manager_test.go @@ -13,7 +13,7 @@ import ( "go.cryptoscope.co/ssb" "go.cryptoscope.co/ssb/internal/extra25519" - "go.cryptoscope.co/ssb/keys" + "go.cryptoscope.co/ssb/private/keys" refs "go.mindeco.de/ssb-refs" ) diff --git a/private/publish_test.go b/private/publish_test.go index f2c752aa..da7162ea 100644 --- a/private/publish_test.go +++ b/private/publish_test.go @@ -16,14 +16,14 @@ import ( "go.cryptoscope.co/librarian" "go.cryptoscope.co/luigi" "go.cryptoscope.co/margaret" - refs "go.mindeco.de/ssb-refs" "go.cryptoscope.co/ssb" "go.cryptoscope.co/ssb/client" - "go.cryptoscope.co/ssb/indexes" + "go.cryptoscope.co/ssb/internal/storedrefs" "go.cryptoscope.co/ssb/multilogs" "go.cryptoscope.co/ssb/private" "go.cryptoscope.co/ssb/sbot" + refs "go.mindeco.de/ssb-refs" ) func TestPrivatePublish(t *testing.T) { @@ -53,8 +53,8 @@ func testPublishPerAlgo(algo string) func(t *testing.T) { sbot.WithRepoPath(srvRepo), sbot.WithListenAddr(":0"), sbot.LateOption(sbot.WithUNIXSocket()), - sbot.LateOption(sbot.MountSimpleIndex("get", indexes.OpenGet)), // todo muxrpc plugin is hardcoded ) + r.NoError(err, "failed to init sbot") const n = 32 for i := n; i > 0; i-- { @@ -102,10 +102,10 @@ func testPublishPerAlgo(algo string) func(t *testing.T) { pl, ok := srv.GetMultiLog(multilogs.IndexNamePrivates) r.True(ok) - userPrivs, err := pl.Get(librarian.Addr("box1:") + srv.KeyPair.Id.StoredAddr()) + userPrivs, err := pl.Get(librarian.Addr("box1:") + storedrefs.Feed(srv.KeyPair.Id)) r.NoError(err) - unboxlog := private.NewUnboxerLog(srv.RootLog, userPrivs, srv.KeyPair) + unboxlog := private.NewUnboxerLog(srv.ReceiveLog, userPrivs, srv.KeyPair) src, err = unboxlog.Query(margaret.SeqWrap(true)) r.NoError(err) diff --git a/private/tangles.go b/private/tangles.go index b1176b7b..a73b3e8a 100644 --- a/private/tangles.go +++ b/private/tangles.go @@ -1,12 +1,9 @@ package private import ( - "bytes" "context" - "encoding/base64" "encoding/json" "fmt" - "sort" "go.cryptoscope.co/librarian" "go.cryptoscope.co/luigi" @@ -16,18 +13,20 @@ import ( ) func (mgr *Manager) getTangleState(root *refs.MessageRef, tname string) refs.TanglePoint { - addr := librarian.Addr(append(root.Hash, []byte(tname)...)) // TODO: fill tangle + addr := librarian.Addr(append([]byte("v2:"+tname+":"), root.Hash...)) thandle, err := mgr.tangles.Get(addr) if err != nil { return refs.TanglePoint{Root: root, Previous: []*refs.MessageRef{root}} } - prev, err := mgr.getLooseEnds(thandle, tname) + heads, err := mgr.getLooseEnds(thandle, tname) if err != nil { panic(err) } - - return refs.TanglePoint{Root: root, Previous: append(prev, root)} + if len(heads) == 0 { + heads = refs.MessageRefs{root} + } + return refs.TanglePoint{Root: root, Previous: heads} } func (mgr *Manager) getLooseEnds(l margaret.Log, tname string) (refs.MessageRefs, error) { @@ -51,15 +50,9 @@ func (mgr *Manager) getLooseEnds(l margaret.Log, tname string) (refs.MessageRefs return nil, fmt.Errorf("not a mesg %T", src) } - // decrypt message - ctxt := mgr.getCiphertext(msg) - if ctxt == nil { - continue - } - - content, err := mgr.DecryptBox2(ctxt, msg.Author(), msg.Previous()) + content, err := mgr.DecryptBox2Message(msg) if err != nil { - fmt.Println("not for us?", err) + // fmt.Println("not for us?", err) // or deleted key? continue } @@ -76,34 +69,11 @@ func (mgr *Manager) getLooseEnds(l margaret.Log, tname string) (refs.MessageRefs } sorter := refs.ByPrevious{Items: tps, TangleName: tname} - sort.Sort(sorter) - - fmt.Println(len(tps), "message in tangle") - var refs = make(refs.MessageRefs, len(tps)) - for i, post := range tps { - fmt.Println("ref:", post.Key().Ref()) - refs[i] = post.Key() - } + sorter.FillLookup() + // sort.Sort(sorter) // not required for Heads() - return refs, nil -} - -var boxSuffix = []byte(".box2\"") - -func (mgr *Manager) getCiphertext(m refs.Message) []byte { - content := m.ContentBytes() - - if !bytes.HasSuffix(content, boxSuffix) { - return nil - } - - n := base64.StdEncoding.DecodedLen(len(content)) - ctxt := make([]byte, n) - decn, err := base64.StdEncoding.Decode(ctxt, bytes.TrimSuffix(content, boxSuffix)[1:]) - if err != nil { - return nil - } - return ctxt[:decn] + h := sorter.Heads() + return h, nil } type tangledPost struct { diff --git a/repo/migrations/badgerToroaring.go b/repo/migrations/badgerToroaring.go deleted file mode 100644 index 882e0037..00000000 --- a/repo/migrations/badgerToroaring.go +++ /dev/null @@ -1,47 +0,0 @@ -package migrations - -import ( - "os" - - "github.com/cryptix/go/logging" - "github.com/go-kit/kit/log/level" - "github.com/pkg/errors" - "go.cryptoscope.co/ssb/multilogs" - "go.cryptoscope.co/ssb/repo" -) - -func StillUsingBadger(log logging.Interface, r repo.Interface) (bool, error) { - v := CurrentVersion(r) - switch { - case v == 1: - // do the deed - case v < 1: - level.Error(log).Log("event", "repo is not version 1 yet", "v", v) - return false, nil - default: - return false, errors.Errorf("sbot/repo migrate: invalid version: %d", v) - } - - // check the db and the state file - var hasDB, hasState bool - - dbPath := r.GetPath(repo.PrefixMultiLog, multilogs.IndexNameFeeds, "db") - _, err := os.Stat(dbPath) - if err != nil && !os.IsNotExist(err) { - return false, errors.Wrap(err, "StillUsingBadger: failed to check old db path inside the repo") - } - if err == nil { - hasDB = true - } - - stateFilePath := r.GetPath(repo.PrefixMultiLog, multilogs.IndexNameFeeds, "state.json") - _, err = os.Stat(stateFilePath) - if err != nil && !os.IsNotExist(err) { - return false, errors.Wrap(err, "StillUsingBadger: failed to check old state file path inside the repo") - } - if err == nil { - hasState = true - } - - return hasDB && hasState, nil -} diff --git a/repo/migrations/migrations.go b/repo/migrations/migrations.go deleted file mode 100644 index 1026adb5..00000000 --- a/repo/migrations/migrations.go +++ /dev/null @@ -1,232 +0,0 @@ -// SPDX-License-Identifier: MIT - -package migrations - -import ( - "bytes" - "context" - "fmt" - "io" - "io/ioutil" - "log" - "os" - "strconv" - "time" - - "go.cryptoscope.co/luigi" - "go.cryptoscope.co/margaret" - refs "go.mindeco.de/ssb-refs" - - "github.com/cryptix/go/logging" - "github.com/pkg/errors" - "go.cryptoscope.co/margaret/codec/msgpack" - "go.cryptoscope.co/margaret/offset2" - "go.cryptoscope.co/ssb/message/legacy" - "go.cryptoscope.co/ssb/repo" -) - -func CurrentVersion(r repo.Interface) int { - version, err := ioutil.ReadFile(r.GetPath("version")) - if os.IsNotExist(err) { - return 0 - } else if err != nil { - log.Println("CurrentVersion error:", err) - return -1 - } - v, err := strconv.Atoi(string(version)) - if err != nil { - log.Printf("CurrentVersion failed to parse file content (%q):%s", string(version), err) - return -1 - } - return v -} - -// SetVersion should be atomic -func SetVersion(r repo.Interface, to int) error { - fname := r.GetPath("version") - vstr := []byte(strconv.Itoa(to)) - err := ioutil.WriteFile(fname, vstr, 0700) - return errors.Wrap(err, "SetVersion failed to write file") -} - -func UpgradeToMultiMessage(log logging.Interface, r repo.Interface) (bool, error) { - v := CurrentVersion(r) - switch { - case v == 0: - // do the deed - case v > 0: - log.Log("level", "info", "msg", "repo already updated", "v", v) - return false, nil - default: - return false, errors.Errorf("sbot/repo migrate: invalid version: %d", v) - } - - from, err := checkIfVersion0(r, log) - if err != nil { - return false, errors.Wrap(err, "pre-check failed failed") - } - - to, err := repo.OpenLog(r, "migrate-v1") - if err != nil { - return false, errors.Wrap(err, "error opening new log") - } - - gotMsgs, err := copyOffset(log, from, to) - if err != nil { - return false, errors.Wrap(err, "error copying new log") - } - - if err := validateNewLog(log, gotMsgs, to); err != nil { - return false, errors.Wrap(err, "error validating new log") - } - - err = from.(io.Closer).Close() - if err != nil { - return false, errors.Wrap(err, "error closing from log") - } - err = to.(io.Closer).Close() - if err != nil { - return false, errors.Wrap(err, "error closing to log") - } - - err = os.Rename(r.GetPath("log"), r.GetPath("log-bak-v0")) - if err != nil { - return false, errors.Wrap(err, "error moving old log into backup position") - } - - err = os.Rename(r.GetPath("logs", "migrate-v1"), r.GetPath("log")) - if err != nil { - return false, errors.Wrap(err, "error moving migrated log into position") - } - - return true, SetVersion(r, 1) -} - -func checkIfVersion0(r repo.Interface, log logging.Interface) (margaret.Log, error) { - fromPath := r.GetPath("log") - from, err := offset2.Open(fromPath, msgpack.New(&legacy.OldStoredMessage{})) - if err != nil { - return nil, errors.Wrap(err, "check-v0: failed to open source log") - } - - sv, err := from.Seq().Value() - if err != nil { - return nil, errors.Wrap(err, "check-v0: failed to establish sequence of source log") - } - fromSeq := sv.(margaret.Seq) - - if fromSeq.Seq() == margaret.SeqEmpty.Seq() { - // empty source, totally fine, just make a new empty one - return from, nil - } - - // simple check if we have a valid msg - v, err := from.Get(margaret.BaseSeq(0)) - if err != nil { - return nil, errors.Wrap(err, "check-v0: failed to get first message") - } - osm, ok := v.(legacy.OldStoredMessage) - if !ok { - return nil, errors.Errorf("check-v0: wrong type: %T", v) - } - - ref, _, err := legacy.Verify(osm.Raw, nil) // hmac stuff grm... TODO: env var?! - if err != nil { - return nil, errors.Wrap(err, "check-v0: verify failed") - } - - if !bytes.Equal(ref.Hash, osm.Key.Hash) { - return nil, errors.Errorf("check-v0: msg key and verifyied msg didn't line up?!") - } - return from, nil -} - -func copyOffset(log logging.Interface, from, to margaret.Log) ([]refs.MessageRef, error) { - - sv, err := from.Seq().Value() - if err != nil { - return nil, errors.Wrap(err, "upgrade-v0: no current sequence for from") - } - fromSeq := sv.(margaret.Seq) - - fromSrc, err := from.Query() - if err != nil { - return nil, errors.Wrap(err, "upgrade-v0: failed to construct query on from") - } - - start := time.Now() - - i := 0 - took := time.Now() - onePercent := fromSeq.Seq() / 10 - - var got []refs.MessageRef - track := luigi.FuncSink(func(ctx context.Context, v interface{}, err error) error { - if luigi.IsEOS(err) { // || margaret.IsErrNulled(err) - return nil - } - if err != nil { - return errors.Wrap(err, "pump failed") - } - - msg := v.(legacy.OldStoredMessage) - - got = append(got, *msg.Key) - - seq, err := to.Append(v) - if seq.Seq()%onePercent == 0 { - log.Log("level", "debug", "msg", "copy progress", "left", fromSeq.Seq()-seq.Seq(), "i", i, "took", time.Since(took)) - i++ - took = time.Now() - } - return err - }) - - log.Log("event", "start-copy", "seq", fromSeq.Seq()) - err = luigi.Pump(context.TODO(), track, fromSrc) - if err != nil { - return nil, errors.Wrap(err, "migrate: pumping messages failed") - } - log.Log("event", "copy-done", "took", time.Since(start)) - - return got, nil -} - -func validateNewLog(log logging.Interface, got []refs.MessageRef, to margaret.Log) error { - toSeq, err := to.Seq().Value() - if err != nil { - return err - } - - log.Log("event", "validating", "target has", toSeq) - - newTarget, err := to.Query() - if err != nil { - return err - } - start := time.Now() - i := 0 - n := len(got) - onePercent := n / 10 - for { - v, err := newTarget.Next(context.TODO()) - if luigi.IsEOS(err) { - break - } else if err != nil { - return err - } - - msg := v.(refs.Message) - - if !bytes.Equal(got[i].Hash, msg.Key().Hash) { - return fmt.Errorf("migrate failed - msg%d diverges", i) - } - if i%onePercent == 0 { - log.Log("level", "debug", "msg", "validate progress", "left", n-i) - } - i++ - } - - log.Log("event", "hash-check-done", "took", time.Since(start)) - return nil -} diff --git a/repo/migrations/migrations_test.go b/repo/migrations/migrations_test.go deleted file mode 100644 index 31f2491f..00000000 --- a/repo/migrations/migrations_test.go +++ /dev/null @@ -1,65 +0,0 @@ -// SPDX-License-Identifier: MIT - -package migrations - -import ( - "os" - "os/exec" - "path/filepath" - "testing" - - "github.com/stretchr/testify/require" - "go.cryptoscope.co/margaret" - "go.cryptoscope.co/margaret/codec/msgpack" - "go.cryptoscope.co/margaret/offset2" - - "go.cryptoscope.co/ssb/internal/testutils" - "go.cryptoscope.co/ssb/message/legacy" - "go.cryptoscope.co/ssb/repo" -) - -func TestUpgradeToMultiMessage(t *testing.T) { - r := require.New(t) - - logger := testutils.NewRelativeTimeLogger(nil) - - testPath := filepath.Join("testrun", t.Name()) - os.RemoveAll(testPath) - os.MkdirAll(testPath, 0700) - testRepo := repo.New(testPath) - - // testlog is a link to plugins/gossip/testdata/largeRepo/log - out, err := exec.Command("cp", "-rL", "testlog", testRepo.GetPath("log")).CombinedOutput() - r.NoError(err, "copy failed:%s", string(out)) - - from, err := offset2.Open(testRepo.GetPath("log"), msgpack.New(&legacy.OldStoredMessage{})) - r.NoError(err) - - // check for legacy testdata from the gossip plugin - currSeq, err := from.Seq().Value() - r.NoError(err) - r.EqualValues(431, currSeq) - msgV, err := from.Get(margaret.BaseSeq(430)) - r.NoError(err) - osm, ok := msgV.(legacy.OldStoredMessage) - r.True(ok, "wrong type: %T", msgV) - r.Equal("%OnSDT0rFoLnUVkp2VoCZ4bAmsTQiI8LbWUTdaE5j9KM=.sha256", osm.Key.Ref()) - r.NoError(from.Close()) - - // do the dance - did, err := UpgradeToMultiMessage(logger, testRepo) - r.NoError(err) - r.True(did) - - migratedLog, err := repo.OpenLog(testRepo) - r.NoError(err) - - migratedSeq, err := migratedLog.Seq().Value() - r.NoError(err) - r.EqualValues(431, migratedSeq) - - r.Equal(1, CurrentVersion(testRepo)) - did, err = UpgradeToMultiMessage(logger, testRepo) - r.NoError(err) - r.False(did) -} diff --git a/repo/migrations/testlog b/repo/migrations/testlog deleted file mode 120000 index 5b81c9bf..00000000 --- a/repo/migrations/testlog +++ /dev/null @@ -1 +0,0 @@ -../../plugins/gossip/testdata/largeRepo/log \ No newline at end of file diff --git a/repo/timestamp_sorter.go b/repo/timestamp_sorter.go new file mode 100644 index 00000000..c769e4fd --- /dev/null +++ b/repo/timestamp_sorter.go @@ -0,0 +1,451 @@ +package repo + +import ( + "context" + "encoding/binary" + "fmt" + "io" + "io/ioutil" + "os" + "sort" + "time" + + bmap "github.com/RoaringBitmap/roaring" + "go.cryptoscope.co/luigi" + "go.cryptoscope.co/margaret" + + refs "go.mindeco.de/ssb-refs" +) + +// SortedSequence holds the sequence value of the message and the domain value it should be sorted by. +type SortedSequence struct { + By int64 // fill with the value of the domain + Seq int64 // the sequence of the entry we are looking for +} + +// SortedSeqSlice a slice of SortedSequences that can be sorted +type SortedSeqSlice []SortedSequence + +func (ts SortedSeqSlice) Len() int { return len(ts) } + +// Swap swaps the elements with indexes i and j. +func (ts SortedSeqSlice) Swap(i int, j int) { + ts[i], ts[j] = ts[j], ts[i] +} + +func (ts SortedSeqSlice) AsLuigiSource() luigi.Source { + return &sortedSource{ + elems: ts, + } +} + +type sortedSource struct{ elems SortedSeqSlice } + +func (ss *sortedSource) Next(_ context.Context) (interface{}, error) { + if len(ss.elems) == 0 { + return nil, luigi.EOS{} + } + next := ss.elems[0] + ss.elems = ss.elems[1:] + return next, nil +} + +// SortedAscending wraps around SortedSeqSlice to give it a Less that sorts values from small to large. +type SortedAscending struct{ SortedSeqSlice } + +// Less sorts values up +func (ts SortedAscending) Less(i int, j int) bool { + vi := ts.SortedSeqSlice[i] + vj := ts.SortedSeqSlice[j] + return vi.By < vj.By +} + +// SortedDescending wraps around SortedSeqSlice to give it a Less that sorts values from large to small. +type SortedDescending struct{ SortedSeqSlice } + +// Less sorts values down +func (ts SortedDescending) Less(i int, j int) bool { + vi := ts.SortedSeqSlice[i] + vj := ts.SortedSeqSlice[j] + return vi.By > vj.By +} + +// SequenceResolver holds three gigantic arrays for each of the understood ResolveDomains. +// +// It should be hooked into a receive log, and filled with Append. +// +// TODO: a better approach might be to fetch these lazyly from disk if they become too large. +// At 1mio messages we roughly look at 8mb per domain. +type SequenceResolver struct { + seq2claimed []int64 + seq2received []int64 + seq2feedseq []int64 + + dirty bool // has not been written to disk yet + repo Interface // where to store the arrays +} + +// NewSequenceResolver opens the stored resolver at r.GetPath("seqmaps") and Loads existing values. +func NewSequenceResolver(r Interface) (*SequenceResolver, error) { + var sr SequenceResolver + sr.repo = r + + n, err := sr.Load() + if err != nil { + return nil, fmt.Errorf("seq resolver: failed to load: %w", err) + } + fmt.Printf("sr: has %d entries\n", n) + + return &sr, nil +} + +// NewSequenceResolverFromLog creates a fresh resolver reading the full margaret log. +// Expects to read refs.Message from the log. +// Useful for testing. +func NewSequenceResolverFromLog(l margaret.Log) (*SequenceResolver, error) { + ctx := context.Background() + + start := time.Now() + + var sr SequenceResolver + + src, err := l.Query() + if err != nil { + return nil, err + } + + for { + v, err := src.Next(ctx) + if err != nil { + if luigi.IsEOS(err) { + break + } + return nil, err + } + + msg, ok := v.(refs.Message) + if !ok { + return nil, fmt.Errorf("ts: wrong type: %T", v) + } + + // TODO: seqWrap and use sr.Append() + sr.seq2claimed = append(sr.seq2claimed, msg.Claimed().Unix()) + sr.seq2received = append(sr.seq2received, msg.Received().Unix()) + sr.seq2feedseq = append(sr.seq2feedseq, msg.Seq()) + } + + took := time.Since(start) + fmt.Println("resolving all claimed time took: ", took) + return &sr, nil +} + +// ResolverFilter get's passed the value from the domain that is searched. +// Should return true if the value should be included and sorted +type ResolverFilter func(int64) bool + +// SortDomain an enum for the understood domains +type SortDomain uint + +// The known domains are: Claimed timestamp, Received Timestamp +// and Sequence number of the messageon the feed (this is important for partial replication, where feeds are not fetched in full and correct order) +const ( + _ SortDomain = iota + SortByClaimed + SortByReceived + SortByFeedSeq +) + +// TODO: maybe some utilities, OTOH it's just generating the seq array +// func (sr SequenceResolver) SortByRange(from, to, by, ok) ... + +func (sr SequenceResolver) prepare(by SortDomain) ([]int64, int64, error) { + err := sr.checkConsistency() + if err != nil { + return nil, -2, err + } + max := int64(len(sr.seq2claimed)) - 1 + // select the array that should be searched + var domain []int64 + switch by { + case SortByClaimed: + domain = sr.seq2claimed + case SortByReceived: + domain = sr.seq2received + case SortByFeedSeq: + domain = sr.seq2feedseq + default: + return nil, -2, fmt.Errorf("seq resolver: invalid domain: %d", by) + } + + return domain, max, nil +} + +func (sr SequenceResolver) SortAndFilterBitmap(seqs *bmap.Bitmap, by SortDomain, ok ResolverFilter, desc bool) (SortedSeqSlice, error) { + domain, max, err := sr.prepare(by) + if err != nil { + return nil, err + } + + var result SortedSeqSlice + + it := seqs.Iterator() + + // pick and filter + for it.HasNext() { + s := int64(it.Next()) + + if s < 0 || s > max { + return nil, fmt.Errorf("seq resolver: out of bounds (%d - %d)", s, max) + } + + sortme := SortedSequence{Seq: s} + + sortme.By = domain[s] + + if ok(sortme.By) { + result = append(result, sortme) + } + } + + if desc { + sort.Sort(SortedDescending{result}) + } else { + sort.Sort(SortedAscending{result}) + } + + return result, nil +} + +func (sr SequenceResolver) SortAndFilterAll(by SortDomain, ok ResolverFilter, desc bool) (SortedSeqSlice, error) { + domain, _, err := sr.prepare(by) + if err != nil { + return nil, err + } + + var result SortedSeqSlice + + // pick and filter + for s := range sr.seq2claimed { // which array doesnt matter since they are all of the same length + sortme := SortedSequence{Seq: int64(s)} + + sortme.By = domain[s] + + if ok(sortme.By) { + result = append(result, sortme) + } + } + + if desc { + sort.Sort(SortedDescending{result}) + } else { + sort.Sort(SortedAscending{result}) + } + + return result, nil +} + +// SortAndFilter goes through seqs in the passed domain using the filter function to include wanted elements. +// desc: true means descending, desc: false means ascending. +func (sr SequenceResolver) SortAndFilter(seqs []int64, by SortDomain, ok ResolverFilter, desc bool) (SortedSeqSlice, error) { + domain, max, err := sr.prepare(by) + if err != nil { + return nil, err + } + + var result SortedSeqSlice + + // pick and filter + for _, s := range seqs { + sortme := SortedSequence{Seq: s} + + if s < 0 || s > max { + return nil, fmt.Errorf("seq resolver: out of bounds (%d - %d)", s, max) + } + + sortme.By = domain[s] + + if ok(sortme.By) { + result = append(result, sortme) + } + } + + if desc { + sort.Sort(SortedDescending{result}) + } else { + sort.Sort(SortedAscending{result}) + } + + return result, nil +} + +func (sr *SequenceResolver) checkConsistency() error { + n := len(sr.seq2claimed) + m := len(sr.seq2received) + o := len(sr.seq2feedseq) + + if n != m { + return fmt.Errorf("seq resolver: consistency error (claimed:%d, received:%d)", n, m) + } + + if n != o { + return fmt.Errorf("seq resolver: consistency error (timestamps:%d, feedseq:%d)", n, o) + } + + return nil +} + +// Append adds all three domains to the resolver. +func (sr *SequenceResolver) Append(seq int64, feed int64, claimed, received time.Time) error { + if err := sr.checkConsistency(); err != nil { + return err + } + + if has := int64(len(sr.seq2claimed)); has != seq { + return fmt.Errorf("seq resolver: would break const (has:%d, will: %d)", has, seq) + } + + sr.seq2claimed = append(sr.seq2claimed, claimed.Unix()) + sr.seq2received = append(sr.seq2received, received.Unix()) + sr.seq2feedseq = append(sr.seq2feedseq, feed) + + sr.dirty = true + return nil +} + +func (sr SequenceResolver) String() string { + return fmt.Sprintf("seq resolver: %d elements", len(sr.seq2claimed)) +} + +// Load reads the files from repo and deserializes them. +func (sr *SequenceResolver) Load() (int64, error) { + if sr.repo == nil { + return -1, fmt.Errorf("seq resolver: not initialized with repo to read from") + } + + var idxes = []struct { + name string + arr *[]int64 + }{ + {"ts-claimed", &sr.seq2claimed}, + {"ts-received", &sr.seq2received}, + {"feed-seqs", &sr.seq2feedseq}, + } + + for _, idx := range idxes { + f, err := os.Open(sr.repo.GetPath("seqmaps", idx.name)) + if err != nil { + if os.IsNotExist(err) { + return 0, nil + } + return -1, fmt.Errorf("seq resolver: failed to create temp file for %s: %w", idx.name, err) + } + + stat, err := f.Stat() + if err != nil { + return -1, fmt.Errorf("seq resolver: failed stat data file %s: %w", idx.name, err) + } + + var arr = make([]int64, stat.Size()/8) + err = binary.Read(f, binary.BigEndian, arr) + if err != nil { + return -1, fmt.Errorf("seq resolver: failed to encode array of %s: %w", idx.name, err) + } + + err = f.Close() + if err != nil { + return -1, fmt.Errorf("seq resolver: failed to close temfile for %s: %w", idx.name, err) + } + + *idx.arr = arr + } + if err := sr.checkConsistency(); err != nil { + return -1, err + } + + return int64(len(sr.seq2claimed)), nil +} + +// Serialize does the reverse from Load. It saves the three domains to disk. +func (sr *SequenceResolver) Serialize() error { + if err := sr.checkConsistency(); err != nil { + return err + } + + os.MkdirAll(sr.repo.GetPath("seqmaps"), 0700) + + var idxes = []struct { + name string + arr []int64 + }{ + {"ts-claimed", sr.seq2claimed}, + {"ts-received", sr.seq2received}, + {"feed-seqs", sr.seq2feedseq}, + } + + for _, idx := range idxes { + f, err := ioutil.TempFile("", idx.name+"-*") + if err != nil { + return fmt.Errorf("seq resolver: failed to create temp file for %s: %w", idx.name, err) + } + + err = binary.Write(f, binary.BigEndian, idx.arr) + if err != nil { + return fmt.Errorf("seq resolver: failed to encode array of %s: %w", idx.name, err) + } + + err = f.Close() + if err != nil { + return fmt.Errorf("seq resolver: failed to close temfile for %s: %w", idx.name, err) + } + + err = moveFile(f.Name(), sr.repo.GetPath("seqmaps", idx.name)) + if err != nil { + return fmt.Errorf("seq resolver: failed to move updated file in place %s: %w", idx.name, err) + } + } + sr.dirty = false + return nil +} + +// Seq returns the number of entries held by the resolver. +func (sr SequenceResolver) Seq() int64 { + err := sr.checkConsistency() + if err != nil { + panic(err) + } + + return int64(len(sr.seq2claimed)) +} + +// Close serialzes the resolver to disk. +func (sr SequenceResolver) Close() error { + return sr.Serialize() +} + +// util + +// os.Rename doesn't work across filesystems. +// /tmp is often on tmpfs or similar. +func moveFile(sourcePath, destPath string) error { + inputFile, err := os.Open(sourcePath) + if err != nil { + return fmt.Errorf("failed to open source file: %s", err) + } + outputFile, err := os.Create(destPath) + if err != nil { + inputFile.Close() + return fmt.Errorf("failed to open dest file: %s", err) + } + defer outputFile.Close() + _, err = io.Copy(outputFile, inputFile) + inputFile.Close() + if err != nil { + return fmt.Errorf("Writing to output file failed: %s", err) + } + // The copy was successful, so now delete the original file + err = os.Remove(sourcePath) + if err != nil { + return fmt.Errorf("Failed removing original file: %s", err) + } + return nil +} diff --git a/repo/timestamp_test.go b/repo/timestamp_test.go new file mode 100644 index 00000000..89a97485 --- /dev/null +++ b/repo/timestamp_test.go @@ -0,0 +1,127 @@ +package repo_test + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.cryptoscope.co/luigi" + "go.cryptoscope.co/margaret/mem" + "go.cryptoscope.co/ssb/repo" + + refs "go.mindeco.de/ssb-refs" +) + +func TestTimestampSorting(t *testing.T) { + r := require.New(t) + a := assert.New(t) + + rxlog := mem.New() + var allSeqs []int64 + + // make log with messages in reverse chronological order + for i := 10000; i > 0; i -= 1000 { + seq, err := rxlog.Append(mkTestMessage(i)) + r.NoError(err) + t.Log(seq, i) + allSeqs = append(allSeqs, seq.Seq()) + } + + sorter, err := repo.NewSequenceResolverFromLog(rxlog) + r.NoError(err, "failed to get sliced array") + + under9000 := func(v int64) bool { + ok := v < 9000 + t.Log(v, ok) + return ok + } + sorted, err := sorter.SortAndFilter(allSeqs, repo.SortByClaimed, under9000, true) + r.NoError(err) + a.Len(sorted, 8) + a.EqualValues(2, sorted[0].Seq) + t.Log(sorted) + + under3000 := func(v int64) bool { + ok := v < 3000 + t.Log(v, ok) + return ok + } + smaller, err := sorter.SortAndFilter(allSeqs, repo.SortByClaimed, under3000, false) + r.NoError(err) + a.Len(smaller, 2) + a.EqualValues(9, smaller[0].Seq) + t.Log(smaller) + + yes := func(_ int64) bool { return true } + empty, err := sorter.SortAndFilter([]int64{10}, repo.SortByReceived, yes, false) + r.Error(err) + a.Nil(empty) + + // luigi.Source for backwards compat with the existing margaret code + src := sorted.AsLuigiSource() + + var elems []interface{} + snk := luigi.NewSliceSink(&elems) + + err = luigi.Pump(context.TODO(), snk, src) + r.NoError(err) + r.Len(elems, 8) + t.Log(elems) +} + +// utils + +var seq int64 + +func mkTestMessage(ts int) refs.Message { + sm := testMessage{ + seq: seq, + claimed: time.Unix(int64(ts), 0), + } + seq++ + return sm +} + +type testMessage struct { + seq int64 + claimed time.Time +} + +func (ts testMessage) Seq() int64 { + return ts.seq +} + +func (ts testMessage) Claimed() time.Time { + return ts.claimed +} + +func (ts testMessage) Received() time.Time { + return time.Now() +} + +func (ts testMessage) Key() *refs.MessageRef { + panic("not implemented") +} + +func (ts testMessage) Previous() *refs.MessageRef { + panic("not implemented") +} + +func (ts testMessage) Author() *refs.FeedRef { + panic("not implemented") +} + +func (ts testMessage) ContentBytes() []byte { + panic("not implemented") +} + +func (ts testMessage) ValueContent() *refs.Value { + panic("not implemented") +} + +func (ts testMessage) ValueContentJSON() json.RawMessage { + panic("not implemented") +} diff --git a/sbot.go b/sbot.go index 70d13f6d..8f004e0b 100644 --- a/sbot.go +++ b/sbot.go @@ -9,6 +9,7 @@ import ( "go.cryptoscope.co/margaret" "go.cryptoscope.co/margaret/multilog" refs "go.mindeco.de/ssb-refs" + "go.mindeco.de/ssb-refs/tfk" ) type Publisher interface { @@ -107,17 +108,12 @@ func FeedsWithSequnce(feedIndex multilog.MultiLog) (luigi.Source, error) { var feedsWithSeqs []interface{} for i, author := range storedFeeds { - var sr refs.StorageRef - err := sr.Unmarshal([]byte(author)) + var sr tfk.Feed + err := sr.UnmarshalBinary([]byte(author)) if err != nil { return nil, errors.Wrapf(err, "feedSrc(%d): invalid storage ref", i) - - } - authorRef, err := sr.FeedRef() - if err != nil { - return nil, errors.Wrapf(err, "feedSrc(%d): stored ref not a feed?", i) - } + authorRef := sr.Feed() subLog, err := feedIndex.Get(author) if err != nil { diff --git a/sbot/blobs_test.go b/sbot/blobs_test.go index 569b6a0d..0e0ec9af 100644 --- a/sbot/blobs_test.go +++ b/sbot/blobs_test.go @@ -55,7 +55,6 @@ func TestBlobsPair(t *testing.T) { // }), WithRepoPath(filepath.Join("testrun", t.Name(), "ali")), WithListenAddr(":0"), - // LateOption(MountPlugin(&bytype.Plugin{}, plugins2.AuthMaster)), ) r.NoError(err) @@ -72,7 +71,6 @@ func TestBlobsPair(t *testing.T) { // }), WithRepoPath(filepath.Join("testrun", t.Name(), "bob")), WithListenAddr(":0"), - // LateOption(MountPlugin(&bytype.Plugin{}, plugins2.AuthMaster)), ) r.NoError(err) botgroup.Go(bs.Serve(bob)) @@ -362,7 +360,6 @@ func TestBlobsWithHops(t *testing.T) { WithInfo(log.With(mainLog, "peer", "ali")), WithRepoPath(filepath.Join("testrun", t.Name(), "ali")), WithListenAddr(":0"), - // LateOption(MountPlugin(&bytype.Plugin{}, plugins2.AuthMaster)), ) r.NoError(err) botgroup.Go(bs.Serve(ali)) @@ -379,7 +376,6 @@ func TestBlobsWithHops(t *testing.T) { // return debug.WrapConn(log.With(mainLog, "remote", addr.String()[1:5]), conn), nil // }), WithListenAddr(":0"), - // LateOption(MountPlugin(&bytype.Plugin{}, plugins2.AuthMaster)), ) r.NoError(err) botgroup.Go(bs.Serve(bob)) @@ -492,7 +488,6 @@ func TestBlobsTooBig(t *testing.T) { // }), WithRepoPath(filepath.Join("testrun", t.Name(), "ali")), WithListenAddr(":0"), - // LateOption(MountPlugin(&bytype.Plugin{}, plugins2.AuthMaster)), ) r.NoError(err) srvBot(ali, "ali") @@ -508,7 +503,6 @@ func TestBlobsTooBig(t *testing.T) { // }), WithRepoPath(filepath.Join("testrun", t.Name(), "bob")), WithListenAddr(":0"), - // LateOption(MountPlugin(&bytype.Plugin{}, plugins2.AuthMaster)), ) r.NoError(err) srvBot(bob, "bob") diff --git a/sbot/feeds_test.go b/sbot/feeds_test.go index 7ab2ee38..f8341843 100644 --- a/sbot/feeds_test.go +++ b/sbot/feeds_test.go @@ -19,6 +19,7 @@ import ( "go.cryptoscope.co/margaret" "go.cryptoscope.co/ssb/internal/leakcheck" + "go.cryptoscope.co/ssb/internal/storedrefs" "go.cryptoscope.co/ssb/internal/testutils" ) @@ -97,7 +98,7 @@ func TestFeedsOneByOne(t *testing.T) { uf, ok := bob.GetMultiLog("userFeeds") r.True(ok) - alisLog, err := uf.Get(ali.KeyPair.Id.StoredAddr()) + alisLog, err := uf.Get(storedrefs.Feed(ali.KeyPair.Id)) r.NoError(err) for i := 0; i < 50; i++ { diff --git a/sbot/fsck.go b/sbot/fsck.go index 387436d5..8e841092 100644 --- a/sbot/fsck.go +++ b/sbot/fsck.go @@ -15,6 +15,7 @@ import ( "go.cryptoscope.co/margaret" "go.cryptoscope.co/margaret/multilog" refs "go.mindeco.de/ssb-refs" + "go.mindeco.de/ssb-refs/tfk" "go.cryptoscope.co/ssb" "go.cryptoscope.co/ssb/multilogs" @@ -125,10 +126,10 @@ func (s *Sbot) FSCK(opts ...FSCKOption) error { switch opt.mode { case FSCKModeLength: - return lengthFSCK(opt.feedsIdx, s.RootLog) + return lengthFSCK(opt.feedsIdx, s.ReceiveLog) case FSCKModeSequences: - return sequenceFSCK(s.RootLog, opt.progressFn) + return sequenceFSCK(s.ReceiveLog, opt.progressFn) default: return errors.New("sbot: unknown fsck mode") @@ -145,12 +146,8 @@ func lengthFSCK(authorMlog multilog.MultiLog, receiveLog margaret.Log) error { } for _, author := range feeds { - var sr refs.StorageRef - err := sr.Unmarshal([]byte(author)) - if err != nil { - return err - } - authorRef, err := sr.FeedRef() + var sr tfk.Feed + err := sr.UnmarshalBinary([]byte(author)) if err != nil { return err } @@ -185,7 +182,7 @@ func lengthFSCK(authorMlog multilog.MultiLog, receiveLog margaret.Log) error { // margaret indexes are 0-based, therefore +1 if msg.Seq() != currentSeqFromIndex.Seq()+1 { return ssb.ErrWrongSequence{ - Ref: authorRef, + Ref: sr.Feed(), Stored: currentSeqFromIndex, Logical: msg, } @@ -361,7 +358,7 @@ func (s *Sbot) HealRepo(report ErrConsistencyProblems) error { it := report.Sequences.Iterator() for it.HasNext() { seq := it.Next() - err := s.RootLog.Null(margaret.BaseSeq(seq)) + err := s.ReceiveLog.Null(margaret.BaseSeq(seq)) if err != nil { return errors.Wrapf(err, "failed to null message (%d) in receive log", seq) } diff --git a/sbot/fsck_test.go b/sbot/fsck_test.go index 43fe5847..cdf1ae00 100644 --- a/sbot/fsck_test.go +++ b/sbot/fsck_test.go @@ -95,7 +95,7 @@ func testFSCKdouble(t *testing.T) { // now do some nasty magic, double the log by appending it to itself again // TODO: refactor to only have Add() on the bot, not the internal rootlog // Add() should do the append logic - src, err := theBot.RootLog.Query(margaret.Limit(n)) + src, err := theBot.ReceiveLog.Query(margaret.Limit(n)) r.NoError(err) for { @@ -107,13 +107,13 @@ func testFSCKdouble(t *testing.T) { r.NoError(err) } - seq, err := theBot.RootLog.Append(v) + seq, err := theBot.ReceiveLog.Append(v) r.NoError(err) t.Log("doubled:", seq.Seq()) } // check duplication - seqv, err := theBot.RootLog.Seq().Value() + seqv, err := theBot.ReceiveLog.Seq().Value() r.NoError(err) seq := seqv.(margaret.Seq) r.EqualValues(seq.Seq()+1, n*2) @@ -185,7 +185,7 @@ func testFSCKmultipleFeeds(t *testing.T) { } // copy the messages from one and two (leaving "main" intact) - src, err := theBot.RootLog.Query( + src, err := theBot.ReceiveLog.Query( margaret.Gt(margaret.BaseSeq(n-1)), margaret.Limit(5)) r.NoError(err) @@ -201,7 +201,7 @@ func testFSCKmultipleFeeds(t *testing.T) { msg, ok := v.(refs.Message) r.True(ok) - seq, err := theBot.RootLog.Append(v) + seq, err := theBot.ReceiveLog.Append(v) r.NoError(err) t.Log("doubled:", msg.Author().ShortRef(), seq.Seq()) } @@ -248,7 +248,7 @@ func testFSCKrepro(t *testing.T) { theBot, _ := makeTestBot(t) - seqV, err := theBot.RootLog.Seq().Value() + seqV, err := theBot.ReceiveLog.Seq().Value() r.NoError(err) latestSeq := seqV.(margaret.Seq) r.EqualValues(latestSeq.Seq(), 6699) diff --git a/sbot/gabby_test.go b/sbot/gabby_test.go index 5f533255..6ab42310 100644 --- a/sbot/gabby_test.go +++ b/sbot/gabby_test.go @@ -11,6 +11,7 @@ import ( "time" "go.cryptoscope.co/ssb/internal/leakcheck" + "go.cryptoscope.co/ssb/internal/storedrefs" "go.cryptoscope.co/ssb/internal/testutils" refs "go.mindeco.de/ssb-refs" "golang.org/x/sync/errgroup" @@ -88,7 +89,7 @@ func TestFeedsGabbySync(t *testing.T) { // sanity, check bob has his shit together uf, ok := bob.GetMultiLog("userFeeds") r.True(ok) - bobsOwnLog, err := uf.Get(bob.KeyPair.Id.StoredAddr()) + bobsOwnLog, err := uf.Get(storedrefs.Feed(bob.KeyPair.Id)) r.NoError(err) seqv, err := bobsOwnLog.Seq().Value() @@ -107,14 +108,14 @@ func TestFeedsGabbySync(t *testing.T) { // check that bobs messages got to ali auf, ok := ali.GetMultiLog("userFeeds") r.True(ok) - bosLogAtAli, err := auf.Get(bob.KeyPair.Id.StoredAddr()) + bosLogAtAli, err := auf.Get(storedrefs.Feed(bob.KeyPair.Id)) r.NoError(err) seqv, err = bosLogAtAli.Seq().Value() r.NoError(err) r.Equal(margaret.BaseSeq(8), seqv) - src, err := mutil.Indirect(ali.RootLog, bosLogAtAli).Query() + src, err := mutil.Indirect(ali.ReceiveLog, bosLogAtAli).Query() r.NoError(err) for { v, err := src.Next(ctx) diff --git a/sbot/get.go b/sbot/get.go index 83094e56..3d308a64 100644 --- a/sbot/get.go +++ b/sbot/get.go @@ -37,7 +37,7 @@ func (s Sbot) Get(ref refs.MessageRef) (refs.Message, error) { return nil, errors.Errorf("sbot/get: wrong sequence type in index: %T", v) } - storedV, err := s.RootLog.Get(seq) + storedV, err := s.ReceiveLog.Get(seq) if err != nil { return nil, errors.Wrap(err, "sbot/get: failed to load message") } diff --git a/sbot/identities.go b/sbot/identities.go index 16111ec0..4d4770ed 100644 --- a/sbot/identities.go +++ b/sbot/identities.go @@ -31,7 +31,7 @@ func (sbot *Sbot) PublishAs(nick string, val interface{}) (*refs.MessageRef, err pubopts = append(pubopts, message.SetHMACKey(sbot.signHMACsecret)) } - pl, err := message.OpenPublishLog(sbot.RootLog, uf, kp, pubopts...) + pl, err := message.OpenPublishLog(sbot.ReceiveLog, uf, kp, pubopts...) if err != nil { return nil, errors.Wrap(err, "publishAs: failed to create publish log") } diff --git a/sbot/identities_test.go b/sbot/identities_test.go index 03c319a3..8f1508ce 100644 --- a/sbot/identities_test.go +++ b/sbot/identities_test.go @@ -17,8 +17,8 @@ import ( refs "go.mindeco.de/ssb-refs" "go.cryptoscope.co/ssb" - "go.cryptoscope.co/ssb/indexes" "go.cryptoscope.co/ssb/internal/leakcheck" + "go.cryptoscope.co/ssb/internal/storedrefs" "go.cryptoscope.co/ssb/private/box" "go.cryptoscope.co/ssb/repo" ) @@ -67,7 +67,6 @@ func XTestMultipleIdentities(t *testing.T) { WithInfo(logger), WithRepoPath(tRepoPath), WithHMACSigning(hk), - LateOption(MountSimpleIndex("get", indexes.OpenGet)), DisableNetworkNode(), ) r.NoError(err) @@ -121,9 +120,9 @@ func XTestMultipleIdentities(t *testing.T) { r.EqualValues(seq, v.(margaret.Seq).Seq()) } - checkLogSeq(mainbot.RootLog, len(intros)-1) // got all the messages + checkLogSeq(mainbot.ReceiveLog, len(intros)-1) // got all the messages - src, err := mainbot.RootLog.Query() + src, err := mainbot.ReceiveLog.Query() r.NoError(err) ctx := context.Background() @@ -147,11 +146,11 @@ func XTestMultipleIdentities(t *testing.T) { pl, ok := mainbot.GetMultiLog("privLogs") r.True(ok, "no privLogs") - arnies, err := pl.Get(kpArny.Id.StoredAddr()) + arnies, err := pl.Get(storedrefs.Feed(kpArny.Id)) r.NoError(err) - berts, err := pl.Get(kpBert.Id.StoredAddr()) + berts, err := pl.Get(storedrefs.Feed(kpBert.Id)) r.NoError(err) - cloes, err := pl.Get(kpCloe.Id.StoredAddr()) + cloes, err := pl.Get(storedrefs.Feed(kpCloe.Id)) r.NoError(err) // 0 indexed diff --git a/sbot/indicies.go b/sbot/indicies.go index 457020ec..672a2e47 100644 --- a/sbot/indicies.go +++ b/sbot/indicies.go @@ -24,7 +24,7 @@ import ( func MountPlugin(plug ssb.Plugin, mode plugins2.AuthMode) Option { return func(s *Sbot) error { if wrl, ok := plug.(plugins2.NeedsRootLog); ok { - wrl.WantRootLog(s.RootLog) + wrl.WantRootLog(s.ReceiveLog) } if wrl, ok := plug.(plugins2.NeedsMultiLog); ok { @@ -123,14 +123,14 @@ func (s *Sbot) WaitUntilIndexesAreSynced() { // the default is to fill an index with all messages func (s *Sbot) serveIndex(name string, snk librarian.SinkIndex) { - s.serveIndexFrom(name, snk, s.RootLog) + s.serveIndexFrom(name, snk, s.ReceiveLog) } /* some indexes just require a certain kind of message, like type:contact or type:about. contactLog, err := s.ByType.Get(librarian.Addr("contact")) if err != nil { ... } -msgs := mutil.Indirect(s.RootLog, contactLog) +msgs := mutil.Indirect(s.ReceiveLog, contactLog) */ func (s *Sbot) serveIndexFrom(name string, snk librarian.SinkIndex, msgs margaret.Log) { s.idxInSync.Add(1) diff --git a/sbot/manifest.go b/sbot/manifest.go index 24cae91a..c429bd63 100644 --- a/sbot/manifest.go +++ b/sbot/manifest.go @@ -37,22 +37,11 @@ func (h manifestHandler) HandleCall(ctx context.Context, req *muxrpc.Request, ed // this is a very simple hardcoded manifest.json dump which oasis' ssb-client expects to do it's magic. const manifestBlob = ` { - "auth": "async", - "address": "sync", "manifest": "sync", - "multiserverNet": {}, "get": "async", "createFeedStream": "source", "createUserStream": "source", - "createWriteStream": "sink", - "links": "source", - - "add": "async", - - "getLatest": "async", - "latest": "source", - "latestSequence": "async", "createSequenceStream": "source", "createLogStream": "source", @@ -68,10 +57,13 @@ const manifestBlob = ` "private": { -"read":"source" + "read":"source" }, - "tangles": "source", + "tangles": { + "replies": "source" + }, + "names": { "get": "async", "getImageFor": "async", @@ -94,6 +86,11 @@ const manifestBlob = ` "upto": "source" }, + "groups": { + "create":"async", + "publishTo":"async" + }, + "blobs": { "get": "source", diff --git a/sbot/manifest_test.go b/sbot/manifest_test.go new file mode 100644 index 00000000..0c46c401 --- /dev/null +++ b/sbot/manifest_test.go @@ -0,0 +1,14 @@ +package sbot + +import ( + "encoding/json" + "testing" +) + +func TestManifest(t *testing.T) { + var manifestObj map[string]interface{} + err := json.Unmarshal(json.RawMessage(manifestBlob), &manifestObj) + if err != nil { + t.Error(err) + } +} diff --git a/sbot/new.go b/sbot/new.go index 381fbaa0..a65ee096 100644 --- a/sbot/new.go +++ b/sbot/new.go @@ -3,6 +3,7 @@ package sbot import ( + "fmt" "io" "net" "net/http" @@ -17,6 +18,7 @@ import ( "go.cryptoscope.co/librarian" libmkv "go.cryptoscope.co/librarian/mkv" "go.cryptoscope.co/margaret/multilog/roaring" + multifs "go.cryptoscope.co/margaret/multilog/roaring/fs" "go.cryptoscope.co/muxrpc" "go.cryptoscope.co/ssb" @@ -25,7 +27,7 @@ import ( "go.cryptoscope.co/ssb/indexes" "go.cryptoscope.co/ssb/internal/ctxutils" "go.cryptoscope.co/ssb/internal/mutil" - "go.cryptoscope.co/ssb/keys" + "go.cryptoscope.co/ssb/internal/storedrefs" "go.cryptoscope.co/ssb/message" "go.cryptoscope.co/ssb/multilogs" "go.cryptoscope.co/ssb/network" @@ -34,6 +36,7 @@ import ( "go.cryptoscope.co/ssb/plugins/friends" "go.cryptoscope.co/ssb/plugins/get" "go.cryptoscope.co/ssb/plugins/gossip" + "go.cryptoscope.co/ssb/plugins/groups" "go.cryptoscope.co/ssb/plugins/legacyinvites" "go.cryptoscope.co/ssb/plugins/partial" privplug "go.cryptoscope.co/ssb/plugins/private" @@ -41,17 +44,16 @@ import ( "go.cryptoscope.co/ssb/plugins/rawread" "go.cryptoscope.co/ssb/plugins/replicate" "go.cryptoscope.co/ssb/plugins/status" + "go.cryptoscope.co/ssb/plugins/tangles" "go.cryptoscope.co/ssb/plugins/whoami" - "go.cryptoscope.co/ssb/plugins2/bytype" "go.cryptoscope.co/ssb/plugins2/names" - "go.cryptoscope.co/ssb/plugins2/tangles" "go.cryptoscope.co/ssb/private" + "go.cryptoscope.co/ssb/private/keys" "go.cryptoscope.co/ssb/repo" refs "go.mindeco.de/ssb-refs" - - multifs "go.cryptoscope.co/margaret/multilog/roaring/fs" ) +// Close closes the bot by stopping network connections and closing the internal databases func (s *Sbot) Close() error { s.closedMu.Lock() defer s.closedMu.Unlock() @@ -97,11 +99,11 @@ func initSbot(s *Sbot) (*Sbot, error) { r := repo.New(s.repoPath) // optionize?! - s.RootLog, err = repo.OpenLog(r) + s.ReceiveLog, err = repo.OpenLog(r) if err != nil { return nil, errors.Wrap(err, "sbot: failed to open rootlog") } - s.closers.addCloser(s.RootLog.(io.Closer)) + s.closers.addCloser(s.ReceiveLog.(io.Closer)) if s.BlobStore == nil { // load default, local file blob store s.BlobStore, err = repo.OpenBlobStore(r) @@ -126,6 +128,13 @@ func initSbot(s *Sbot) (*Sbot, error) { } } + s.SeqResolver, err = repo.NewSequenceResolver(r) + if err != nil { + return nil, errors.Wrap(err, "error opening sequence resolver") + } + s.closers.addCloser(s.SeqResolver) + + // default multilogs var mlogs = []struct { Name string Mlog **roaring.MultiLog @@ -152,6 +161,23 @@ func initSbot(s *Sbot) (*Sbot, error) { *index.Mlog = ml } + // publish + var pubopts = []message.PublishOption{ + message.UseNowTimestamps(true), + } + if s.signHMACsecret != nil { + pubopts = append(pubopts, message.SetHMACKey(s.signHMACsecret)) + } + s.PublishLog, err = message.OpenPublishLog(s.ReceiveLog, s.Users, s.KeyPair, pubopts...) + if err != nil { + return nil, errors.Wrap(err, "sbot: failed to create publish log") + } + + err = MountSimpleIndex("get", indexes.OpenGet)(s) + if err != nil { + return nil, err + } + // groups2 pth := r.GetPath(repo.PrefixIndex, "groups-keys", "mkv") err = os.MkdirAll(pth, 0700) @@ -170,12 +196,13 @@ func initSbot(s *Sbot) (*Sbot, error) { } s.closers.addCloser(idx) - s.Groups = private.NewManager(s.KeyPair, s.PublishLog, ks, s.Tangles) + s.Groups = private.NewManager(s.KeyPair, s.PublishLog, ks, s.ReceiveLog, s, s.Tangles) combIdx, err := multilogs.NewCombinedIndex( s.repoPath, s.Groups, s.KeyPair.Id, + s.SeqResolver, s.Users, s.Private, s.ByType, @@ -184,12 +211,13 @@ func initSbot(s *Sbot) (*Sbot, error) { return nil, errors.Wrap(err, "sbot: failed to open combined application index") } s.serveIndex("combined", combIdx) + s.closers.addCloser(combIdx) /* TODO: fix deadlock in index update locking if _, ok := s.simpleIndex["content-delete-requests"]; !ok { var dcrTrigger dropContentTrigger dcrTrigger.logger = kitlog.With(log, "module", "dcrTrigger") - dcrTrigger.root = s.RootLog + dcrTrigger.root = s.ReceiveLog dcrTrigger.feeds = uf dcrTrigger.nuller = s err = MountSimpleIndex("content-delete-requests", dcrTrigger.MakeSimpleIndex)(s) @@ -199,24 +227,12 @@ func initSbot(s *Sbot) (*Sbot, error) { } */ - // publish - var pubopts = []message.PublishOption{ - message.UseNowTimestamps(true), - } - if s.signHMACsecret != nil { - pubopts = append(pubopts, message.SetHMACKey(s.signHMACsecret)) - } - s.PublishLog, err = message.OpenPublishLog(s.RootLog, s.Users, s.KeyPair, pubopts...) - if err != nil { - return nil, errors.Wrap(err, "sbot: failed to create publish log") - } - // contact/follow graph - contactLog, err := s.ByType.Get(librarian.Addr("contact")) + contactLog, err := s.ByType.Get(librarian.Addr("string:contact")) if err != nil { return nil, errors.Wrap(err, "sbot: failed to open message contact sublog") } - justContacts := mutil.Indirect(s.RootLog, contactLog) + justContacts := mutil.Indirect(s.ReceiveLog, contactLog) // LogBuilder doesn't fully work yet if false { @@ -238,11 +254,11 @@ func initSbot(s *Sbot) (*Sbot, error) { } // abouts - aboutSeqs, err := s.ByType.Get(librarian.Addr("about")) + aboutSeqs, err := s.ByType.Get(librarian.Addr("string:about")) if err != nil { return nil, errors.Wrap(err, "sbot: failed to open message about sublog") } - aboutsOnly := mutil.Indirect(s.RootLog, aboutSeqs) + aboutsOnly := mutil.Indirect(s.ReceiveLog, aboutSeqs) var namesPlug names.Plugin _, aboutSnk, err := namesPlug.MakeSimpleIndex(r) @@ -268,7 +284,7 @@ func initSbot(s *Sbot) (*Sbot, error) { // TODO: make plugabble // var peerPlug *peerinvites.Plugin // if mt, ok := s.mlogIndicies[multilogs.IndexNameFeeds]; ok { - // peerPlug = peerinvites.New(kitlog.With(log, "plugin", "peerInvites"), s, mt, s.RootLog, s.PublishLog) + // peerPlug = peerinvites.New(kitlog.With(log, "plugin", "peerInvites"), s, mt, s.ReceiveLog, s.PublishLog) // s.public.Register(peerPlug) // _, peerServ, err := peerPlug.OpenIndex(r) // if err != nil { @@ -341,18 +357,12 @@ func initSbot(s *Sbot) (*Sbot, error) { return nil, err } - // - err = MountSimpleIndex("get", indexes.OpenGet)(s) - if err != nil { - return nil, err - } - // publish - s.master.Register(publish.NewPlug(kitlog.With(log, "plugin", "publish"), s.PublishLog, s.RootLog)) + s.master.Register(publish.NewPlug(kitlog.With(log, "plugin", "publish"), s.PublishLog, s.ReceiveLog)) // private // TODO: box2 - userPrivs, err := s.Private.Get(librarian.Addr("box1:") + s.KeyPair.Id.StoredAddr()) + userPrivs, err := s.Private.Get(librarian.Addr("box1:") + storedrefs.Feed(s.KeyPair.Id)) if err != nil { return nil, errors.Wrap(err, "failed to open user private index") } @@ -361,7 +371,7 @@ func initSbot(s *Sbot) (*Sbot, error) { s.KeyPair.Id, s.Groups, s.PublishLog, - private.NewUnboxerLog(s.RootLog, userPrivs, s.KeyPair))) + private.NewUnboxerLog(s.ReceiveLog, userPrivs, s.KeyPair))) // whoami whoami := whoami.New(kitlog.With(log, "plugin", "whoami"), s.KeyPair.Id) @@ -395,7 +405,7 @@ func initSbot(s *Sbot) (*Sbot, error) { fm := gossip.NewFeedManager( ctx, - s.RootLog, + s.ReceiveLog, s.Users, kitlog.With(log, "feedmanager"), s.systemGauge, @@ -403,21 +413,21 @@ func initSbot(s *Sbot) (*Sbot, error) { ) s.public.Register(gossip.New(ctx, kitlog.With(log, "plugin", "gossip"), - s.KeyPair.Id, s.RootLog, s.Users, fm, s.Replicator.Lister(), + s.KeyPair.Id, s.ReceiveLog, s.Users, fm, s.Replicator.Lister(), histOpts...)) // incoming createHistoryStream handler hist := gossip.NewHist(ctx, kitlog.With(log, "plugin", "gossip/hist"), s.KeyPair.Id, - s.RootLog, s.Users, + s.ReceiveLog, s.Users, s.Replicator.Lister(), fm, histOpts...) s.public.Register(hist) // get idx muxrpc handler - s.master.Register(get.New(s, s.RootLog)) + s.master.Register(get.New(s, s.ReceiveLog)) // s.master.Register(namesPlug) @@ -428,14 +438,28 @@ func initSbot(s *Sbot) (*Sbot, error) { s.Users, s.ByType, s.Tangles, - s.RootLog, s) + s.ReceiveLog, s) s.public.Register(plug) s.master.Register(plug) + // group managment + s.master.Register(groups.New(s.info, s.Groups)) + // raw log plugins - s.master.Register(rawread.NewSequenceStream(s.RootLog)) - s.master.Register(rawread.NewRXLog(s.RootLog)) // createLogStream - s.master.Register(hist) // createHistoryStream + + sc := selfChecker{*s.KeyPair.Id} + s.master.Register(rawread.NewByTypePlugin( + s.info, + s.ReceiveLog, + s.ByType, + s.Private, + s.Groups, + s.SeqResolver, + sc)) + s.master.Register(rawread.NewSequenceStream(s.ReceiveLog)) + s.master.Register(rawread.NewRXLog(s.ReceiveLog)) // createLogStream + s.master.Register(rawread.NewSortedStream(s.info, s.ReceiveLog, s.SeqResolver)) // createLogStream + s.master.Register(hist) // createHistoryStream s.master.Register(replicate.NewPlug(s.Users)) @@ -446,14 +470,7 @@ func initSbot(s *Sbot) (*Sbot, error) { name: "manifest"} s.master.Register(mh) - var bytypePlug bytype.Plugin - bytypePlug.WantRootLog(s.RootLog) - bytypePlug.UseMultiLog(s.ByType) - s.master.Register(bytypePlug) - - var tplug tangles.Plugin - tplug.WantRootLog(s.RootLog) - tplug.UseMultiLog(s.Tangles) + var tplug = tangles.NewPlugin(s.ReceiveLog, s.Tangles, s.Private, s.Groups, sc) s.master.Register(tplug) // tcp+shs @@ -521,7 +538,7 @@ func initSbot(s *Sbot) (*Sbot, error) { s.KeyPair.Id, s.Network, s.PublishLog, - s.RootLog, + s.ReceiveLog, ) if err != nil { return nil, errors.Wrap(err, "sbot: failed to open legacy invites plugin") @@ -534,3 +551,14 @@ func initSbot(s *Sbot) (*Sbot, error) { return s, nil } + +type selfChecker struct { + me refs.FeedRef +} + +func (sc selfChecker) Authorize(remote *refs.FeedRef) error { + if sc.me.Equal(remote) { + return nil + } + return fmt.Errorf("not authorized") +} diff --git a/sbot/null.go b/sbot/null.go index 3922c2af..f62d07f4 100644 --- a/sbot/null.go +++ b/sbot/null.go @@ -16,6 +16,7 @@ import ( refs "go.mindeco.de/ssb-refs" "go.cryptoscope.co/ssb/indexes" + "go.cryptoscope.co/ssb/internal/storedrefs" "go.cryptoscope.co/ssb/multilogs" "go.cryptoscope.co/ssb/repo" ) @@ -29,7 +30,7 @@ func (s *Sbot) NullFeed(ref *refs.FeedRef) error { return errors.Errorf("NullFeed: failed to open multilog") } - feedAddr := ref.StoredAddr() + feedAddr := storedrefs.Feed(ref) userSeqs, err := uf.Get(feedAddr) if err != nil { return errors.Wrap(err, "NullFeed: failed to open log for feed argument") @@ -52,7 +53,7 @@ func (s *Sbot) NullFeed(ref *refs.FeedRef) error { if !ok { return errors.Errorf("NullFeed: not a sequence from userlog query") } - err = s.RootLog.Null(seq) + err = s.ReceiveLog.Null(seq) if err != nil { return err } diff --git a/sbot/null_content.go b/sbot/null_content.go index 1e10c283..e4e92ce6 100644 --- a/sbot/null_content.go +++ b/sbot/null_content.go @@ -14,6 +14,7 @@ import ( "go.cryptoscope.co/ssb" "go.cryptoscope.co/ssb/internal/mutil" + "go.cryptoscope.co/ssb/internal/storedrefs" "go.cryptoscope.co/ssb/message/multimsg" "go.cryptoscope.co/ssb/multilogs" "go.cryptoscope.co/ssb/repo" @@ -31,7 +32,7 @@ func (s *Sbot) NullContent(fr *refs.FeedRef, seq uint) error { return errors.Errorf("userFeeds mlog not present") } - userLog, err := uf.Get(fr.StoredAddr()) + userLog, err := uf.Get(storedrefs.Feed(fr)) if err != nil { return errors.Wrap(err, "nullContent: unable to load feed") } @@ -47,7 +48,7 @@ func (s *Sbot) NullContent(fr *refs.FeedRef, seq uint) error { return errors.Errorf("not a sequence type: %T", seqv) } - msgv, err := s.RootLog.Get(rootLogSeq) + msgv, err := s.ReceiveLog.Get(rootLogSeq) if err != nil { return errors.Wrap(err, "nullContent: failed to get message in rootLog") } @@ -69,7 +70,7 @@ func (s *Sbot) NullContent(fr *refs.FeedRef, seq uint) error { return errors.Wrap(err, "nullContent: unable to marshall nulled content transfer") } - err = s.RootLog.Replace(rootLogSeq, nulled) + err = s.ReceiveLog.Replace(rootLogSeq, nulled) return errors.Wrap(err, "nullContent: failed to execute replace operation") } @@ -95,7 +96,7 @@ func (cdr *dropContentTrigger) consume() { evtLog := kitlog.With(cdr.logger, "event", "null content trigger") for evt := range cdr.check { - feed, err := cdr.feeds.Get(evt.author.StoredAddr()) + feed, err := cdr.feeds.Get(storedrefs.Feed(evt.author)) if err != nil { level.Warn(evtLog).Log("msg", "no such feed?", "err", err) continue diff --git a/sbot/null_content_test.go b/sbot/null_content_test.go index 707a6e9f..c46935d4 100644 --- a/sbot/null_content_test.go +++ b/sbot/null_content_test.go @@ -20,9 +20,9 @@ import ( "golang.org/x/sync/errgroup" "go.cryptoscope.co/ssb" - "go.cryptoscope.co/ssb/indexes" "go.cryptoscope.co/ssb/internal/leakcheck" "go.cryptoscope.co/ssb/internal/mutil" + "go.cryptoscope.co/ssb/internal/storedrefs" "go.cryptoscope.co/ssb/internal/testutils" "go.cryptoscope.co/ssb/repo" ) @@ -66,7 +66,6 @@ func XTestNullContentRequest(t *testing.T) { WithInfo(logger), WithRepoPath(tRepoPath), WithHMACSigning(hk), - LateOption(MountSimpleIndex("get", indexes.OpenGet)), DisableNetworkNode(), ) r.NoError(err) @@ -114,13 +113,13 @@ func XTestNullContentRequest(t *testing.T) { uf, ok := bot.GetMultiLog("userFeeds") r.True(ok, "userFeeds mlog not present") - l, err := uf.Get(kp.Id.StoredAddr()) + l, err := uf.Get(storedrefs.Feed(kp.Id)) r.NoError(err) checkLogSeq(l, seq) } - checkLogSeq(mainbot.RootLog, len(intros)-1) // got all the messages + checkLogSeq(mainbot.ReceiveLog, len(intros)-1) // got all the messages // check before drop checkUserLogSeq(mainbot, "arny", 2) @@ -131,18 +130,18 @@ func XTestNullContentRequest(t *testing.T) { r.True(ok, "userFeeds mlog not present") // try to request on arnies feed fails because the formt doesn't support it - arniesLog, err := uf.Get(kpArny.Id.StoredAddr()) + arniesLog, err := uf.Get(storedrefs.Feed(kpArny.Id)) r.NoError(err) - arniesLog = mutil.Indirect(mainbot.RootLog, arniesLog) + arniesLog = mutil.Indirect(mainbot.ReceiveLog, arniesLog) dcr := refs.NewDropContentRequest(1, allMessages[0]) r.False(dcr.Valid(arniesLog)) // bert is in gg format so it works - bertLog, err := uf.Get(kpBert.Id.StoredAddr()) + bertLog, err := uf.Get(storedrefs.Feed(kpBert.Id)) r.NoError(err) - bertLog = mutil.Indirect(mainbot.RootLog, bertLog) + bertLog = mutil.Indirect(mainbot.ReceiveLog, bertLog) msgv, err := bertLog.Get(margaret.BaseSeq(2)) // 0-indexed r.NoError(err) msg, ok := msgv.(refs.Message) @@ -264,7 +263,7 @@ func XTestNullContentAndSync(t *testing.T) { uf, ok := bot.GetMultiLog("userFeeds") r.True(ok, "userFeeds mlog not present") - l, err := uf.Get(kp.Id.StoredAddr()) + l, err := uf.Get(storedrefs.Feed(kp.Id)) r.NoError(err) v, err := l.Seq().Value() @@ -279,10 +278,10 @@ func XTestNullContentAndSync(t *testing.T) { uf, ok := bot.GetMultiLog("userFeeds") r.True(ok, "userFeeds mlog not present") - userLog, err := uf.Get(kp.Id.StoredAddr()) + userLog, err := uf.Get(storedrefs.Feed(kp.Id)) r.NoError(err) - msgv, err := mutil.Indirect(bot.RootLog, userLog).Get(margaret.BaseSeq(seq - 1)) // 0-indexed + msgv, err := mutil.Indirect(bot.ReceiveLog, userLog).Get(margaret.BaseSeq(seq - 1)) // 0-indexed r.NoError(err) msg, ok := msgv.(refs.Message) @@ -396,7 +395,7 @@ func XTestNullContentAndSync(t *testing.T) { t.Log(sw.Seq().Seq(), string(msg.ContentBytes())) return err }) - src, err := otherBot.RootLog.Query(margaret.SeqWrap(true)) + src, err := otherBot.ReceiveLog.Query(margaret.SeqWrap(true)) r.NoError(err) luigi.Pump(context.Background(), printSink, src) diff --git a/sbot/null_feed_test.go b/sbot/null_feed_test.go index 02111685..8c007661 100644 --- a/sbot/null_feed_test.go +++ b/sbot/null_feed_test.go @@ -18,8 +18,8 @@ import ( "golang.org/x/sync/errgroup" "go.cryptoscope.co/ssb" - "go.cryptoscope.co/ssb/indexes" "go.cryptoscope.co/ssb/internal/leakcheck" + "go.cryptoscope.co/ssb/internal/storedrefs" "go.cryptoscope.co/ssb/internal/testutils" "go.cryptoscope.co/ssb/repo" ) @@ -63,7 +63,6 @@ func TestNullFeed(t *testing.T) { WithRepoPath(filepath.Join(tRepoPath, "main")), WithHops(2), WithHMACSigning(hk), - LateOption(MountSimpleIndex("get", indexes.OpenGet)), WithListenAddr(":0"), ) r.NoError(err) @@ -106,7 +105,7 @@ func TestNullFeed(t *testing.T) { uf, ok := bot.GetMultiLog("userFeeds") r.True(ok, "userFeeds mlog not present") - l, err := uf.Get(kp.Id.StoredAddr()) + l, err := uf.Get(storedrefs.Feed(kp.Id)) r.NoError(err) return l @@ -118,7 +117,7 @@ func TestNullFeed(t *testing.T) { checkLogSeq(l, seq) } - checkLogSeq(mainbot.RootLog, len(intros)-1) // got all the messages + checkLogSeq(mainbot.ReceiveLog, len(intros)-1) // got all the messages // check before drop checkUserLogSeq(mainbot, "arny", 1) @@ -256,7 +255,7 @@ func TestNullFetched(t *testing.T) { aliUF, ok := ali.GetMultiLog("userFeeds") r.True(ok) - alisVersionOfBobsLog, err := aliUF.Get(bob.KeyPair.Id.StoredAddr()) + alisVersionOfBobsLog, err := aliUF.Get(storedrefs.Feed(bob.KeyPair.Id)) r.NoError(err) mainLog.Log("msg", "check we got all the messages") @@ -289,7 +288,7 @@ func TestNullFetched(t *testing.T) { r.NoError(err) mainLog.Log("msg", "get a fresh view (shoild be empty now)") - alisVersionOfBobsLog, err = aliUF.Get(bob.KeyPair.Id.StoredAddr()) + alisVersionOfBobsLog, err = aliUF.Get(storedrefs.Feed(bob.KeyPair.Id)) r.NoError(err) mainLog.Log("msg", "sync should give us the messages again") diff --git a/sbot/options.go b/sbot/options.go index 2722b9f4..a433504d 100644 --- a/sbot/options.go +++ b/sbot/options.go @@ -36,8 +36,10 @@ import ( "go.cryptoscope.co/ssb/repo" ) +// MuxrpcEndpointWrapper can be used to wrap ever call a endpoint makes type MuxrpcEndpointWrapper func(muxrpc.Endpoint) muxrpc.Endpoint +// Sbot is the database and replication server type Sbot struct { info kitlog.Logger @@ -85,7 +87,9 @@ type Sbot struct { Groups *private.Manager - RootLog multimsg.AlterableLog // aka receive log (the stream of messages as it arrived) + ReceiveLog multimsg.AlterableLog // the stream of messages as they arrived + + SeqResolver *repo.SequenceResolver PublishLog ssb.Publisher signHMACsecret []byte @@ -117,8 +121,10 @@ type Sbot struct { ssb.Replicator } +// Option is a functional option type definition to change sbot behaviour type Option func(*Sbot) error +// WithBlobStore can be used to use a different storage backend for blobs. func WithBlobStore(bs ssb.BlobStore) Option { return func(s *Sbot) error { s.BlobStore = bs @@ -135,6 +141,7 @@ func DisableLiveIndexMode() Option { } } +// WithRepoPath changes where the replication database and blobs are stored. func WithRepoPath(path string) Option { return func(s *Sbot) error { s.repoPath = path @@ -142,6 +149,7 @@ func WithRepoPath(path string) Option { } } +// DisableNetworkNode disables all networking, in turn it only serves the database. func DisableNetworkNode() Option { return func(s *Sbot) error { s.disableNetwork = true @@ -149,6 +157,7 @@ func DisableNetworkNode() Option { } } +// WithListenAddr changes the muxrpc listener address. By default it listens to ':8008'. func WithListenAddr(addr string) Option { return func(s *Sbot) error { var err error @@ -157,6 +166,8 @@ func WithListenAddr(addr string) Option { } } +// WithDialer changes the function that is used to dial remote peers. +// This could be a sock5 connection builder to support tor proxying to hidden services. func WithDialer(dial netwrap.Dialer) Option { return func(s *Sbot) error { s.dialer = dial @@ -164,6 +175,7 @@ func WithDialer(dial netwrap.Dialer) Option { } } +// WithNetworkConnTracker changes the connection tracker. See network.NewLastWinsTracker and network.NewAcceptAllTracker. func WithNetworkConnTracker(ct ssb.ConnTracker) Option { return func(s *Sbot) error { s.networkConnTracker = ct @@ -171,6 +183,8 @@ func WithNetworkConnTracker(ct ssb.ConnTracker) Option { } } +// WithUNIXSocket enables listening for muxrpc connections on a unix socket files ($repo/socket). +// This socket is not encrypted or authenticated since access to it is mediated by filesystem ownership. func WithUNIXSocket() Option { return func(s *Sbot) error { // this races because sbot might not be done with init yet @@ -259,6 +273,8 @@ func WithUNIXSocket() Option { } } +// WithAppKey changes the appkey (aka secret-handshake network cap). +// See https://ssbc.github.io/scuttlebutt-protocol-guide/#handshake for more. func WithAppKey(k []byte) Option { return func(s *Sbot) error { if n := len(k); n != 32 { @@ -269,6 +285,7 @@ func WithAppKey(k []byte) Option { } } +// WithNamedKeyPair changes from the default `secret` file, useful for testing. func WithNamedKeyPair(name string) Option { return func(s *Sbot) error { r := repo.New(s.repoPath) @@ -278,6 +295,8 @@ func WithNamedKeyPair(name string) Option { } } +// WithJSONKeyPair expectes a JSON-string as blob and calls ssb.ParseKeyPair on it. +// This is useful if you dont't want to place the keypair on the filesystem. func WithJSONKeyPair(blob string) Option { return func(s *Sbot) error { var err error @@ -286,6 +305,7 @@ func WithJSONKeyPair(blob string) Option { } } +// WithKeyPair exepect a initialized ssb.KeyPair. Useful for testing. func WithKeyPair(kp *ssb.KeyPair) Option { return func(s *Sbot) error { s.KeyPair = kp @@ -293,6 +313,7 @@ func WithKeyPair(kp *ssb.KeyPair) Option { } } +// WithInfo changes the info/warn/debug loging output. func WithInfo(log kitlog.Logger) Option { return func(s *Sbot) error { s.info = log @@ -300,6 +321,9 @@ func WithInfo(log kitlog.Logger) Option { } } +// WithContext changes the context that is context.Background() by default. +// Handy to setup cancelation against a interup signal like ctrl+c. +// Canceling the context also shuts down indexing. If no context is passed sbot.Shutdown() can be used. func WithContext(ctx context.Context) Option { return func(s *Sbot) error { s.rootCtx = ctx @@ -307,17 +331,10 @@ func WithContext(ctx context.Context) Option { } } -func WithKeyManager(log kitlog.Logger) Option { - return func(s *Sbot) error { - return fmt.Errorf("TODO: make private manager an index") - /* - mount := MountSimpleIndex("keys", // key manager here // ) - return mount(s) - */ - } -} - // TODO: remove all this network stuff and make them options on network + +// WithPreSecureConnWrapper wrapps the connection after it is encrypted. +// Usefull for debugging and measuring traffic. func WithPreSecureConnWrapper(cw netwrap.ConnWrapper) Option { return func(s *Sbot) error { s.preSecureWrappers = append(s.preSecureWrappers, cw) @@ -325,7 +342,8 @@ func WithPreSecureConnWrapper(cw netwrap.ConnWrapper) Option { } } -// TODO: remove all this network stuff and make them options on network +// WithPostSecureConnWrapper wrapps the connection before it is encrypted. +// Usefull to insepct the muxrpc frames before they go into boxstream. func WithPostSecureConnWrapper(cw netwrap.ConnWrapper) Option { return func(s *Sbot) error { s.postSecureWrappers = append(s.postSecureWrappers, cw) @@ -333,6 +351,7 @@ func WithPostSecureConnWrapper(cw netwrap.ConnWrapper) Option { } } +// WithEventMetrics sets up latency and counter metrics func WithEventMetrics(ctr metrics.Counter, lvls metrics.Gauge, lat metrics.Histogram) Option { return func(s *Sbot) error { s.eventCounter = ctr @@ -342,6 +361,7 @@ func WithEventMetrics(ctr metrics.Counter, lvls metrics.Gauge, lat metrics.Histo } } +// WithEndpointWrapper sets a MuxrpcEndpointWrapper for new connections. func WithEndpointWrapper(mw MuxrpcEndpointWrapper) Option { return func(s *Sbot) error { s.edpWrapper = mw @@ -357,7 +377,7 @@ func EnableAdvertismentBroadcasts(do bool) Option { } } -// EnableAdvertismentBroadcasts controls local peer discovery through listening for and connecting to UDP broadcasts +// EnableAdvertismentDialing controls local peer discovery through listening for and connecting to UDP broadcasts func EnableAdvertismentDialing(do bool) Option { return func(s *Sbot) error { s.enableDiscovery = do @@ -365,6 +385,8 @@ func EnableAdvertismentDialing(do bool) Option { } } +// WithHMACSigning sets an HMAC signing key for messages. +// Useful for testing, see https://github.com/ssb-js/ssb-validate#state--validateappendstate-hmac_key-msg for more. func WithHMACSigning(key []byte) Option { return func(s *Sbot) error { if n := len(key); n != 32 { @@ -375,6 +397,7 @@ func WithHMACSigning(key []byte) Option { } } +// WithWebsocketAddress changes the HTTP listener address, by default it's :8989. func WithWebsocketAddress(addr string) Option { return func(s *Sbot) error { // TODO: listen here? @@ -433,6 +456,7 @@ func LateOption(o Option) Option { } } +// New creates an sbot instance using the passed options to configure it. func New(fopts ...Option) (*Sbot, error) { var s Sbot s.liveIndexUpdates = true diff --git a/sbot/replicate.go b/sbot/replicate.go index e62e542f..15041f21 100644 --- a/sbot/replicate.go +++ b/sbot/replicate.go @@ -32,7 +32,7 @@ func (s *Sbot) newGraphReplicator() (*graphReplicator, error) { update := r.makeUpdater(replicateEvt, s.KeyPair.Id, int(s.hopCount)) // update for new messages but only every 15seconds - go debounce(s.rootCtx, 15*time.Second, s.RootLog.Seq(), update) + go debounce(s.rootCtx, 15*time.Second, s.ReceiveLog.Seq(), update) return &r, nil } diff --git a/sbot/status.go b/sbot/status.go index b755efd7..67eb918b 100644 --- a/sbot/status.go +++ b/sbot/status.go @@ -17,7 +17,7 @@ import ( ) func (sbot *Sbot) Status() (ssb.Status, error) { - v, err := sbot.RootLog.Seq().Value() + v, err := sbot.ReceiveLog.Seq().Value() if err != nil { return ssb.Status{}, errors.Wrap(err, "failed to get root log sequence") } diff --git a/sbot/unipub_test.go b/sbot/unipub_test.go index 92f5345a..4c5b36a5 100644 --- a/sbot/unipub_test.go +++ b/sbot/unipub_test.go @@ -36,7 +36,6 @@ func TestPublishUnicode(t *testing.T) { WithInfo(mainLog), WithRepoPath(filepath.Join("testrun", t.Name(), "ali")), WithListenAddr(":0"), - // LateOption(MountPlugin(&bytype.Plugin{}, plugins2.AuthMaster)), ) r.NoError(err) @@ -60,7 +59,7 @@ func TestPublishUnicode(t *testing.T) { _, err = ali.PublishLog.Append(newMsg) r.NoError(err) - src, err := ali.RootLog.Query() + src, err := ali.ReceiveLog.Query() r.NoError(err) var i = 0 for { diff --git a/tests/blobs_test.go b/tests/blobs_test.go index d2e6ee1d..7b37eb64 100644 --- a/tests/blobs_test.go +++ b/tests/blobs_test.go @@ -15,6 +15,7 @@ import ( "go.cryptoscope.co/ssb/internal/leakcheck" "go.cryptoscope.co/ssb/internal/mutil" + "go.cryptoscope.co/ssb/internal/storedrefs" refs "go.mindeco.de/ssb-refs" "github.com/stretchr/testify/require" @@ -163,7 +164,7 @@ func TestBlobWithHop(t *testing.T) { uf, ok := bob.GetMultiLog("userFeeds") r.True(ok) - aliceLog, err := uf.Get(alice.StoredAddr()) + aliceLog, err := uf.Get(storedrefs.Feed(alice)) r.NoError(err) gotMessage := make(chan struct{}) @@ -184,7 +185,7 @@ func TestBlobWithHop(t *testing.T) { var wantBlob *refs.BlobRef - msg, err := mutil.Indirect(bob.RootLog, aliceLog).Get(margaret.BaseSeq(0)) + msg, err := mutil.Indirect(bob.ReceiveLog, aliceLog).Get(margaret.BaseSeq(0)) r.NoError(err) storedMsg, ok := msg.(refs.Message) r.True(ok, "wrong type of message: %T", msg) @@ -326,9 +327,9 @@ func TestBlobTooBigWantedByGo(t *testing.T) { uf, ok := s.GetMultiLog("userFeeds") r.True(ok) - jsFeedSeqs, err := uf.Get(jsBot.StoredAddr()) + jsFeedSeqs, err := uf.Get(storedrefs.Feed(jsBot)) r.NoError(err) - jsFeed := mutil.Indirect(s.RootLog, jsFeedSeqs) + jsFeed := mutil.Indirect(s.ReceiveLog, jsFeedSeqs) tries := 10 var testData struct { Type string `json:"test-data"` diff --git a/tests/feeds_test.go b/tests/feeds_test.go index cd2667b3..ec660674 100644 --- a/tests/feeds_test.go +++ b/tests/feeds_test.go @@ -14,6 +14,7 @@ import ( "go.cryptoscope.co/ssb/internal/leakcheck" "go.cryptoscope.co/ssb/internal/mutil" + "go.cryptoscope.co/ssb/internal/storedrefs" "go.cryptoscope.co/ssb/sbot" ) @@ -60,7 +61,7 @@ func TestFeedFromJS(t *testing.T) { uf, ok := bob.GetMultiLog("userFeeds") r.True(ok) - aliceLog, err := uf.Get(alice.StoredAddr()) + aliceLog, err := uf.Get(storedrefs.Feed(alice)) r.NoError(err) seq, err := aliceLog.Seq().Value() r.NoError(err) @@ -68,7 +69,7 @@ func TestFeedFromJS(t *testing.T) { var lastMsg string for i := 0; i < n; i++ { // don't check the contact:following message from A to C - msg, err := mutil.Indirect(bob.RootLog, aliceLog).Get(margaret.BaseSeq(i)) + msg, err := mutil.Indirect(bob.ReceiveLog, aliceLog).Get(margaret.BaseSeq(i)) r.NoError(err) storedMsg, ok := msg.(refs.Message) r.True(ok, "wrong type of message: %T", msg) @@ -224,12 +225,12 @@ func TestFeedFromGo(t *testing.T) { uf, ok := s.GetMultiLog("userFeeds") r.True(ok) - aliceLog, err := uf.Get(alice.StoredAddr()) + aliceLog, err := uf.Get(storedrefs.Feed(alice)) r.NoError(err) seqMsg, err := aliceLog.Get(margaret.BaseSeq(1)) r.NoError(err) - msg, err := s.RootLog.Get(seqMsg.(margaret.BaseSeq)) + msg, err := s.ReceiveLog.Get(seqMsg.(margaret.BaseSeq)) r.NoError(err) storedMsg, ok := msg.(refs.Message) r.True(ok, "wrong type of message: %T", msg) @@ -246,25 +247,25 @@ func TestFeedFromGo(t *testing.T) { ml, ok := s.GetMultiLog("userFeeds") r.True(ok) - aliceLog, err = ml.Get(alice.StoredAddr()) + aliceLog, err = ml.Get(storedrefs.Feed(alice)) r.NoError(err) aseq, err := aliceLog.Seq().Value() r.NoError(err) seqMsg, err = aliceLog.Get(aseq.(margaret.BaseSeq)) r.NoError(err) - msg, err = s.RootLog.Get(seqMsg.(margaret.BaseSeq)) + msg, err = s.ReceiveLog.Get(seqMsg.(margaret.BaseSeq)) r.NoError(err) storedMsg, ok = msg.(refs.Message) r.True(ok, "wrong type of message: %T", msg) r.EqualValues(storedMsg.Seq(), 2) - bobLog, err := ml.Get(s.KeyPair.Id.StoredAddr()) + bobLog, err := ml.Get(storedrefs.Feed(s.KeyPair.Id)) r.NoError(err) bseq, err := bobLog.Seq().Value() r.NoError(err) seqMsg, err = bobLog.Get(bseq.(margaret.BaseSeq)) r.NoError(err) - msg, err = s.RootLog.Get(seqMsg.(margaret.BaseSeq)) + msg, err = s.ReceiveLog.Get(seqMsg.(margaret.BaseSeq)) r.NoError(err) storedMsg, ok = msg.(refs.Message) r.True(ok, "wrong type of message: %T", msg) @@ -366,12 +367,12 @@ func XTestFeedFromGoLive(t *testing.T) { uf, ok := s.GetMultiLog("userFeeds") r.True(ok) - aliceLog, err := uf.Get(alice.StoredAddr()) + aliceLog, err := uf.Get(storedrefs.Feed(alice)) r.NoError(err) seqMsg, err := aliceLog.Get(margaret.BaseSeq(1)) r.NoError(err) - msg, err := s.RootLog.Get(seqMsg.(margaret.BaseSeq)) + msg, err := s.ReceiveLog.Get(seqMsg.(margaret.BaseSeq)) r.NoError(err) storedMsg, ok := msg.(refs.Message) r.True(ok, "wrong type of message: %T", msg) diff --git a/tests/format_ggrove_test.go b/tests/format_ggrove_test.go index b62ed2a7..cd07f941 100644 --- a/tests/format_ggrove_test.go +++ b/tests/format_ggrove_test.go @@ -15,6 +15,7 @@ import ( "go.cryptoscope.co/muxrpc" "go.cryptoscope.co/muxrpc/codec" "go.cryptoscope.co/ssb" + "go.cryptoscope.co/ssb/internal/storedrefs" "go.cryptoscope.co/ssb/message" "go.cryptoscope.co/ssb/sbot" refs "go.mindeco.de/ssb-refs" @@ -100,7 +101,7 @@ func TestGabbyFeedFromGo(t *testing.T) { } return err } - _, err = s.RootLog.Append(val) + _, err = s.ReceiveLog.Append(val) return errors.Wrap(err, "failed to append verified message to rootLog") }) snk := message.NewVerifySink(&aliceAsGabby, margaret.BaseSeq(1), nil, store, nil) @@ -112,7 +113,7 @@ func TestGabbyFeedFromGo(t *testing.T) { uf, ok := s.GetMultiLog("userFeeds") r.True(ok) - demoLog, err := uf.Get(aliceAsGabby.StoredAddr()) + demoLog, err := uf.Get(storedrefs.Feed(&aliceAsGabby)) r.NoError(err) demoLogSeq, err := demoLog.Seq().Value() @@ -122,7 +123,7 @@ func TestGabbyFeedFromGo(t *testing.T) { for demoFeedSeq := margaret.BaseSeq(1); demoFeedSeq < 3; demoFeedSeq++ { seqMsg, err := demoLog.Get(demoFeedSeq - 1) r.NoError(err) - msg, err := s.RootLog.Get(seqMsg.(margaret.BaseSeq)) + msg, err := s.ReceiveLog.Get(seqMsg.(margaret.BaseSeq)) r.NoError(err) storedMsg, ok := msg.(refs.Message) r.True(ok, "wrong type of message: %T", msg) diff --git a/tests/interop_test.go b/tests/interop_test.go index 45f26317..a0f59c6c 100644 --- a/tests/interop_test.go +++ b/tests/interop_test.go @@ -198,12 +198,12 @@ func (ts *testSession) startJSBotWithName(name, jsbefore, jsafter string) *refs. ts.backgroundErrs = append(ts.backgroundErrs, errc) pubScanner := bufio.NewScanner(outrc) // TODO muxrpc comms? - r.True(pubScanner.Scan(), "multiple lines of output from js - expected #1 to be alices pubkey/id") + r.True(pubScanner.Scan(), "multiple lines of output from js - expected #1 to be %s pubkey/id", name) - alice, err := refs.ParseFeedRef(pubScanner.Text()) - r.NoError(err, "failed to get alice key from JS process") - ts.t.Logf("JS alice: %d %s", jsBotCnt, alice.Ref()) - return alice + jsBotRef, err := refs.ParseFeedRef(pubScanner.Text()) + r.NoError(err, "failed to get %s key from JS process") + ts.t.Logf("JS %s:%d %s", name, jsBotCnt, jsBotRef.Ref()) + return jsBotRef } func (ts *testSession) wait() { diff --git a/tests/invite_peer_test.go b/tests/invite_peer_test.go index cf3ba280..7105e324 100644 --- a/tests/invite_peer_test.go +++ b/tests/invite_peer_test.go @@ -15,15 +15,11 @@ import ( "github.com/stretchr/testify/require" "go.cryptoscope.co/netwrap" - "go.cryptoscope.co/ssb/indexes" "go.cryptoscope.co/ssb/message/legacy" - "go.cryptoscope.co/ssb/plugins2" - "go.cryptoscope.co/ssb/plugins2/bytype" refs "go.mindeco.de/ssb-refs" "golang.org/x/crypto/nacl/auth" "go.cryptoscope.co/ssb" - "go.cryptoscope.co/ssb/sbot" ) // first js creates an invite @@ -38,10 +34,7 @@ func XTestPeerInviteJSCreate(t *testing.T) { ts := newRandomSession(t) // ts := newSession(t, nil, nil) - ts.startGoBot( - sbot.LateOption(sbot.MountPlugin(&bytype.Plugin{}, plugins2.AuthMaster)), - sbot.LateOption(sbot.MountSimpleIndex("get", indexes.OpenGet)), - ) + ts.startGoBot() bob := ts.gobot wrappedAddr := bob.Network.GetListenAddr() diff --git a/tests/package-lock.json b/tests/package-lock.json index 96e288b0..9986e82c 100644 --- a/tests/package-lock.json +++ b/tests/package-lock.json @@ -1,9 +1,52 @@ { "name": "go-sbot-tests", - "version": "1.0.0", + "version": "1.0.1", "lockfileVersion": 1, "requires": true, "dependencies": { + "@tangle/graph": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@tangle/graph/-/graph-1.2.0.tgz", + "integrity": "sha512-5p61oJJ+WjLedqlaFIsBKhRmP3YdOoJhVBZPUvYojY6PWzQHqe32tHt2Ah4ABr1Pb9g9G/ehd9Dp+65ZnwByOA==", + "requires": { + "lodash.clone": "^4.5.0", + "lodash.set": "^4.3.2" + } + }, + "@tangle/overwrite": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@tangle/overwrite/-/overwrite-1.2.0.tgz", + "integrity": "sha512-b6+/thHuJiW6B7bOaoX0mEUAJmfk0q7lyN910l0TTg4qJKSpOcQZfLJxQBo5WVYjMV5MVOkYj7mKhXm/N3JRVg==", + "requires": { + "is-my-json-valid": "^2.20.5", + "lodash.isequal": "^4.5.0" + } + }, + "@tangle/reduce": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@tangle/reduce/-/reduce-1.0.3.tgz", + "integrity": "sha512-WU1cZLN4GToV5Gf5k0gnDzUW9CxU23PqzaBq85bqoWz/iqU7nyY769d4eppVkAIJ1a9O158RE3C19aQyuQT8tA==", + "requires": { + "@tangle/graph": "^1.2.0" + } + }, + "@tangle/simple-set": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@tangle/simple-set/-/simple-set-1.2.2.tgz", + "integrity": "sha512-IqID8LLCb2IYecUzDuzXZc+4vPLTlQq9is0KPU/c+ekFrVxaxg6efi14oTkPHIhw/HBwE3SdnZyDnQrhVtv6ig==", + "requires": { + "is-my-json-valid": "^2.20.5" + } + }, + "@tangle/strategy": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/@tangle/strategy/-/strategy-1.4.1.tgz", + "integrity": "sha512-OFFFcPH07hy6G54pnqWEVoJRd1Wro0CUAYMS4yLsbhLt7dlE814ZFpF50AFpRehFGBFsI/AsWoa9jEVRmXf3zA==", + "requires": { + "is-my-json-valid": "^2.20.5", + "lodash.isequal": "^4.5.0" + } + }, "abstract-leveldown": { "version": "6.2.3", "resolved": "https://registry.npmjs.org/abstract-leveldown/-/abstract-leveldown-6.2.3.tgz", @@ -125,6 +168,14 @@ "ieee754": "^1.1.4" } }, + "buffer-xor": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/buffer-xor/-/buffer-xor-2.0.2.tgz", + "integrity": "sha512-eHslX0bin3GB+Lx2p7lEYRShRewuNZL3fUl4qlVJGGiwoPGftmt8JQgk2Y9Ji5/01TnVDo33E5b5O3vUB1HdqQ==", + "requires": { + "safe-buffer": "^5.1.1" + } + }, "cbor": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/cbor/-/cbor-4.3.0.tgz", @@ -371,6 +422,22 @@ "level-errors": "^2.0.0" } }, + "envelope-js": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/envelope-js/-/envelope-js-1.0.0.tgz", + "integrity": "sha512-Z0zHgYNMZIOqR6CehiPowBwzq9uB9ikcp2WexCiOGcBMOsfsB0dQR65VnVkqZnTFUmi9fGC1syptGEM1TULz7g==", + "requires": { + "buffer-xor": "^2.0.2", + "envelope-spec": "^1.0.0", + "futoin-hkdf": "^1.3.2", + "sodium-native": "^3.2.0" + } + }, + "envelope-spec": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/envelope-spec/-/envelope-spec-1.0.0.tgz", + "integrity": "sha512-NJ/neUZbvGgrq/cVmMZvNR58aypn3017/WIVIeOJCtMdREdkKa3G6tTMo40RqONNI3QyPtDocpBdtkbisHim2Q==" + }, "errno": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/errno/-/errno-0.1.7.tgz", @@ -516,25 +583,53 @@ }, "dependencies": { "deep-equal": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-2.0.3.tgz", - "integrity": "sha512-Spqdl4H+ky45I9ByyJtXteOm9CaIrPmnIPmOhrkKGNYWeDgCvJ8jNYVCTjChxW4FqGuZnLHADc8EKRMX6+CgvA==", + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-2.0.4.tgz", + "integrity": "sha512-BUfaXrVoCfgkOQY/b09QdO9L3XNoF2XH0A3aY9IQwQL/ZjLOe8FQgCNVl1wiolhsFo8kFdO9zdPViCPbmaJA5w==", "requires": { - "es-abstract": "^1.17.5", + "es-abstract": "^1.18.0-next.1", "es-get-iterator": "^1.1.0", "is-arguments": "^1.0.4", "is-date-object": "^1.0.2", - "is-regex": "^1.0.5", + "is-regex": "^1.1.1", "isarray": "^2.0.5", - "object-is": "^1.1.2", + "object-is": "^1.1.3", "object-keys": "^1.1.1", - "object.assign": "^4.1.0", + "object.assign": "^4.1.1", "regexp.prototype.flags": "^1.3.0", - "side-channel": "^1.0.2", + "side-channel": "^1.0.3", "which-boxed-primitive": "^1.0.1", "which-collection": "^1.0.1", "which-typed-array": "^1.1.2" } + }, + "es-abstract": { + "version": "1.18.0-next.1", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.18.0-next.1.tgz", + "integrity": "sha512-I4UGspA0wpZXWENrdA0uHbnhte683t3qT/1VFH9aX2dA5PPSf6QW5HHXf5HImaqPmjXaVeVk4RGWnaylmV7uAA==", + "requires": { + "es-to-primitive": "^1.2.1", + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.1", + "is-callable": "^1.2.2", + "is-negative-zero": "^2.0.0", + "is-regex": "^1.1.1", + "object-inspect": "^1.8.0", + "object-keys": "^1.1.1", + "object.assign": "^4.1.1", + "string.prototype.trimend": "^1.0.1", + "string.prototype.trimstart": "^1.0.1" + } + }, + "object-is": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/object-is/-/object-is-1.1.3.tgz", + "integrity": "sha512-teyqLvFWzLkq5B9ki8FVWA902UER2qkxmdA4nLf+wjOLAWgxzCWZNCxpDq9MvE8MmhWNr+I8w3BN49Vx36Y6Xg==", + "requires": { + "define-properties": "^1.1.3", + "es-abstract": "^1.18.0-next.1" + } } } }, @@ -589,6 +684,11 @@ "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" }, + "futoin-hkdf": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/futoin-hkdf/-/futoin-hkdf-1.3.2.tgz", + "integrity": "sha512-3EVi3ETTyJg5PSXlxLCaUVVn0pSbDf62L3Gwxne7Uq+d8adOSNWQAad4gg7WToHkcgnCJb3Wlb1P8r4Evj4GPw==" + }, "gabbygrove": { "version": "github:cryptix/js-gabbygrove#1680eec8a52dc55b162e2039a09e505f3b63a8ca", "from": "github:cryptix/js-gabbygrove", @@ -609,6 +709,22 @@ } } }, + "generate-function": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/generate-function/-/generate-function-2.3.1.tgz", + "integrity": "sha512-eeB5GfMNeevm/GRYq20ShmsaGcmI81kIX2K9XQx5miC8KdHaC6Jm0qQ8ZNeGOi7wYB8OsdxKs+Y2oVuTFuVwKQ==", + "requires": { + "is-property": "^1.0.2" + } + }, + "generate-object-property": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/generate-object-property/-/generate-object-property-1.2.0.tgz", + "integrity": "sha1-nA4cQDCM6AT0eDYYuTf6iPmdUNA=", + "requires": { + "is-property": "^1.0.0" + } + }, "glob": { "version": "7.1.6", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", @@ -748,6 +864,32 @@ "resolved": "https://registry.npmjs.org/is-map/-/is-map-2.0.1.tgz", "integrity": "sha512-T/S49scO8plUiAOA2DBTBG3JHpn1yiw0kRp6dgiZ0v2/6twi5eiB0rHtHFH9ZIrvlWc6+4O+m4zg5+Z833aXgw==" }, + "is-my-ip-valid": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-my-ip-valid/-/is-my-ip-valid-1.0.0.tgz", + "integrity": "sha512-gmh/eWXROncUzRnIa1Ubrt5b8ep/MGSnfAUI3aRp+sqTCs1tv1Isl8d8F6JmkN3dXKc3ehZMrtiPN9eL03NuaQ==" + }, + "is-my-json-valid": { + "version": "2.20.5", + "resolved": "https://registry.npmjs.org/is-my-json-valid/-/is-my-json-valid-2.20.5.tgz", + "integrity": "sha512-VTPuvvGQtxvCeghwspQu1rBgjYUT6FGxPlvFKbYuFtgc4ADsX3U5ihZOYN0qyU6u+d4X9xXb0IT5O6QpXKt87A==", + "requires": { + "generate-function": "^2.0.0", + "generate-object-property": "^1.1.0", + "is-my-ip-valid": "^1.0.0", + "jsonpointer": "^4.0.0", + "xtend": "^4.0.0" + } + }, + "is-my-ssb-valid": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-my-ssb-valid/-/is-my-ssb-valid-1.1.0.tgz", + "integrity": "sha512-HAcl2v9tIkpjfVqaNgaqYWk2MW3WPZtj+XVpKUMgQRWEbLJdZ4orE5l9SJ5t+bQO1LB5rKeYGTlMfxSXEG1NTw==", + "requires": { + "is-my-json-valid": "^2.20.0", + "ssb-msg-content": "^1.0.1" + } + }, "is-negative-zero": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.0.tgz", @@ -758,6 +900,11 @@ "resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.0.4.tgz", "integrity": "sha512-zohwelOAur+5uXtk8O3GPQ1eAcu4ZX3UwxQhUlfFFMNpUd83gXgjbhJh6HmB6LUNV/ieOLQuDwJO3dWJosUeMw==" }, + "is-property": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz", + "integrity": "sha1-V/4cTkhHTt1lsJkR8msc1Ald2oQ=" + }, "is-regex": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.1.tgz", @@ -828,6 +975,11 @@ "delimit-stream": "0.1.0" } }, + "jsonpointer": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/jsonpointer/-/jsonpointer-4.1.0.tgz", + "integrity": "sha512-CXcRvMyTlnR53xMcKnuMzfCA5i/nfblTnnr74CZb6C4vG39eu6w51t7nKmU5MfLfbTgGItliNyjO/ciNPDqClg==" + }, "layered-graph": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/layered-graph/-/layered-graph-1.1.3.tgz", @@ -968,6 +1120,11 @@ "libsodium": "0.7.8" } }, + "lodash.clone": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.clone/-/lodash.clone-4.5.0.tgz", + "integrity": "sha1-GVhwRQ9aExkkeN9Lw9I9LeoZB7Y=" + }, "lodash.clonedeep": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz", @@ -978,6 +1135,16 @@ "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", "integrity": "sha1-LRd/ZS+jHpObRDjVNBSZ36OCXpk=" }, + "lodash.isequal": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz", + "integrity": "sha1-QVxEePK8wwEgwizhDtMib30+GOA=" + }, + "lodash.set": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/lodash.set/-/lodash.set-4.3.2.tgz", + "integrity": "sha1-2HV7HagH3eJIFrDWqEvqGnYjCyM=" + }, "looper": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/looper/-/looper-4.0.0.tgz", @@ -1309,6 +1476,11 @@ "chloride": "^2.2.9" } }, + "private-group-spec": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/private-group-spec/-/private-group-spec-1.0.0.tgz", + "integrity": "sha512-C4EJF39/13tKfUGbUzF/pgYTVNIOB6yWuWhz3IKTt4bmPeIG3EUAFsCsmk8E5x8Mzrg78fg2zpvstGoah11RHA==" + }, "promisize": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/promisize/-/promisize-1.1.2.tgz", @@ -1868,6 +2040,17 @@ "resolved": "https://registry.npmjs.org/split-buffer/-/split-buffer-1.0.0.tgz", "integrity": "sha1-t+jgq1E0UVi3LB9tvvJAbVHx0Cc=" }, + "ssb-backlinks": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ssb-backlinks/-/ssb-backlinks-2.1.1.tgz", + "integrity": "sha512-iv/B5EyjvNiKOeT3RkTuBRyU14GJ1sf5wnr07JOWeVI3OyNzedZ8z229yzZSaGHTYpbiSOcs9Z2CR8CkLQupQQ==", + "requires": { + "base64-url": "^2.3.3", + "flumeview-links": "^2.1.0", + "pull-stream": "^3.6.14", + "ssb-ref": "^2.14.0" + } + }, "ssb-blobs": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/ssb-blobs/-/ssb-blobs-1.2.3.tgz", @@ -2149,6 +2332,11 @@ } } }, + "ssb-msg-content": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/ssb-msg-content/-/ssb-msg-content-1.0.1.tgz", + "integrity": "sha512-M6W0Ef+jif829USmGvh6XeS4lYb/F2lgFhfEoCE/md7ESILNOGidp8frJE2uVOzSr2wVRA265tPrnVb7rYHkug==" + }, "ssb-msgs": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/ssb-msgs/-/ssb-msgs-5.2.0.tgz", @@ -2198,15 +2386,13 @@ } } }, - "ssb-private": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/ssb-private/-/ssb-private-1.1.0.tgz", - "integrity": "sha512-6KbOOCzZcYmhrWvw3hdfhPgkoLRfXha5Rw7Kzu+IafhhC7lQR0hI1aD3v7swp1IeofYICFtdXih+AUoTQcvykA==", + "ssb-private1": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/ssb-private1/-/ssb-private1-1.0.1.tgz", + "integrity": "sha512-x69YHNhjxCrknkK7XbEJyk2P0P3p52t6NF74I8ObHIrBdWnyRrO6iUH8K5b8CkaHawM4giXdZG5cyrOPzPN/Fg==", "requires": { - "base64-url": "^2.2.0", - "explain-error": "^1.0.4", - "flumeview-links": "^2.0.1", - "ssb-keys": "^7.0.14" + "ssb-keys": "^7.2.2", + "ssb-ref": "^2.13.9" }, "dependencies": { "ssb-keys": { @@ -2259,6 +2445,78 @@ "ssb-ref": "^2.13.9" } }, + "ssb-schema-definitions": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ssb-schema-definitions/-/ssb-schema-definitions-3.2.1.tgz", + "integrity": "sha512-+KBljo89bBi4gv3dnoiop4PexKLlySzq5bDcHvQ5aE4zXB8V33hocHuoJbihPr/jxry4kiqqo31TETwjV8YerQ==", + "requires": { + "lodash.clone": "^4.5.0", + "ssb-ref": "^2.14.0" + } + }, + "ssb-tangle": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ssb-tangle/-/ssb-tangle-2.1.3.tgz", + "integrity": "sha512-zrwfFQOoxn9cE9DI452+vnj6BMLwFvuZ8zhokGFuIu1SE/WYAwxWb52E3q7shg0aVQbYnpaVKnhssRo3HzuMuQ==", + "requires": { + "@tangle/graph": "^1.1.0", + "@tangle/overwrite": "^1.0.0", + "@tangle/reduce": "^1.0.1", + "@tangle/simple-set": "^1.0.0", + "@tangle/strategy": "^1.0.0" + } + }, + "ssb-tribes": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/ssb-tribes/-/ssb-tribes-0.4.1.tgz", + "integrity": "sha512-iE6haw3CwSHGJbrFwBPpp8eM2IaWqRYLyspXLQz58+QjUqoGWWq8T74aIR5VxwCyhXQE8Ur8BOMLYLCLnwUAyA==", + "requires": { + "charwise": "^3.0.1", + "envelope-js": "^1.0.0", + "envelope-spec": "^1.0.0", + "futoin-hkdf": "^1.3.2", + "is-my-ssb-valid": "^1.1.0", + "level": "^6.0.1", + "lodash.set": "^4.3.2", + "mkdirp": "^1.0.4", + "private-group-spec": "^1.0.0", + "pull-level": "^2.0.4", + "pull-paramap": "^1.2.2", + "pull-stream": "^3.6.14", + "sodium-native": "^3.2.0", + "ssb-keys": "^7.2.2", + "ssb-ref": "^2.14.0", + "ssb-schema-definitions": "^3.1.0", + "ssb-tangle": "^2.1.0" + }, + "dependencies": { + "mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==" + }, + "ssb-keys": { + "version": "7.2.2", + "resolved": "https://registry.npmjs.org/ssb-keys/-/ssb-keys-7.2.2.tgz", + "integrity": "sha512-FPeyYU/3LpxcagnbmVWE+Q/qzg6keqeOBPbD7sEH9UKixUASeufPKiORDgh8nVX7J9Z+0vUaHt/WG999kGjvVQ==", + "requires": { + "chloride": "^2.2.8", + "mkdirp": "~0.5.0", + "private-box": "^0.3.0" + }, + "dependencies": { + "mkdirp": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", + "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", + "requires": { + "minimist": "^1.2.5" + } + } + } + } + } + }, "ssb-validate": { "version": "3.0.11", "resolved": "https://registry.npmjs.org/ssb-validate/-/ssb-validate-3.0.11.tgz", diff --git a/tests/package.json b/tests/package.json index 444353f4..3b5b5c50 100644 --- a/tests/package.json +++ b/tests/package.json @@ -10,8 +10,9 @@ "dependencies": { "gabbygrove": "github:cryptix/js-gabbygrove", "run-parallel": "^1.1.9", - "sodium-native": "^3.2.0", "secret-stack": "^6.3.1", + "sodium-native": "^3.2.0", + "ssb-backlinks": "^2.1.1", "ssb-blobs": "^1.2.3", "ssb-config": "^3.4.5", "ssb-db": "^20.3.0", @@ -22,9 +23,10 @@ "ssb-invite": "^2.1.6", "ssb-keys": "^8.0.0", "ssb-peer-invites": "^2.1.0", - "ssb-private": "^1.1.0", + "ssb-private1": "^1.0.1", "ssb-query": "^2.4.5", "ssb-replicate": "^1.3.2", + "ssb-tribes": "^0.4.1", "tape": "^5.0.1" } } diff --git a/tests/pms_test.go b/tests/pms_test.go index 9b6c059e..cc2632ee 100644 --- a/tests/pms_test.go +++ b/tests/pms_test.go @@ -12,6 +12,7 @@ import ( "go.cryptoscope.co/margaret" refs "go.mindeco.de/ssb-refs" + "go.cryptoscope.co/ssb/internal/storedrefs" "go.cryptoscope.co/ssb/private/box" ) @@ -112,12 +113,12 @@ func TestPrivMsgsFromGo(t *testing.T) { <-ts.doneJS - aliceLog, err := s.Users.Get(alice.StoredAddr()) + aliceLog, err := s.Users.Get(storedrefs.Feed(alice)) r.NoError(err) seqMsg, err := aliceLog.Get(margaret.BaseSeq(1)) r.NoError(err) - msg, err := s.RootLog.Get(seqMsg.(margaret.BaseSeq)) + msg, err := s.ReceiveLog.Get(seqMsg.(margaret.BaseSeq)) r.NoError(err) storedMsg, ok := msg.(refs.Message) r.True(ok, "wrong type of message: %T", msg) @@ -174,7 +175,7 @@ func TestPrivMsgsFromJS(t *testing.T) { uf, ok := bob.GetMultiLog("userFeeds") r.True(ok) - aliceLog, err := uf.Get(alice.StoredAddr()) + aliceLog, err := uf.Get(storedrefs.Feed(alice)) r.NoError(err) seq, err := aliceLog.Seq().Value() r.NoError(err) @@ -187,7 +188,7 @@ func TestPrivMsgsFromJS(t *testing.T) { r.NoError(err) r.Equal(seqMsg, margaret.BaseSeq(1+i)) - msg, err := bob.RootLog.Get(seqMsg.(margaret.BaseSeq)) + msg, err := bob.ReceiveLog.Get(seqMsg.(margaret.BaseSeq)) r.NoError(err) absMsg, ok := msg.(refs.Message) r.True(ok, "wrong type of message: %T", msg) diff --git a/tests/privategroups_test.go b/tests/privategroups_test.go new file mode 100644 index 00000000..f9bfe196 --- /dev/null +++ b/tests/privategroups_test.go @@ -0,0 +1,267 @@ +package tests + +import ( + "encoding/json" + "fmt" + "testing" + + "go.cryptoscope.co/librarian" + "go.cryptoscope.co/ssb/internal/mutil" + "go.cryptoscope.co/ssb/private" + refs "go.mindeco.de/ssb-refs" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.cryptoscope.co/margaret" + "go.cryptoscope.co/ssb/internal/storedrefs" +) + +func TestGroupsJSCreate(t *testing.T) { + // defer leakcheck.Check(t) + r := require.New(t) + const n = 24 + 4 // m*spam + (create, 2*contact, invite, hint) + + ts := newRandomSession(t) + + ts.startGoBot() + bob := ts.gobot + + // just for keygen, needed later + claire := ts.startJSBotWithName("claire", `exit()`, "") + + alice := ts.startJSBot(fmt.Sprintf(` + let claireRef = %q + + sbot.tribes.create({}, (err, data) => { + const { groupId } = data + t.comment(groupId) + + function mkMsg(msg) { + return function(cb) { + sbot.publish(msg, cb) + } + } + n = 24 + let msgs = [] + for (var i = n; i>0; i--) { + msgs.push(mkMsg({type:"test", text:"foo", i:i, "recps": [groupId]})) + } + msgs.push(mkMsg({type: 'contact', contact: claireRef, following: true})) + msgs.push(mkMsg({type: 'contact', contact: testBob, following: true})) + + let welcome = { + text: 'this is a test group' + } + sbot.tribes.invite(groupId, [testBob, claireRef], welcome, (err, msg) => { + t.error(err, 'invite worked') + msgs.push(mkMsg({type: 'test-hint', invite: msg.key, groupId})) + parallel(msgs, function(err, results) { + t.error(err, "parallel of publish") + t.equal(msgs.length, results.length, "message count") + run() + }) + }) + }) // tribes.create + + + // be done when the other party is done + sbot.on('rpc:connect', rpc => rpc.on('closed', exit)) + + +`, claire.Ref()), ``) + + bob.PublishLog.Publish(refs.NewContactFollow(alice)) + bob.PublishLog.Publish(refs.NewContactFollow(claire)) + + bob.Replicate(alice) + bob.Replicate(claire) + + dmKey, err := bob.Groups.GetOrDeriveKeyFor(alice) + r.NoError(err) + r.Len(dmKey, 1) + + dmKey, err = bob.Groups.GetOrDeriveKeyFor(claire) + r.NoError(err) + r.Len(dmKey, 1) + + <-ts.doneJS + + aliceLog, err := bob.Users.Get(storedrefs.Feed(alice)) + r.NoError(err) + seq, err := aliceLog.Seq().Value() + r.NoError(err) + r.Equal(margaret.BaseSeq(n), seq) + + bob.Network.GetConnTracker().CloseAll() + + // testutils.StreamLog(t, bob.ReceiveLog) + + hintSeqs, err := bob.ByType.Get(librarian.Addr("test-hint")) + r.NoError(err) + + hints := mutil.Indirect(bob.ReceiveLog, hintSeqs) + seq, err = hints.Seq().Value() + r.NoError(err) + firstMsg := margaret.BaseSeq(0) + r.Equal(firstMsg, seq) + + testHintV, err := hints.Get(firstMsg) + r.NoError(err) + + testHint := testHintV.(refs.Message) + + // this is a public message which just tells the go-side whice one the invite is + var hintContent struct { + Invite *refs.MessageRef + GroupID string + } + err = json.Unmarshal(testHint.ContentBytes(), &hintContent) + r.NoError(err) + r.NotNil(hintContent.Invite) + + t.Log("hint:", hintContent.Invite.Ref()) + inviteMsg, err := bob.Get(*hintContent.Invite) + r.NoError(err) + r.True(inviteMsg.Author().Equal(alice), "not from alice?!") + r.EqualValues(2, inviteMsg.Seq(), "not from alice?!") + + decr, err := bob.Groups.DecryptBox2Message(inviteMsg) + r.NoError(err) + t.Log("decrypted invite:", string(decr)) + + var ga private.GroupAddMember + err = json.Unmarshal(decr, &ga) + r.NoError(err) + + // use the add to join the group + cloaked, err := bob.Groups.Join(ga.GroupKey, ga.Root) + r.NoError(err) + assert.Equal(t, hintContent.GroupID, cloaked.Ref(), "wrong derived cloaked id") + + // reply after joining + helloGroup, err := bob.Groups.PublishPostTo(cloaked, "hello test group!") + r.NoError(err) + + // now start claire and let her read bobs hello. + ts.startJSBotWithName("claire", fmt.Sprintf(` + let testAlice = %q + let helloGroup = %q + + let gotInvite = false + pull( + sbot.createLogStream({ + keys:true, + live:true, + private:true + }), + pull.drain((msg) => { + if (msg.sync) return + if (typeof msg.value.content === "string") return + + if (msg.value.content.type == "group/add-member") { + gotInvite = true + } + }) + ) + + sbot.replicate.request(testAlice, true) + sbot.replicate.request(testBob, true) + run() + + // check if we got the stuff once bob disconnects + sbot.on('rpc:connect', rpc => rpc.on('closed', () => { + setTimeout(() => { + sbot.get({private: true, id: helloGroup}, (err, msg) => { + t.error(err, "received and read hello group msg") + t.true(gotInvite, "got the add-member msg") + t.equal(msg.content.text, "hello test group!", "correct text on group reply from bob") + exit() + }) + }, 2000) + })) +`, alice.Ref(), helloGroup.Ref()), ``) + + bob.Network.GetConnTracker().CloseAll() + + ts.wait() +} + +func TestGroupsGoCreate(t *testing.T) { + // defer leakcheck.Check(t) + r := require.New(t) + + ts := newRandomSession(t) + + ts.startGoBot() + bob := ts.gobot + + // just for keygen, needed later + // or fix live-streaming + alice := ts.startJSBotWithName("alice", `exit()`, "") + + bob.Replicate(alice) + + // TODO: fix first message bug + bob.PublishLog.Publish(map[string]string{ + "type": "needed", + "hello": "world"}) + + groupID, _, err := bob.Groups.Create("yet another test group") + r.NoError(err) + + for i := 7; i > 0; i-- { + jsonContent := fmt.Sprintf(`{"type":"test", "count":%d}`, i) + spam, err := bob.Groups.PublishTo(groupID, []byte(jsonContent)) + r.NoError(err) + t.Logf("%d: spam %s", i, spam.Ref()) + } + + inviteRef, err := bob.Groups.AddMember(groupID, alice, "what's up alice?") + r.NoError(err) + + ts.startJSBotWithName("alice", fmt.Sprintf(` + + let inviteMsg = %q + sbot.replicate.request(testBob, true) + run() // connect to the go side (which already published it's group messages) + + // check if we got the stuff once bob disconnects + sbot.on('rpc:connect', rpc => rpc.on('closed', () => { + setTimeout(() => { + sbot.get({private: true, id: inviteMsg}, (err, msg) => { + t.error(err, "got hello group msg") + t.true(msg.meta.private, 'did decrypt') + t.equal("what's up alice?", msg.content.text, 'has the right welcome text') + // console.warn(JSON.stringify(msg, null, 2)) + exit() + }) + },10000) // was getting the error below + })) +`, inviteRef.Ref()), ``) + + ts.wait() +} + +/* +not ok 3 got hello group msg + --- + operator: error + at: (/home/cryptix/go-repos/ssb/tests/node_modules/ssb-db/create.js:70:25) + stack: |- + TypeError: flumeview-level.get: index for: %x+5CduWrrxQFvNoViY7v8r2In/9v9aIOR1Q97/7pcNM=.sha256pointed at:18247but log error + at /home/cryptix/go-repos/ssb/tests/node_modules/flumeview-level/index.js:170:17 + at /home/cryptix/go-repos/ssb/tests/node_modules/ssb-db/minimal.js:54:7 + at /home/cryptix/go-repos/ssb/tests/node_modules/flumelog-offset/frame/recoverable.js:48:11 + TypeError: Cannot read property 'private' of undefined + at eval (eval at (/home/cryptix/go-repos/ssb/tests/sbot.js:98:3), :11:20) + at /home/cryptix/go-repos/ssb/tests/node_modules/ssb-db/create.js:85:14 + at /home/cryptix/go-repos/ssb/tests/node_modules/flumeview-level/index.js:180:15 + at /home/cryptix/go-repos/ssb/tests/node_modules/ssb-db/minimal.js:52:7 + at /home/cryptix/go-repos/ssb/tests/node_modules/ssb-db/minimal.js:41:35 + at AsyncJobQueue.runAll (/home/cryptix/go-repos/ssb/tests/node_modules/ssb-db/util.js:197:32) + at Array. (/home/cryptix/go-repos/ssb/tests/node_modules/ssb-db/minimal.js:41:22) + at chainMaps (/home/cryptix/go-repos/ssb/tests/node_modules/ssb-db/minimal.js:61:14) + at /home/cryptix/go-repos/ssb/tests/node_modules/flumedb/index.js:119:14 + at /home/cryptix/go-repos/ssb/tests/node_modules/flumelog-offset/inject.js:93:9 + ... +*/ diff --git a/tests/sbot.js b/tests/sbot.js index aa172f08..eaffe450 100644 --- a/tests/sbot.js +++ b/tests/sbot.js @@ -17,13 +17,15 @@ if (testSHSappKey !== false) { const createSbot = theStack({caps: {shs: Buffer.from(testAppkey, 'base64') } }) .use(require('ssb-db')) + .use(require('ssb-backlinks')) + .use(require('ssb-query')) + .use(require('ssb-tribes')) + .use(require('ssb-private1')) .use(require('ssb-gossip')) .use(require('ssb-replicate')) - .use(require('ssb-private')) .use(require('ssb-friends')) .use(require('ssb-blobs')) .use(require('./ggdemo')) - .use(require('ssb-query')) .use(require('ssb-device-address')) .use(require('ssb-identities')) .use(require('ssb-peer-invites'))