dnproto/src/repo/Repo.cs at main · threddyrex/dnproto · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132

namespace dnproto.repo;

public class Repo
{
    /// ----------------------------------------------------------------------------------------------------
    /// CAR format
    /// ----------------------------------------------------------------------------------------------------
    ///
    /// Let's start by describing CAR.
    ///
    /// CAR Specs:
    ///     https://ipld.io/specs/transport/car/carv1/
    ///     https://ipld.io/specs/codecs/dag-cbor/spec/
    ///
    /// A CAR contains one header, followed by a set of data items:
    ///
    ///     [---  header  -------- ]   [----------------- data ---------------------------------]
    ///     [varint | header block ]   [varint | cid | data block]....[varint | cid | data block]
    ///
    /// We represent this in dnproto this using the VarInt, CidV1, and DagCborObject classes:
    ///
    ///     [---  header  -------- ]   [----------------- data -------------------------------------------]
    ///     [VarInt | DagCborObject]   [VarInt | CidV1 | DagCborObject]....[VarInt | CidV1 | DagCborObject]
    ///
    /// and then wrap them up in RepoHeader and RepoRecord classes:
    ///
    ///     [---  header  -------- ]   [----------------- data -------------------------------------------]
    ///     [RepoHeader]               [RepoRecord]....[RepoRecord][RepoRecord][RepoRecord][RepoRecord]
    ///
    /// The "WalkRepo" function returns those RepoHeader and RepoRecord classes.
    ///
    /// ----------------------------------------------------------------------------------------------------


    /// ----------------------------------------------------------------------------------------------------
    /// atproto repo format
    /// ----------------------------------------------------------------------------------------------------
    ///
    /// atproto uses CAR. atproto repo spec:
    ///     https://atproto.com/specs/repository
    ///
    /// Each RepoRecord data item is either a RepoCommmit, MstNode,
    /// or atproto record (RepoRecord). Look for the IsRepoCommit()
    /// IsAtProtoRecord(), and IsMstNode() functions.
    ///
    /// An atproto repo has the following format:
    ///
    ///     RepoHeader.cs (only 1 in the repo)
    ///         "roots" -> CidV1 RepoCommitCid (points to RepoCommit.cs)
    ///         "version" -> Int Version
    ///
    ///     RepoCommit.cs (only 1 in the repo)
    ///         CidV1 Cid;
    ///         "did" -> string Did (user's did)
    ///         "rev" -> string Rev (increases monotonically, typically timestamp)
    ///         "sig" -> string Signature (computed each time repo changes, from the private key)
    ///         "data" -> CidV1 RootMstNodeCid (points to root MstNode cid)
    ///         "prev" -> CidV1? PrevMstNodeCid (usually null)
    ///         "version" -> int Version (always 3)
    ///
    ///     MstNode.cs (1 or more)
    ///         CidV1 Cid
    ///         int KeyDepth (depth of this node in the tree)
    ///         "e" -> List<MstEntry> Entries
    ///         "l" -> MstNode? LeftTree (optional to a sub-tree MstNode)
    ///
    ///     MstEntry.cs (1 or more)
    ///         string Key (the full key, built from the collection, suffix, and prefix length)
    ///         string Value (the cid of the repo record, in base32)
    ///         "k" -> (key suffix)
    ///         "p" -> (prefix length)
    ///         "t" -> MstNode? RightTree (optional to a sub-tree MstNode)
    ///         "v" -> (cid of repo record, see "Value" above)
    ///
    ///     RepoRecord.cs (1 or more)
    ///         CidV1 Cid
    ///         DagCborObject Data (the actual atproto record)
    ///
    ///
    /// (Notes)
    ///     The three Repo* classes are in the "dnproto.repo" namespace.
    ///     The two Mst* classes are in the "dnproto.mst" namespace.
    ///     The "dnproto.mst" namespace is somewhat generic, so RepoMst.cs exists to bridge that gap.
    ///     Items in quotes are the field names in the dag-cbor objects.
    ///
    /// ----------------------------------------------------------------------------------------------------


    /// <summary>
    ///
    /// WalkRepo
    ///
    /// Start at beginning and read through entire repo.
    /// Use callbacks to let caller know when we find things.
    /// This is just the top-level algorithm. Most of the heavy lifting
    /// is done in VarInt, CidV1, and DagCborObject.
    ///
    /// </summary>
    /// <param name="s"></param>
    /// <param name="headerCallback"></param>
    /// <param name="recordCallback"></param>
    public static void WalkRepo(Stream s, Func<RepoHeader, bool> headerCallback, Func<RepoRecord, bool> recordCallback)
    {
        if (s == null) return;
        if (s.Length == 0) return;

        // Read header
        var repoHeader = RepoHeader.ReadFromStream(s);
        bool keepGoing = headerCallback(repoHeader);

        while (s.Position < s.Length && keepGoing)
        {
            // Read data block (record)
            // (this could wind up being either a MstNode, RepoCommit, or atproto RepoRecord)
            var repoRecord = RepoRecord.ReadFromStream(s);
            keepGoing = recordCallback(repoRecord);
        }
    }


    public static void WalkRepo(string repoFile, Func<RepoHeader, bool> headerCallback, Func<RepoRecord, bool> recordCallback)
    {
        if (string.IsNullOrEmpty(repoFile)) return;
        using (var fs = new FileStream(repoFile, FileMode.Open))
        {
            WalkRepo(fs, headerCallback, recordCallback);
        }
    }
}