Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 4 additions & 6 deletions agentcore/chatlog/projection.go
Original file line number Diff line number Diff line change
Expand Up @@ -119,9 +119,8 @@ var SurfaceProjection = module.ProjectionDefinition{
ID: SurfaceProjectionID, Version: 1,
Consumes: chatlogConsumes,
// Assistant and tool_result entries are projected from Turn facts
// a fork inherits that conversation content.
Inherits: module.InheritAll,
Authoritative: true,
// whose domain defaults preserve conversation content across a fork.
RequiredForCommit: true,
Initial: func() (any, error) {
return Surface{}, nil
},
Expand Down Expand Up @@ -402,9 +401,8 @@ type Context struct {

var ContextProjection = module.ProjectionDefinition{
ID: ContextProjectionID, Version: 1,
Consumes: chatlogConsumes,
Inherits: module.InheritAll,
Authoritative: true,
Consumes: chatlogConsumes,
RequiredForCommit: true,
Initial: func() (any, error) {
return Context{Pending: map[InputID]Input{}, Superseded: map[ToolResultID]ToolResultID{}}, nil
},
Expand Down
11 changes: 6 additions & 5 deletions agentcore/compaction/compaction.go
Original file line number Diff line number Diff line change
Expand Up @@ -152,11 +152,12 @@ var requestBindings = module.BindingReferenceDefinition{
// its parent's in-flight summary execution or acknowledgement responsibility.
var Projection = module.ProjectionDefinition{
ID: ProjectionID, Version: 1,
Consumes: []ledger.EventType{TypeRequested, TypeFinished, TypeAcknowledged},
Authoritative: true,
Initial: func() (any, error) { return State{Operations: map[ID]Operation{}}, nil },
StateCodec: module.JSONStateCodec[State]{},
Apply: apply,
Consumes: []ledger.EventType{TypeRequested, TypeFinished, TypeAcknowledged},
RequiredForCommit: true,
InheritanceOverride: module.InheritNone,
Initial: func() (any, error) { return State{Operations: map[ID]Operation{}}, nil },
StateCodec: module.JSONStateCodec[State]{},
Apply: apply,
}

var Module = module.ModuleDescriptor{
Expand Down
68 changes: 28 additions & 40 deletions agentcore/module/projection.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"encoding/json"
"errors"
"fmt"

"github.com/felinics/twilight/agentcore/ledger"

"github.com/felinics/twilight/agentcore/jsonstable"
Expand All @@ -17,50 +18,37 @@ type ProjectionDefinition struct {
Initial func() (any, error)
Apply func(any, DecodedEvent) (any, error)
StateCodec PayloadCodec
// Inherits decides, per logical stream, what the fold takes from the
// commits a fork inherits. nil follows the lineage each
// stream's domain declared: inherited batches of Inherited domains
// are folded and those of Own domains are skipped, so a
// child never interprets its parent's execution history as its own. A
// projection whose content lives in another module's segment-lineage
// facts declares InheritAll, or names the domains it takes with
// InheritStreams.
Inherits InheritPolicy
// Authoritative marks a projection a module guards its stream's
// invariants with: a provisional commit its fold cannot fold is refused
// before anything is persisted. Every other projection is a derived read
// model: a fold failure marks it unhealthy for the Writer's lifetime and
// never blocks the facts. Write-time invariants belong to
// the Writer's decision, not to projections; an authoritative fold
// InheritanceOverride decides which batches from an inherited prefix this
// projection folds. A nil override follows the owning stream domain's
// Inheritance declaration. InheritAll and InheritNone override it for every
// domain.
InheritanceOverride InheritancePolicy
// RequiredForCommit marks a projection whose fold must accept a
// provisional commit before anything is persisted. Replay failures also
// prevent the Writer from opening. Only trusted core modules may set it.
// Every other projection is a derived read model: a fold failure marks it
// Writer's lifetime and never blocks the facts. Write-time invariants
// belong to the Writer's decision, not to projections; a required fold
// failing is a defect, not a business rejection.
Authoritative bool
RequiredForCommit bool
}

// InheritPolicy decides whether a projection folds the batches of one
// logical stream from a fork's inherited prefix.
type InheritPolicy func(ledger.Domain) bool
// InheritancePolicy selects how a projection reads an inherited prefix for a
// particular stream domain. A nil policy means the domain's declaration is
// used instead.
type InheritancePolicy func(ledger.Domain) bool

// InheritAll folds every batch of inherited commits.
func InheritAll(ledger.Domain) bool { return true }

// InheritStreams folds the listed stream domains of inherited commits.
func InheritStreams(domains ...string) InheritPolicy {
return func(stream ledger.Domain) bool {
for _, d := range domains {
if stream.Name == d {
return true
}
}
return false
}
}
// InheritNone skips every batch of inherited commits.
func InheritNone(ledger.Domain) bool { return false }

// inherits applies the definition's policy; nil follows the lineage the
// stream's domain declared, and a domain no module declared is not
// inherited.
// inherits applies the projection policy. A missing policy follows the
// inheritance declared by the stream's owning module.
func (r *Registry) inherits(d *ProjectionDefinition, stream ledger.Domain) bool {
if d.Inherits != nil {
return d.Inherits(stream)
if d.InheritanceOverride != nil {
return d.InheritanceOverride(stream)
}
_, def, ok := r.LookupStream(stream.Name)
return ok && def.Inheritance == Inherited
Expand Down Expand Up @@ -167,29 +155,29 @@ func (r *Registry) applyEvent(s *ProjectionScope, state any, pos ledger.Position
return next, nil
}

// checkpointDomain namespaces the digest of an authoritative checkpoint.
// checkpointDomain namespaces the digest of a required-for-commit checkpoint.
const checkpointDomain = "twilight/projection-checkpoint"

// sealedCheckpoint is the cache Value of an authoritative projection: the
// sealedCheckpoint is the cache Value of a required-for-commit projection: the
// encoded state and a digest over (projection, version, through, state).
type sealedCheckpoint struct {
State json.RawMessage `json:"state"`
Digest jsonstable.Digest `json:"digest"`
}

// checkpointDigest is the digest an authoritative entry must carry.
// checkpointDigest is the digest a required-for-commit entry must carry.
func checkpointDigest(id ProjectionID, v ProjectionVersion, through ledger.Head, state jsonstable.Value) jsonstable.Digest {
preimage, _ := jsonstable.EncodeTypedPayload(1, checkpointDomain, []string{string(id), fmt.Sprint(uint64(v)), fmt.Sprint(uint64(through.Next)), string(state.Bytes())})
return jsonstable.DigestBytes(preimage)
}

// SealCheckpoint wraps an authoritative projection's encoded state with its
// SealCheckpoint wraps a required-for-commit projection's encoded state with its
// digest for the cache.
func SealCheckpoint(id ProjectionID, v ProjectionVersion, through ledger.Head, state jsonstable.Value) (jsonstable.Value, error) {
return jsonstable.FromValue(sealedCheckpoint{State: state.Bytes(), Digest: checkpointDigest(id, v, through, state)})
}

// OpenCheckpoint unwraps a sealed authoritative entry, verifying its digest;
// OpenCheckpoint unwraps a sealed required-for-commit entry, verifying its digest;
// ok is false when the entry is not a checkpoint of this projection at
// through, and the caller treats it as absent.
func OpenCheckpoint(id ProjectionID, v ProjectionVersion, through ledger.Head, sealed jsonstable.Value) (jsonstable.Value, bool) {
Expand Down
88 changes: 88 additions & 0 deletions agentcore/module/projection_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
package module

import (
"testing"

"github.com/felinics/twilight/agentcore/ledger"
)

func TestProjectionInheritanceOverride(t *testing.T) {
const (
inheritedEvent ledger.EventType = "twilight/history/inherited"
ownEvent ledger.EventType = "twilight/history/own"
)
r, err := BuildRegistry(ModuleDescriptor{
Source: SourceTwilight,
ID: "history",
Streams: []StreamDefinition{
{Domain: "inherited", Inheritance: Inherited},
{Domain: "own", Inheritance: Own},
},
Events: []EventDefinition{
{Type: inheritedEvent, Domain: "inherited", Codecs: map[PayloadVersion]PayloadCodec{1: JSONCodec[struct{}]{}}},
{Type: ownEvent, Domain: "own", Codecs: map[PayloadVersion]PayloadCodec{1: JSONCodec[struct{}]{}}},
},
Projections: []ProjectionDefinition{{
ID: "twilight/history/projection",
Version: 1,
Consumes: []ledger.EventType{inheritedEvent, ownEvent},
Initial: func() (any, error) { return []string{}, nil },
Apply: func(state any, event DecodedEvent) (any, error) {
return append(state.([]string), event.Domain.Name), nil
},
StateCodec: JSONStateCodec[[]string]{},
}},
})
if err != nil {
t.Fatal(err)
}
scope, err := r.ScopeFor("twilight/history/projection", 1)
if err != nil {
t.Fatal(err)
}

inheritedPayload, err := r.Encode(inheritedEvent, struct{}{})
if err != nil {
t.Fatal(err)
}
inherited := ledger.Event{Type: inheritedEvent, Payload: inheritedPayload}
ownPayload, err := r.Encode(ownEvent, struct{}{})
if err != nil {
t.Fatal(err)
}
own := ledger.Event{Type: ownEvent, Payload: ownPayload}
commits := []ledger.Commit{{
Seq: 0,
Batches: []ledger.EventBatch{
{Domain: ledger.Domain{Name: "inherited"}, Events: []ledger.Event{inherited}},
{Domain: ledger.Domain{Name: "own"}, Events: []ledger.Event{own}},
},
}}

for _, tc := range []struct {
name string
policy InheritancePolicy
expected []string
}{
{name: "domain defaults", expected: []string{"inherited"}},
{name: "inherit all", policy: InheritAll, expected: []string{"inherited", "own"}},
{name: "inherit none", policy: InheritNone},
} {
t.Run(tc.name, func(t *testing.T) {
scope.Def.InheritanceOverride = tc.policy
state, err := r.FoldFrom(scope, []string{}, commits, ledger.Head{Next: 1})
if err != nil {
t.Fatal(err)
}
got := state.([]string)
if len(got) != len(tc.expected) {
t.Fatalf("folded domains = %v, want %v", got, tc.expected)
}
for i := range got {
if got[i] != tc.expected[i] {
t.Fatalf("folded domains = %v, want %v", got, tc.expected)
}
}
})
}
}
14 changes: 7 additions & 7 deletions agentcore/module/registry.go
Original file line number Diff line number Diff line change
Expand Up @@ -182,16 +182,16 @@ func validSegment(kind, v string) error {

// BuildRegistry validates the module set and freezes the indexes. Every
// module passed here is trusted: the caller vouches for it, so it may
// declare authoritative projections. Modules from outside the
// declare required-for-commit projections. Modules from outside the
// deployment's trust boundary go through BuildRegistryWithExtensions.
func BuildRegistry(modules ...ModuleDescriptor) (*Registry, error) {
return BuildRegistryWithExtensions(modules, nil)
}

// BuildRegistryWithExtensions builds a registry from trusted core modules
// and untrusted extensions. Trust is a property of how a module reached the
// registry, not of what its descriptor says: an extension may not declare an
// authoritative projection and may not use the SourceTwilight source, so no
// registry, not of what its descriptor says: an extension may not declare a
// required-for-commit projection and may not use the SourceTwilight source, so no
// descriptor can claim the first-party capability for itself.
func BuildRegistryWithExtensions(core, extensions []ModuleDescriptor) (*Registry, error) {
modules := make([]ModuleDescriptor, 0, len(core)+len(extensions))
Expand All @@ -206,8 +206,8 @@ func BuildRegistryWithExtensions(core, extensions []ModuleDescriptor) (*Registry
return nil, &ledger.Error{Code: ledger.CodeInvalid, Detail: fmt.Sprintf("extension module %q claims the %s source", m.ID, SourceTwilight)}
}
for _, p := range m.Projections {
if p.Authoritative {
return nil, &ledger.Error{Code: ledger.CodeInvalid, Detail: fmt.Sprintf("projection %q of extension %s/%s declares Authoritative; only trusted core modules may", p.ID, m.Source, m.ID)}
if p.RequiredForCommit {
return nil, &ledger.Error{Code: ledger.CodeInvalid, Detail: fmt.Sprintf("projection %q of extension %s/%s declares RequiredForCommit; only trusted core modules may", p.ID, m.Source, m.ID)}
}
}
modules = append(modules, *m)
Expand Down Expand Up @@ -253,8 +253,8 @@ func BuildRegistryWithExtensions(core, extensions []ModuleDescriptor) (*Registry
// Refusing commits is a capability of trusted core modules, not
// something a descriptor declares for itself; the
// extension path above already refused it, this guards the map.
if p.Authoritative && !trusted[key] {
return nil, &ledger.Error{Code: ledger.CodeInvalid, Detail: fmt.Sprintf("projection %q of module %s/%s declares Authoritative without trust", p.ID, m.Source, m.ID)}
if p.RequiredForCommit && !trusted[key] {
return nil, &ledger.Error{Code: ledger.CodeInvalid, Detail: fmt.Sprintf("projection %q of module %s/%s declares RequiredForCommit without trust", p.ID, m.Source, m.ID)}
}
k := projectionKey{p.ID, p.Version}
if _, dup := r.projections[k]; dup {
Expand Down
14 changes: 7 additions & 7 deletions agentcore/module/registry_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -275,25 +275,25 @@ func TestRegistryMultiVersionCodecsCoexist(t *testing.T) {
}
}

// Authoritative is a capability of trusted core modules. An
// RequiredForCommit is a capability of trusted core modules. An
// extension cannot declare it, nor claim the first-party source to get it.
func TestExtensionsCannotBeAuthoritative(t *testing.T) {
func TestExtensionsCannotDeclareRequiredForCommit(t *testing.T) {
core := ModuleDescriptor{Source: SourceTwilight, ID: "core", Projections: []ProjectionDefinition{{
ID: "twilight/core/p", Version: 1, Authoritative: true,
ID: "twilight/core/p", Version: 1, RequiredForCommit: true,
Initial: func() (any, error) { return struct{}{}, nil }, Apply: func(s any, _ DecodedEvent) (any, error) { return s, nil }, StateCodec: JSONStateCodec[struct{}]{}}}}
ext := ModuleDescriptor{Source: "acme", ID: "plugin", Projections: []ProjectionDefinition{{
ID: "acme/plugin/p", Version: 1, Authoritative: true,
ID: "acme/plugin/p", Version: 1, RequiredForCommit: true,
Initial: func() (any, error) { return struct{}{}, nil }, Apply: func(s any, _ DecodedEvent) (any, error) { return s, nil }, StateCodec: JSONStateCodec[struct{}]{}}}}
if _, err := BuildRegistryWithExtensions([]ModuleDescriptor{core}, []ModuleDescriptor{ext}); err == nil {
t.Fatal("extension declared an authoritative projection")
t.Fatal("extension declared a required-for-commit projection")
}
impostor := ext
impostor.Source = SourceTwilight
impostor.Projections[0].Authoritative = false
impostor.Projections[0].RequiredForCommit = false
if _, err := BuildRegistryWithExtensions([]ModuleDescriptor{core}, []ModuleDescriptor{impostor}); err == nil {
t.Fatal("extension claimed the twilight source")
}
ext.Projections[0].Authoritative = false
ext.Projections[0].RequiredForCommit = false
if _, err := BuildRegistryWithExtensions([]ModuleDescriptor{core}, []ModuleDescriptor{ext}); err != nil {
t.Fatalf("derived extension refused: %v", err)
}
Expand Down
8 changes: 4 additions & 4 deletions agentcore/session/projection.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ type ProjectionReader interface {

// ProjectionCache is the optional derived cache. Entries may be
// lost or stale at any time; readers verify Through against the stream.
// An Authoritative projection's entry is a checkpoint the Writer plans the
// A required-for-commit projection's entry is a checkpoint the Writer plans the
// next commit: its state bytes travel with the digest
// the Writer computed when it saved them, in the same Value (see
// module.SealCheckpoint), and a reader that finds the digest missing or
Expand Down Expand Up @@ -101,7 +101,7 @@ func SaveProjection(ctx context.Context, cache ProjectionCache, registry *module
if err != nil {
return err
}
if def.Authoritative {
if def.RequiredForCommit {
if encoded, err = module.SealCheckpoint(id, v, through, encoded); err != nil {
return err
}
Expand All @@ -110,14 +110,14 @@ func SaveProjection(ctx context.Context, cache ProjectionCache, registry *module
}

// LoadProjectionEntry reads a cache entry for def, unsealing and verifying
// an authoritative one; an entry that does not verify is a
// a required-for-commit one; an entry that does not verify is a
// miss.
func LoadProjectionEntry(ctx context.Context, cache ProjectionCache, def *module.ProjectionDefinition, sid SessionID) (jsonstable.Value, ledger.Head, bool, error) {
encoded, through, ok, err := cache.Load(ctx, sid, def.ID, def.Version)
if err != nil || !ok {
return jsonstable.Value{}, ledger.Head{}, false, err
}
if def.Authoritative {
if def.RequiredForCommit {
state, verified := module.OpenCheckpoint(def.ID, def.Version, through, encoded)
if !verified {
return jsonstable.Value{}, ledger.Head{}, false, nil
Expand Down
12 changes: 6 additions & 6 deletions agentcore/session/writer/cache_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -391,18 +391,18 @@ func TestWriterWithoutCacheFoldsEverything(t *testing.T) {
}
}

// an authoritative projection's entry is a checkpoint sealed
// a required-for-commit projection's entry is a checkpoint sealed
// with a digest over its identity, position and state. A well-formed entry
// whose state was not what the Writer saved (here: a derived projection's
// entry copied under the authoritative one) is a miss, and the Writer
// entry copied under the required-for-commit one) is a miss, and the Writer
// refolds from the log; a derived projection's entry stays a plain state.
func TestAuthoritativeEntryMustVerify(t *testing.T) {
func TestRequiredForCommitEntryMustVerify(t *testing.T) {
ctx := context.Background()
counter := newApplyCounter()
typ := tpfx("k") + "row"
mk := func(id module.ProjectionID, authoritative bool) module.ProjectionDefinition {
mk := func(id module.ProjectionID, requiredForCommit bool) module.ProjectionDefinition {
return module.ProjectionDefinition{
ID: id, Version: 1, Consumes: []ledger.EventType{typ}, Authoritative: authoritative,
ID: id, Version: 1, Consumes: []ledger.EventType{typ}, RequiredForCommit: requiredForCommit,
Initial: func() (any, error) { return noteState{}, nil },
Apply: func(state any, e module.DecodedEvent) (any, error) {
counter.inc(id)
Expand All @@ -429,7 +429,7 @@ func TestAuthoritativeEntryMustVerify(t *testing.T) {
if err := w.Close(ctx); err != nil {
t.Fatal(err)
}
// Both entries are at head; the authoritative one is sealed.
// Both entries are at head; the required-for-commit one is sealed.
sealed, through, ok, err := f.cache.Load(ctx, "s", alphaID, 1)
if err != nil || !ok || through.Next != 2 {
t.Fatalf("alpha entry = ok:%v through:%d %v", ok, through.Next, err)
Expand Down
Loading
Loading