Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 38 additions & 7 deletions internal/dotsync/dotsync_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package dotsync

import (
"bytes"
"encoding/base64"
"encoding/json"
"fmt"
"io"
Expand Down Expand Up @@ -763,10 +764,21 @@ func formatKind(data []byte) string {
case f.values == nil:
return "age" // entirely ciphertext
default:
return "sops" // encrypted values
return "values" // encrypted values in a readable file
}
}

// ageFile is a minimal age file: version line, one recipient stanza, header MAC, payload.
var ageFile = "age-encryption.org/v1\n-> X25519 abc\nZGVm\n--- bWFj\n\x00\xff\x10\x80"

// zstdRaw wraps data in a zstd frame with a single raw block, as zstd stores incompressible data.
func zstdRaw(data string) string {
n := len(data)<<3 | 1 // last block, type raw
return "\x28\xb5\x2f\xfd\x00\x58" + string([]byte{byte(n), byte(n >> 8), byte(n >> 16)}) + data
}

func b64(s string) string { return base64.StdEncoding.EncodeToString([]byte(s)) }

func TestEncryptedFiles(t *testing.T) {
w := newWorld(t)
w.machine("alpha").activate()
Expand All @@ -777,23 +789,36 @@ func TestEncryptedFiles(t *testing.T) {
enc string
blocked bool
}{
{".config/mise/.env.yaml", sopsYAML, "sops", false},
{".config/mise/.env.json", sopsJSON, "sops", false},
{"app/.env.json", `{"k":"ENC[AES256_GCM,data:eA==,iv:A,tag:B,type:str]","sops":{"mac":"ENC[AES256_GCM,data:bQ==,iv:A,tag:B,type:str]"}}`, "sops", false},
{".env", "API_KEY=ENC[AES256_GCM,data:eA==,iv:A,tag:B,type:str]\nsops_mac=ENC[AES256_GCM,data:bQ==,iv:A,tag:B,type:str]\n", "sops", false},
{".config/mise/.env.yaml", sopsYAML, "values", false},
{".config/mise/.env.json", sopsJSON, "values", false},
{"app/.env.json", `{"k":"ENC[AES256_GCM,data:eA==,iv:A,tag:B,type:str]","sops":{"mac":"ENC[AES256_GCM,data:bQ==,iv:A,tag:B,type:str]"}}`, "values", false},
{".env", "API_KEY=ENC[AES256_GCM,data:eA==,iv:A,tag:B,type:str]\nsops_mac=ENC[AES256_GCM,data:bQ==,iv:A,tag:B,type:str]\n", "values", false},
{"secrets/prod.env.age", ageArmored, "age", false},
{"secrets/prod.env.age", "age-encryption.org/v1\n-> X25519 abc\nZGVm\n--- bWFj\n\x00\xff", "age", false},
// The age header alone isn't enough: plaintext after it is scanned (and the path refused).
{"secrets/prod.env.age", "age-encryption.org/v1\npassword = hunter2!xyz\n", "", true},
// In sops files only the ENC[…] values are skipped: a plaintext secret on the same line
// (e.g. a minified JSON file) is still found.
{".config/mise/.env.json", `{"k":"ENC[AES256_GCM,data:eA==,iv:A,tag:B,type:str]","token":"ghp_` + strings.Repeat("a", 36) + `","sops":{"mac":"ENC[AES256_GCM,data:bQ==,iv:A,tag:B,type:str]"}}`, "sops", true},
{".config/mise/.env.json", `{"k":"ENC[AES256_GCM,data:eA==,iv:A,tag:B,type:str]","token":"ghp_` + strings.Repeat("a", 36) + `","sops":{"mac":"ENC[AES256_GCM,data:bQ==,iv:A,tag:B,type:str]"}}`, "values", true},
// sops leaves keys and comments readable; those are still scanned.
{".config/mise/.env.yaml", "# token: " + "ghp_" + strings.Repeat("a", 36) + "\n" + sopsYAML, "sops", true},
{".config/mise/.env.yaml", "# token: " + "ghp_" + strings.Repeat("a", 36) + "\n" + sopsYAML, "values", true},
// ENC[…] lines are only skipped in real sops files.
{"notes.txt", "ENC[AES256_GCM,data:x] token: ghp_" + strings.Repeat("a", 36) + "\n", "", true},
// Plaintext between age armor lines isn't ciphertext.
{"secrets/prod.env.age", "-----BEGIN AGE ENCRYPTED FILE-----\npassword = hunter2!xyz\n-----END AGE ENCRYPTED FILE-----\n", "", true},
// age values embedded in a readable file, plain or zstd-compressed (mise's two formats), and
// in other formats: the path rules don't apply, and neither do the content rules to the value.
{".config/mise/conf.d/secrets.toml", "[env]\nGITHUB_TOKEN = { age = \"" + b64(ageFile) + "\" }\n", "values", false},
{".config/mise/conf.d/secrets.toml", "[env]\nDB_PASSWORD = { age = { value = \"" + b64(zstdRaw(ageFile)) + "\", format = \"zstd\" } }\n", "values", false},
{".config/app/.env", "API_TOKEN=" + strings.TrimRight(b64(ageFile), "=") + "\n", "values", false},
{".config/app/config.yaml", "api:\n token: '" + b64(ageFile) + "'\n", "values", false},
// Everything else in such a file is still scanned, on other lines and on the same line.
{".config/mise/config.toml", "[env]\nA = { age = \"" + b64(ageFile) + "\" }\nB = \"ghp_" + strings.Repeat("a", 36) + "\"\n", "values", true},
{".config/app/config.json", `{"a": {"age": "` + b64(ageFile) + `"}, "password": "hunter2!xyz"}`, "values", true},
// base64 that isn't age ciphertext is plaintext as far as the rules are concerned.
{".config/app/config.yaml", "token: '" + b64("hunter2hunter2hunter2hunter2hunter2") + "'\n", "", true},
{".config/app/config.yaml", "token: '" + b64(zstdRaw("hunter2hunter2hunter2hunter2hunter2")) + "'\n", "", true},
{".config/app/config.yaml", "token: '" + b64("age-encryption.org/v1\npassword = hunter2!xyz\n") + "'\n", "", true},
// Unencrypted files named like secrets are still refused.
{".config/mise/.env.json", `{"API_KEY": "abc"}`, "", true},
} {
Expand All @@ -813,6 +838,8 @@ func TestEncryptedSecretsSyncInManagedDirectory(t *testing.T) {
a.init()
a.write(".config/mise/config.toml", "[env]\n_.file = \".env.yaml\"\n")
a.write(".config/mise/.env.yaml", sopsYAML)
ageValues := "[env]\nDB_PASSWORD = { age = \"" + b64(ageFile) + "\" }\n"
a.write(".config/mise/conf.d/secrets.toml", ageValues)
a.write(".config/mise/age.txt", fakeAgeKey+"\n")
out := a.ok("add", a.path(".config/mise"))
if !strings.Contains(out, "1 file(s) under ~/.config/mise look like secrets") || !strings.Contains(out, "age.txt") {
Expand All @@ -821,9 +848,13 @@ func TestEncryptedSecretsSyncInManagedDirectory(t *testing.T) {
if w.remoteFile("mise/.env.yaml") != sopsYAML {
t.Fatal("sops file was not sent")
}
if w.remoteFile("mise/conf.d/secrets.toml") != ageValues {
t.Fatal("file with age values was not sent")
}

b.init()
b.expect(".config/mise/.env.yaml", sopsYAML)
b.expect(".config/mise/conf.d/secrets.toml", ageValues)
if b.exists(".config/mise/age.txt") {
t.Fatal("age key reached another machine")
}
Expand Down
73 changes: 66 additions & 7 deletions internal/dotsync/secrets.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package dotsync

import (
"bytes"
"encoding/base64"
"fmt"
"path/filepath"
"regexp"
Expand Down Expand Up @@ -71,7 +72,7 @@ func secretContentReason(data []byte) string {
continue
}
if f != nil && f.values != nil {
line = f.values.ReplaceAll(line, nil)
line = f.values(line)
}
for _, r := range secretContent {
if r.re.Match(line) {
Expand Down Expand Up @@ -105,16 +106,14 @@ func secretReason(path string, o *Obj) string {
// cipherFormat recognises an encrypted file format.
type cipherFormat struct {
detect func(data []byte) bool
values *regexp.Regexp // encrypted values in an otherwise readable file; nil when the whole file is ciphertext
values func(line []byte) []byte // removes the encrypted values from a line of an otherwise readable file; nil when the whole file is ciphertext
}

// cipherFormats are the encrypted formats dotsync recognises. They're data: the code above
// never refers to a format by name.
// never refers to a format by name, and none of them names the tool that writes it.
var cipherFormats = []cipherFormat{
// age, binary: the version line, recipient stanzas, then the header MAC line ("--- …").
{detect: func(d []byte) bool {
return bytes.HasPrefix(d, []byte("age-encryption.org/v1\n")) && bytes.Contains(d[:min(len(d), 64<<10)], []byte("\n--- "))
}},
{detect: isAgeFile},
// age, ASCII armor: nothing but base64 between the armor lines.
{detect: func(d []byte) bool {
return armored(d, []byte("-----BEGIN AGE ENCRYPTED FILE-----"), []byte("-----END AGE ENCRYPTED FILE-----"))
Expand All @@ -123,10 +122,70 @@ var cipherFormats = []cipherFormat{
// encrypted individually.
{
detect: regexp.MustCompile(`(?m)(?:"mac"\s*:\s*"|^\s*mac:\s*|^\s*mac\s*=\s*"?|^sops_mac=)ENC\[AES256_GCM,`).Match,
values: regexp.MustCompile(`ENC\[AES256_GCM,[^\]]*\]`),
values: removeAll(regexp.MustCompile(`ENC\[AES256_GCM,[^\]]*\]`)),
},
// age values in a readable file (TOML, YAML, JSON, dotenv…): base64 of an age file, optionally
// zstd-compressed, as mise's age-encrypted environment variables are stored.
{
detect: func(d []byte) bool {
for _, m := range base64Run.FindAll(d, -1) {
if isAgeValue(m) {
return true
}
}
return false
},
values: func(line []byte) []byte {
return base64Run.ReplaceAllFunc(line, func(m []byte) []byte {
if isAgeValue(m) {
return nil
}
return m
})
},
},
}

func removeAll(re *regexp.Regexp) func([]byte) []byte {
return func(line []byte) []byte { return re.ReplaceAll(line, nil) }
}

var ageHeader = []byte("age-encryption.org/v1\n")

// isAgeFile reports whether d is age ciphertext: the version line, then a header ending in its
// MAC line.
func isAgeFile(d []byte) bool {
return bytes.HasPrefix(d, ageHeader) && bytes.Contains(d[:min(len(d), 64<<10)], []byte("\n--- "))
}

// base64Run matches a whole run of base64 long enough to hold an age header; a leftmost greedy
// match always starts and ends at the run's boundaries.
var base64Run = regexp.MustCompile(`[A-Za-z0-9+/]{40,}={0,2}`)

var zstdMagic = []byte{0x28, 0xb5, 0x2f, 0xfd}

// isAgeValue reports whether b64 decodes to an age file, or to a zstd frame holding one.
// Ciphertext doesn't compress, so zstd stores it in a raw block: the age file then starts right
// after the frame and block headers (at most 21 bytes), unchanged.
func isAgeValue(b64 []byte) bool {
enc := base64.StdEncoding
if !bytes.HasSuffix(b64, []byte("=")) && len(b64)%4 != 0 {
enc = base64.RawStdEncoding
}
d := make([]byte, enc.DecodedLen(len(b64)))
n, err := enc.Decode(d, b64)
if err != nil {
return false
}
d = d[:n]
if bytes.HasPrefix(d, zstdMagic) {
if i := bytes.Index(d[:min(len(d), 32)], ageHeader); i > 0 {
d = d[i:]
}
}
return isAgeFile(d)
}

func cipherFormatOf(data []byte) *cipherFormat {
for i := range cipherFormats {
if cipherFormats[i].detect(data) {
Expand Down
2 changes: 1 addition & 1 deletion website/src/content/docs/concepts/security-model.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ description: What dotsync trusts, what it defends against and what it doesn't, w

Encryption means getting a key onto every machine. That key becomes a new secret to bootstrap, rotate and lose, and a single point of compromise. Keeping secrets **out** of the repository is simpler, and fails safe. Use a password manager or the OS keychain for secrets, and let dotsync handle configuration.

dotsync doesn't encrypt anything itself, but it doesn't stand in the way if **you** do. Files already encrypted with [age](https://age-encryption.org) or [sops](https://getsops.io), for example by mise, are ciphertext, so they sync like any other file. Getting the key to each machine is then up to you, and dotsync keeps its side of it: it refuses to sync age private keys, by path and by content. It recognizes encrypted **file formats**, not the tools that write them, so this works whichever tool you use. See [encrypted files](/dotsync/reference/secret-rules/#encrypted-files).
dotsync doesn't encrypt anything itself, but it doesn't stand in the way if **you** do. Files encrypted with [age](https://age-encryption.org) or [sops](https://getsops.io), and age-encrypted values such as mise writes into its configuration, are ciphertext, so they sync like any other file. Getting the key to each machine is then up to you, and dotsync keeps its side of it: it refuses to sync age private keys, by path and by content. It recognizes encrypted **formats**, not the tools that write them, so this works whichever tool you use. See [Encrypt secrets that should sync](/dotsync/guides/secrets/#encrypt-secrets-that-should-sync).

## What dotsync sends, and to whom

Expand Down
22 changes: 13 additions & 9 deletions website/src/content/docs/guides/mise.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -98,18 +98,19 @@ Create `~/.config/mise/local.env` on each machine. The `--ignore '*.env'` above

### Encrypt them with age

mise can store an age-encrypted value directly in the config (an experimental feature at the time of writing):
This is the simplest way to sync secrets, if you already use mise. mise can store an age-encrypted value directly in the config. It's an experimental feature at the time of writing, so turn those on first:

```sh
mise set --age-encrypt --prompt DB_PASSWORD
mise settings experimental=true
mise set -g --age-encrypt --prompt DB_PASSWORD
```

```toml title="~/.config/mise/config.toml"
[env]
DB_PASSWORD = { age = { value = "<base64>" } }
DB_PASSWORD = { age = "<base64>" }
```

Only ciphertext reaches the repository, so the file syncs without any exceptions.
Long values are compressed and stored as `{ age = { value = "<base64>", format = "zstd" } }`. dotsync recognizes both forms as ciphertext, so the file syncs without any exceptions. That's true even for a file named like a secret, such as `conf.d/secrets.toml`. The rest of the file is still checked. [Encrypt secrets that should sync](/dotsync/guides/secrets/#encrypt-secrets-that-should-sync) covers the same approach with other tools.

### Encrypted files with sops

Expand All @@ -124,18 +125,21 @@ dotsync recognizes sops files and syncs them even though the name looks like a s

### The age key

Both approaches depend on an age private key, by default `~/.config/mise/age.txt`. mise can also use your SSH key. Anyone with that key and a copy of the repository can read every secret, so it has to stay off the repository. dotsync refuses it by path (`.config/mise/age.txt`, `.config/sops/age/*`) and by content (`AGE-SECRET-KEY-1…`).
Both approaches depend on an age private key, by default `~/.config/mise/age.txt`. Create it once with `age-keygen -o ~/.config/mise/age.txt` (install age with `mise use -g age`). Anyone with that key and a copy of the repository can read every secret, so it has to stay off the repository. dotsync refuses it by path (`.config/mise/age.txt`, `.config/sops/age/*`) and by content (`AGE-SECRET-KEY-1…`).

Copy it to each new machine yourself, once. dotsync never syncs it, and it doesn't check for it either: until the key is there, the encrypted values simply show up as missing in `mise env`.
Use **one key on all your machines**. mise can also decrypt with each machine's SSH key, but then every secret has to be re-encrypted for each new machine. Keep a copy of the key in your password manager, and copy it to each new machine yourself, once:

<Steps>

1. On a machine that has the key: `cat ~/.config/mise/age.txt`, or store it in your password manager.
2. On the new machine: paste it into `~/.config/mise/age.txt` and `chmod 600 ~/.config/mise/age.txt`.
3. Check it works: `mise env` should now list `DB_PASSWORD`.
1. On the new machine, paste the key from your password manager into `~/.config/mise/age.txt`, then `chmod 600 ~/.config/mise/age.txt`.
2. Check it works: `mise env` should now list `DB_PASSWORD`.

</Steps>

Until the key is there, mise reports an error each time it fails to decrypt a value. If you'd rather it skipped those values quietly, for example on a machine that should never have them, run `mise settings age.strict=false`.

If the key leaks, rotate every secret it protected: old ciphertext stays in the repository's history.

## mise's own dotfiles

mise can also manage dotfiles itself, with `mise dot` and a `[dotfiles]` section in its config. It links, copies or templates files from a source directory, and it can *track* files where they are, keeping their history in git. With the history watcher and `history.sync = "sync"`, it syncs tracked files between machines too. See [mise's dotfiles docs](https://mise.jdx.dev/dotfiles.html), and the [comparison](/dotsync/concepts/comparison/) for how it differs from dotsync.
Expand Down
Loading
Loading