From e69a85769f008a33119eda85d3face948afe7636 Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Sun, 29 Aug 2021 19:43:26 +0200 Subject: [PATCH 01/59] go mod: upgrade go-fuse to fix darwin build failure Upgraded using go get -u github.com/hanwen/go-fuse/v2@master to get https://github.com/hanwen/go-fuse/commit/61df81086038f52fbf3e25ff97373336c83cd3ae Fixes https://github.com/rfjakob/gocryptfs/issues/597 --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 980ce59..29f7c2c 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/rfjakob/gocryptfs/v2 go 1.16 require ( - github.com/hanwen/go-fuse/v2 v2.1.1-0.20210825070001-74a933d6e856 + github.com/hanwen/go-fuse/v2 v2.1.1-0.20210825171523-3ab5d95a30ae github.com/jacobsa/crypto v0.0.0-20190317225127-9f44e2d11115 github.com/jacobsa/oglematchers v0.0.0-20150720000706-141901ea67cd // indirect github.com/jacobsa/oglemock v0.0.0-20150831005832-e94d794d06ff // indirect diff --git a/go.sum b/go.sum index 3d3fbf0..9131acc 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/hanwen/go-fuse/v2 v2.1.1-0.20210825070001-74a933d6e856 h1:rQb7H5igQ2oIeT+Ul1UtIsGhUiSGeCoyLg84otnHdXU= github.com/hanwen/go-fuse/v2 v2.1.1-0.20210825070001-74a933d6e856/go.mod h1:B1nGE/6RBFyBRC1RRnf23UpwCdyJ31eukw34oAKukAc= +github.com/hanwen/go-fuse/v2 v2.1.1-0.20210825171523-3ab5d95a30ae h1:4CB6T4YTUVvnro5ba8ju1QCbOuyGAeF3vvKlo50EJ4k= +github.com/hanwen/go-fuse/v2 v2.1.1-0.20210825171523-3ab5d95a30ae/go.mod h1:B1nGE/6RBFyBRC1RRnf23UpwCdyJ31eukw34oAKukAc= github.com/jacobsa/crypto v0.0.0-20190317225127-9f44e2d11115 h1:YuDUUFNM21CAbyPOpOP8BicaTD/0klJEKt5p8yuw+uY= github.com/jacobsa/crypto v0.0.0-20190317225127-9f44e2d11115/go.mod h1:LadVJg0XuawGk+8L1rYnIED8451UyNxEMdTWCEt5kmU= github.com/jacobsa/oglematchers v0.0.0-20150720000706-141901ea67cd h1:9GCSedGjMcLZCrusBZuo4tyKLpKUPenUUqi34AkuFmA= From b83ca9c921019fc3b790dabb6198bb77ef2f9a34 Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Mon, 30 Aug 2021 09:39:57 +0200 Subject: [PATCH 02/59] Remove serialize_reads package Will be replaced by go-fuse's new SyncRead flag. More info: https://github.com/hanwen/go-fuse/issues/395 SyncRead commit: https://github.com/hanwen/go-fuse/commit/15a8bb029a4e1a51e10043c370970596b1fbb737 --- internal/fusefrontend/file.go | 8 +- internal/fusefrontend/root_node.go | 4 - internal/serialize_reads/sr.go | 150 ----------------------------- 3 files changed, 1 insertion(+), 161 deletions(-) delete mode 100644 internal/serialize_reads/sr.go diff --git a/internal/fusefrontend/file.go b/internal/fusefrontend/file.go index 9481abf..6f33a60 100644 --- a/internal/fusefrontend/file.go +++ b/internal/fusefrontend/file.go @@ -20,7 +20,6 @@ import ( "github.com/rfjakob/gocryptfs/v2/internal/contentenc" "github.com/rfjakob/gocryptfs/v2/internal/inomap" "github.com/rfjakob/gocryptfs/v2/internal/openfiletable" - "github.com/rfjakob/gocryptfs/v2/internal/serialize_reads" "github.com/rfjakob/gocryptfs/v2/internal/stupidgcm" "github.com/rfjakob/gocryptfs/v2/internal/syscallcompat" "github.com/rfjakob/gocryptfs/v2/internal/tlog" @@ -252,13 +251,7 @@ func (f *File) Read(ctx context.Context, buf []byte, off int64) (resultData fuse defer f.fileTableEntry.ContentLock.RUnlock() tlog.Debug.Printf("ino%d: FUSE Read: offset=%d length=%d", f.qIno.Ino, off, len(buf)) - if f.rootNode.args.SerializeReads { - serialize_reads.Wait(off, len(buf)) - } out, errno := f.doRead(buf[:0], uint64(off), uint64(len(buf))) - if f.rootNode.args.SerializeReads { - serialize_reads.Done() - } if errno != 0 { return nil, errno } @@ -389,6 +382,7 @@ func (f *File) Write(ctx context.Context, data []byte, off int64) (uint32, sysca // But if the write directly follows an earlier write, it cannot create a // hole, and we can save one Stat() call. if !f.isConsecutiveWrite(off) { + fmt.Printf("isConsecutiveWrite=false, off=%d\n", off) errno := f.writePadHole(off) if errno != 0 { return 0, errno diff --git a/internal/fusefrontend/root_node.go b/internal/fusefrontend/root_node.go index 34b084b..7d37520 100644 --- a/internal/fusefrontend/root_node.go +++ b/internal/fusefrontend/root_node.go @@ -11,7 +11,6 @@ import ( "github.com/rfjakob/gocryptfs/v2/internal/contentenc" "github.com/rfjakob/gocryptfs/v2/internal/inomap" "github.com/rfjakob/gocryptfs/v2/internal/nametransform" - "github.com/rfjakob/gocryptfs/v2/internal/serialize_reads" "github.com/rfjakob/gocryptfs/v2/internal/syscallcompat" "github.com/rfjakob/gocryptfs/v2/internal/tlog" ) @@ -63,9 +62,6 @@ type RootNode struct { } func NewRootNode(args Args, c *contentenc.ContentEnc, n *nametransform.NameTransform) *RootNode { - if args.SerializeReads { - serialize_reads.InitSerializer() - } if len(args.Exclude) > 0 { tlog.Warn.Printf("Forward mode does not support -exclude") } diff --git a/internal/serialize_reads/sr.go b/internal/serialize_reads/sr.go deleted file mode 100644 index 88115f9..0000000 --- a/internal/serialize_reads/sr.go +++ /dev/null @@ -1,150 +0,0 @@ -package serialize_reads - -import ( - "log" - "sync" - "time" - - "github.com/rfjakob/gocryptfs/v2/internal/tlog" -) - -// serializerState is used by the Wait and Done functions -type serializerState struct { - // we get submissions through the "input" channel - input chan *submission - // q = Queue - q []*submission - // wg is used to wait for the read to complete before unblocking the next - wg sync.WaitGroup -} - -// Wait places the caller into a queue and blocks -func Wait(offset int64, size int) { - serializer.wait(offset, size) -} - -// Done signals that the read operation has finished -func Done() { - serializer.wg.Done() -} - -type submission struct { - // "ch" is closed by "eventLoop" once it wants to unblock the caller - ch chan struct{} - // submissions are prioritized by offset (lowest offset gets unblocked first) - offset int64 - // size will be used in the future to detect consecutive read requests. These - // can be unblocked immediately. - size int -} - -func (sr *serializerState) wait(offset int64, size int) { - ch := make(chan struct{}) - sb := &submission{ - ch: ch, - offset: offset, - size: size, - } - // Send our submission - sr.input <- sb - // Wait till we get unblocked - <-ch -} - -// push returns true if the queue is full after the element has been stored. -// It panics if it did not have space to store the element. -func (sr *serializerState) push(sb *submission) (full bool) { - free := 0 - stored := false - for i, v := range sr.q { - if v != nil { - continue - } - if !stored { - sr.q[i] = sb - stored = true - continue - } - free++ - } - if !stored { - // This should never happen because eventLoop checks if the queue got full - log.Panic("BUG: unhandled queue overflow") - } - if free == 0 { - return true - } - return false -} - -// pop the submission with the lowest offset off the queue -func (sr *serializerState) pop() *submission { - var winner *submission - var winnerIndex int - for i, v := range sr.q { - if v == nil { - continue - } - if winner == nil { - winner = v - winnerIndex = i - continue - } - if v.offset < winner.offset { - winner = v - winnerIndex = i - } - } - if winner == nil { - return nil - } - sr.q[winnerIndex] = nil - return winner -} - -func (sr *serializerState) eventLoop() { - sr.input = make(chan *submission) - empty := true - for { - if empty { - // If the queue is empty we block on the channel to conserve CPU - sb := <-sr.input - sr.push(sb) - empty = false - } - select { - case sb := <-sr.input: - full := sr.push(sb) - if full { - // Queue is full, unblock the new request immediately - tlog.Warn.Printf("serialize_reads: queue full, forcing unblock") - sr.unblockOne() - } - case <-time.After(time.Microsecond * 500): - // Looks like we have waited out all concurrent requests. - empty = sr.unblockOne() - } - } -} - -// Unblock a submission and wait for completion -func (sr *serializerState) unblockOne() (empty bool) { - winner := sr.pop() - if winner == nil { - return true - } - sr.wg.Add(1) - close(winner.ch) - sr.wg.Wait() - return false -} - -var serializer serializerState - -// InitSerializer sets up the internal serializer state and starts the event loop. -// Called by fusefrontend.NewFS. -func InitSerializer() { - serializer.input = make(chan *submission) - serializer.q = make([]*submission, 10) - go serializer.eventLoop() -} From a99051b32452c9a781efe248c0014b65d4abddf7 Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Mon, 30 Aug 2021 09:53:36 +0200 Subject: [PATCH 03/59] Reimplement -serialize_reads flag using new SyncRead mount flag Let the kernel do the work for us. See https://github.com/hanwen/go-fuse/commit/15a8bb029a4e1a51e10043c370970596b1fbb737 for more info. --- internal/fusefrontend/args.go | 2 -- mount.go | 9 ++++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/internal/fusefrontend/args.go b/internal/fusefrontend/args.go index 677ffd6..8e28d14 100644 --- a/internal/fusefrontend/args.go +++ b/internal/fusefrontend/args.go @@ -26,8 +26,6 @@ type Args struct { ConfigCustom bool // NoPrealloc disables automatic preallocation before writing NoPrealloc bool - // Try to serialize read operations, "-serialize_reads" - SerializeReads bool // Force decode even if integrity check fails (openSSL only) ForceDecode bool // Exclude is a list of paths to make inaccessible, starting match at diff --git a/mount.go b/mount.go index 44d5878..60ff1ba 100644 --- a/mount.go +++ b/mount.go @@ -273,7 +273,6 @@ func initFuseFrontend(args *argContainer) (rootNode fs.InodeEmbedder, wipeKeys f LongNames: args.longnames, ConfigCustom: args._configCustom, NoPrealloc: args.noprealloc, - SerializeReads: args.serialize_reads, ForceDecode: args.forcedecode, ForceOwner: args._forceOwner, Exclude: args.exclude, @@ -377,6 +376,14 @@ func initGoFuse(rootNode fs.InodeEmbedder, args *argContainer) *fuse.Server { MaxWrite: fuse.MAX_KERNEL_WRITE, Options: []string{fmt.Sprintf("max_read=%d", fuse.MAX_KERNEL_WRITE)}, Debug: args.fusedebug, + // The kernel usually submits multiple read requests in parallel, + // which means we serve them in any order. Out-of-order reads are + // expensive on some backing network filesystems + // ( https://github.com/rfjakob/gocryptfs/issues/92 ). + // + // Setting SyncRead disables FUSE_CAP_ASYNC_READ. This makes the kernel + // do everything in-order without parallelism. + SyncRead: args.serialize_reads, } mOpts := &fuseOpts.MountOptions From fab4ca07de338911966c096169c3885758486649 Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Mon, 30 Aug 2021 10:18:23 +0200 Subject: [PATCH 04/59] README: update changelog --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 6901638..2a40861 100644 --- a/README.md +++ b/README.md @@ -212,9 +212,16 @@ v2.2, IN PROGRESS * `-deterministic-names`: new option for `-init`, both for reverse and forward mode. Disables file name randomisation & `gocryptfs.diriv` files ([#151](https://github.com/rfjakob/gocryptfs/issues/151), [#402](https://github.com/rfjakob/gocryptfs/issues/402), [#592](https://github.com/rfjakob/gocryptfs/pull/592)) + * New feature flag! You need gocryptfs v2.2 or higher to mount a filesystem that uses this flag. * `-xchacha`: new option for `-init` (forward mode only). Selects XChaCha20-Poly1305 for content encryption. Gives [better performance on embedded CPUs](https://gist.github.com/rfjakob/b28383f4c84263ac7c5388ccc262e38b) ([#452](https://github.com/rfjakob/gocryptfs/issues/452)) + * New feature flag! You need gocryptfs v2.2 or higher to mount a filesystem that uses this flag. +* `-serialize_reads`: get rid of delay logic by taking advantage of the kernel flag + `FUSE_CAP_ASYNC_READ` + ([go-fuse commit](https://github.com/hanwen/go-fuse/commit/15a8bb029a4e1a51e10043c370970596b1fbb737), + [gocryptfs commit](https://github.com/rfjakob/gocryptfs/commit/a99051b32452c9a781efe248c0014b65d4abddf7)) +* Make obsolete `-devrandom` flag a no-op ([commit](https://github.com/rfjakob/gocryptfs/commit/61ef6b00a675456ee05d40f1ce44d693bc4be350)) v2.1, 2021-08-18 * `-fido2`: do not request PIN on `gocryptfs -init` fixing `FIDO_ERR_UNSUPPORTED_OPTION` with YubiKey From 17fe50ef742869e50b18d90846436215f798b548 Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Mon, 30 Aug 2021 10:18:33 +0200 Subject: [PATCH 05/59] README: compress Installation section More content, less whitespace. --- README.md | 30 ++++++++---------------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 2a40861..44e7896 100644 --- a/README.md +++ b/README.md @@ -55,30 +55,16 @@ A standalone Python tool that can decrypt files & file names is here: Installation ------------ -Precompiled binaries that work on all x86_64 Linux systems are available for download from the github releases page. +Precompiled binaries that work on all x86_64 Linux systems are available +for download from the github releases page. The `fuse` package from your +distribution must be installed for mounting to work. -On Debian, gocryptfs is available as a deb package: -```bash -apt install gocryptfs -``` +gocryptfs is also available as a package in most distributions. Examples: -On macOS, gocryptfs is available as a Homebrew formula: -```bash -brew install gocryptfs -``` - -Alternatively, gocryptfs is also available via [MacPorts](https://www.macports.org/) on macOS: -```bash -sudo port install gocryptfs -``` - -On Fedora, gocryptfs is available as an rpm package: -```bash -sudo dnf install gocryptfs -``` - -If you use the standalone binary, make sure you install the `fuse` package -from your distributions package repository before running `gocryptfs`. +* Debian, Ubuntu: `apt install gocryptfs` +* Fedora: `dnf install gocryptfs` +* Arch: `pacman -S gocryptfs` +* MacPorts: `port install gocryptfs` See the [Quickstart](https://nuetzlich.net/gocryptfs/quickstart/) page for more info. From 34d8a498c4899b1493f7bea16c22486d6725c9b1 Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Mon, 30 Aug 2021 11:31:01 +0200 Subject: [PATCH 06/59] Unbreak hyperlinks broken by go mod v2 conversion Commit 69d88505fd7f4cb0d9e4f1918de296342fe05858 go mod: declare module version v2 translated all instances of "github.com/rfjakob/gocryptfs/" to "github.com/rfjakob/gocryptfs/v2/". Unfortunately, this included hyperlinks. Unbreak the hyperlinks like this: find . -name \*.go | xargs sed -i s%https://github.com/rfjakob/gocryptfs/v2/%https://github.com/rfjakob/gocryptfs/v2/% --- contrib/getdents-debug/getdents/getdents.go | 2 +- .../getdents-debug/readdirnames/readdirnames.go | 2 +- internal/configfile/config_file.go | 2 +- internal/contentenc/content.go | 2 +- internal/ensurefds012/ensurefds012.go | 2 +- internal/fusefrontend/args.go | 2 +- internal/fusefrontend/node_xattr.go | 4 ++-- internal/nametransform/diriv.go | 2 +- internal/nametransform/perms.go | 4 ++-- internal/stupidgcm/prefer.go | 4 ++-- internal/syscallcompat/eintr.go | 2 +- internal/syscallcompat/getdents_linux.go | 6 +++--- internal/syscallcompat/getdents_test.go | 2 +- internal/syscallcompat/quirks.go | 6 +++--- internal/syscallcompat/quirks_darwin.go | 6 +++--- internal/syscallcompat/quirks_linux.go | 6 +++--- internal/syscallcompat/sys_darwin.go | 2 +- internal/syscallcompat/sys_linux.go | 2 +- main.go | 2 +- mount.go | 2 +- tests/cli/cli_test.go | 16 ++++++++-------- tests/defaults/acl_test.go | 4 ++-- tests/defaults/main_test.go | 2 +- tests/matrix/concurrency_test.go | 4 ++-- tests/matrix/dir_test.go | 2 +- tests/matrix/fallocate_test.go | 2 +- tests/matrix/matrix_test.go | 6 +++--- tests/reverse/correctness_test.go | 2 +- tests/reverse/exclude_test.go | 2 +- 29 files changed, 51 insertions(+), 51 deletions(-) diff --git a/contrib/getdents-debug/getdents/getdents.go b/contrib/getdents-debug/getdents/getdents.go index 781a23d..a3cdac4 100644 --- a/contrib/getdents-debug/getdents/getdents.go +++ b/contrib/getdents-debug/getdents/getdents.go @@ -1,6 +1,6 @@ /* Small tool to try to debug unix.Getdents problems on CIFS mounts -( https://github.com/rfjakob/gocryptfs/v2/issues/483 ) +( https://github.com/rfjakob/gocryptfs/issues/483 ) Example output: diff --git a/contrib/getdents-debug/readdirnames/readdirnames.go b/contrib/getdents-debug/readdirnames/readdirnames.go index 1a29290..dc33512 100644 --- a/contrib/getdents-debug/readdirnames/readdirnames.go +++ b/contrib/getdents-debug/readdirnames/readdirnames.go @@ -1,6 +1,6 @@ /* Small tool to try to debug unix.Getdents problems on CIFS mounts -( https://github.com/rfjakob/gocryptfs/v2/issues/483 ) +( https://github.com/rfjakob/gocryptfs/issues/483 ) Example output: diff --git a/internal/configfile/config_file.go b/internal/configfile/config_file.go index 951dce8..06b665b 100644 --- a/internal/configfile/config_file.go +++ b/internal/configfile/config_file.go @@ -276,7 +276,7 @@ func (cf *ConfFile) WriteFile() error { err = fd.Sync() if err != nil { // This can happen on network drives: FRITZ.NAS mounted on MacOS returns - // "operation not supported": https://github.com/rfjakob/gocryptfs/v2/issues/390 + // "operation not supported": https://github.com/rfjakob/gocryptfs/issues/390 tlog.Warn.Printf("Warning: fsync failed: %v", err) // Try sync instead syscall.Sync() diff --git a/internal/contentenc/content.go b/internal/contentenc/content.go index 8e7ca04..13e0ce0 100644 --- a/internal/contentenc/content.go +++ b/internal/contentenc/content.go @@ -167,7 +167,7 @@ func (be *ContentEnc) DecryptBlock(ciphertext []byte, blockNo uint64, fileID []b nonce := ciphertext[:be.cryptoCore.IVLen] if bytes.Equal(nonce, be.allZeroNonce) { // Bug in tmpfs? - // https://github.com/rfjakob/gocryptfs/v2/issues/56 + // https://github.com/rfjakob/gocryptfs/issues/56 // http://www.spinics.net/lists/kernel/msg2370127.html return nil, errors.New("all-zero nonce") } diff --git a/internal/ensurefds012/ensurefds012.go b/internal/ensurefds012/ensurefds012.go index 09b3fc2..54a1ac1 100644 --- a/internal/ensurefds012/ensurefds012.go +++ b/internal/ensurefds012/ensurefds012.go @@ -25,7 +25,7 @@ // l-wx------. 1 jakob jakob 64 Jan 5 15:54 3 -> /dev/null // lrwx------. 1 jakob jakob 64 Jan 5 15:54 4 -> 'anon_inode:[eventpoll]' // -// See https://github.com/rfjakob/gocryptfs/v2/issues/320 for details. +// See https://github.com/rfjakob/gocryptfs/issues/320 for details. package ensurefds012 import ( diff --git a/internal/fusefrontend/args.go b/internal/fusefrontend/args.go index 8e28d14..4aedf2e 100644 --- a/internal/fusefrontend/args.go +++ b/internal/fusefrontend/args.go @@ -40,7 +40,7 @@ type Args struct { // Suid is true if the filesystem has been mounted with the "-suid" flag. // If it is false, we can ignore the GETXATTR "security.capability" calls, // which are a performance problem for writes. See - // https://github.com/rfjakob/gocryptfs/v2/issues/515 for details. + // https://github.com/rfjakob/gocryptfs/issues/515 for details. Suid bool // Enable the FUSE kernel_cache option KernelCache bool diff --git a/internal/fusefrontend/node_xattr.go b/internal/fusefrontend/node_xattr.go index d5d8c5b..09ee5ef 100644 --- a/internal/fusefrontend/node_xattr.go +++ b/internal/fusefrontend/node_xattr.go @@ -24,7 +24,7 @@ var xattrNameIV = []byte("xattr_name_iv_xx") var xattrStorePrefix = "user.gocryptfs." // We get one read of this xattr for each write - -// see https://github.com/rfjakob/gocryptfs/v2/issues/515 for details. +// see https://github.com/rfjakob/gocryptfs/issues/515 for details. var xattrCapability = "security.capability" // isAcl returns true if the attribute name is for storing ACLs @@ -41,7 +41,7 @@ func (n *Node) Getxattr(ctx context.Context, attr string, dest []byte) (uint32, rn := n.rootNode() // If we are not mounted with -suid, reading the capability xattr does not // make a lot of sense, so reject the request and gain a massive speedup. - // See https://github.com/rfjakob/gocryptfs/v2/issues/515 . + // See https://github.com/rfjakob/gocryptfs/issues/515 . if !rn.args.Suid && attr == xattrCapability { // Returning EOPNOTSUPP is what we did till // ca9e912a28b901387e1dbb85f6c531119f2d5ef2 "fusefrontend: drop xattr user namespace restriction" diff --git a/internal/nametransform/diriv.go b/internal/nametransform/diriv.go index dc6b21d..7929c40 100644 --- a/internal/nametransform/diriv.go +++ b/internal/nametransform/diriv.go @@ -68,7 +68,7 @@ func WriteDirIVAt(dirfd int) error { iv := cryptocore.RandBytes(DirIVLen) // 0400 permissions: gocryptfs.diriv should never be modified after creation. // Don't use "ioutil.WriteFile", it causes trouble on NFS: - // https://github.com/rfjakob/gocryptfs/v2/commit/7d38f80a78644c8ec4900cc990bfb894387112ed + // https://github.com/rfjakob/gocryptfs/commit/7d38f80a78644c8ec4900cc990bfb894387112ed fd, err := syscallcompat.Openat(dirfd, DirIVFilename, os.O_WRONLY|os.O_CREATE|os.O_EXCL, dirivPerms) if err != nil { tlog.Warn.Printf("WriteDirIV: Openat: %v", err) diff --git a/internal/nametransform/perms.go b/internal/nametransform/perms.go index 6b88afd..cfcd062 100644 --- a/internal/nametransform/perms.go +++ b/internal/nametransform/perms.go @@ -6,14 +6,14 @@ const ( // never chmod'ed or chown'ed. // // Group-readable so the FS can be mounted by several users in the same group - // (see https://github.com/rfjakob/gocryptfs/v2/issues/387 ). + // (see https://github.com/rfjakob/gocryptfs/issues/387 ). // // Note that gocryptfs.conf is still created with 0400 permissions so the // owner must explicitly chmod it to permit access. // // World-readable so an encrypted directory can be copied by the non-root // owner when gocryptfs is running as root - // ( https://github.com/rfjakob/gocryptfs/v2/issues/539 ). + // ( https://github.com/rfjakob/gocryptfs/issues/539 ). dirivPerms = 0444 // Permissions for gocryptfs.longname.[sha256].name files. diff --git a/internal/stupidgcm/prefer.go b/internal/stupidgcm/prefer.go index 94c1b6c..bacd56a 100644 --- a/internal/stupidgcm/prefer.go +++ b/internal/stupidgcm/prefer.go @@ -14,7 +14,7 @@ import ( // 2) Is ARM64 && has AES instructions && Go is v1.11 or higher // (commit https://github.com/golang/go/commit/4f1f503373cda7160392be94e3849b0c9b9ebbda) // -// See https://github.com/rfjakob/gocryptfs/v2/wiki/CPU-Benchmarks +// See https://github.com/rfjakob/gocryptfs/wiki/CPU-Benchmarks // for benchmarks. func PreferOpenSSL() bool { if BuiltWithoutOpenssl { @@ -26,7 +26,7 @@ func PreferOpenSSL() bool { return false } // On the Apple M1, Go stdlib is faster than OpenSSL, despite cpu.ARM64.HasAES - // reading false: https://github.com/rfjakob/gocryptfs/v2/issues/556#issuecomment-848079309 + // reading false: https://github.com/rfjakob/gocryptfs/issues/556#issuecomment-848079309 if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" { return false } diff --git a/internal/syscallcompat/eintr.go b/internal/syscallcompat/eintr.go index 2e2bb18..cdde806 100644 --- a/internal/syscallcompat/eintr.go +++ b/internal/syscallcompat/eintr.go @@ -12,7 +12,7 @@ import ( // https://github.com/golang/go/blob/d2a80f3fb5b44450e0b304ac5a718f99c053d82a/src/os/file_posix.go#L243 // // This is needed because CIFS throws lots of EINTR errors: -// https://github.com/rfjakob/gocryptfs/v2/issues/483 +// https://github.com/rfjakob/gocryptfs/issues/483 // // Don't use retryEINTR() with syscall.Close()! // See https://code.google.com/p/chromium/issues/detail?id=269623 . diff --git a/internal/syscallcompat/getdents_linux.go b/internal/syscallcompat/getdents_linux.go index 2ad7cde..cedb463 100644 --- a/internal/syscallcompat/getdents_linux.go +++ b/internal/syscallcompat/getdents_linux.go @@ -23,7 +23,7 @@ const sizeofDirent = int(unsafe.Sizeof(unix.Dirent{})) // maxReclen sanity check: Reclen should never be larger than this. // Due to padding between entries, it is 280 even on 32-bit architectures. -// See https://github.com/rfjakob/gocryptfs/v2/issues/197 for details. +// See https://github.com/rfjakob/gocryptfs/issues/197 for details. const maxReclen = 280 // getdents wraps unix.Getdents and converts the result to []fuse.DirEntry. @@ -43,7 +43,7 @@ func getdents(fd int) (entries []fuse.DirEntry, entriesSpecial []fuse.DirEntry, continue } else if err != nil { if smartBuf.Len() > 0 { - tlog.Warn.Printf("warning: unix.Getdents returned errno %d in the middle of data ( https://github.com/rfjakob/gocryptfs/v2/issues/483 )", err.(syscall.Errno)) + tlog.Warn.Printf("warning: unix.Getdents returned errno %d in the middle of data ( https://github.com/rfjakob/gocryptfs/issues/483 )", err.(syscall.Errno)) return nil, nil, syscall.EIO } return nil, nil, err @@ -145,7 +145,7 @@ func dtUnknownWarn(dirfd int) { if err == nil && buf.Type == XFS_SUPER_MAGIC { // Old XFS filesystems always return DT_UNKNOWN. Downgrade the message // to "info" level if we are on XFS. - // https://github.com/rfjakob/gocryptfs/v2/issues/267 + // https://github.com/rfjakob/gocryptfs/issues/267 tlog.Info.Printf("Getdents: convertDType: received DT_UNKNOWN, fstype=xfs, falling back to stat") } else { tlog.Warn.Printf("Getdents: convertDType: received DT_UNKNOWN, fstype=%#x, falling back to stat", diff --git a/internal/syscallcompat/getdents_test.go b/internal/syscallcompat/getdents_test.go index 6c746bc..a6f41ca 100644 --- a/internal/syscallcompat/getdents_test.go +++ b/internal/syscallcompat/getdents_test.go @@ -27,7 +27,7 @@ func TestGetdents(t *testing.T) { // skipOnGccGo skips the emulateGetdents test when we are // running linux and were compiled with gccgo. The test is known to fail -// (https://github.com/rfjakob/gocryptfs/v2/issues/201), but getdents emulation +// (https://github.com/rfjakob/gocryptfs/issues/201), but getdents emulation // is not used on linux, so let's skip the test and ignore the failure. func skipOnGccGo(t *testing.T) { if !emulate || runtime.GOOS != "linux" { diff --git a/internal/syscallcompat/quirks.go b/internal/syscallcompat/quirks.go index cfcf3ff..110c00d 100644 --- a/internal/syscallcompat/quirks.go +++ b/internal/syscallcompat/quirks.go @@ -6,12 +6,12 @@ import ( const ( // QuirkBrokenFalloc means the falloc is broken. - // Preallocation on Btrfs is broken ( https://github.com/rfjakob/gocryptfs/v2/issues/395 ) - // and slow ( https://github.com/rfjakob/gocryptfs/v2/issues/63 ). + // Preallocation on Btrfs is broken ( https://github.com/rfjakob/gocryptfs/issues/395 ) + // and slow ( https://github.com/rfjakob/gocryptfs/issues/63 ). QuirkBrokenFalloc = uint64(1 << iota) // QuirkDuplicateIno1 means that we have duplicate inode numbers. // On MacOS ExFAT, all empty files share inode number 1: - // https://github.com/rfjakob/gocryptfs/v2/issues/585 + // https://github.com/rfjakob/gocryptfs/issues/585 QuirkDuplicateIno1 ) diff --git a/internal/syscallcompat/quirks_darwin.go b/internal/syscallcompat/quirks_darwin.go index aca3d99..4adeea1 100644 --- a/internal/syscallcompat/quirks_darwin.go +++ b/internal/syscallcompat/quirks_darwin.go @@ -8,7 +8,7 @@ import ( func DetectQuirks(cipherdir string) (q uint64) { const ( - // From https://github.com/rfjakob/gocryptfs/v2/issues/585#issuecomment-887370065 + // From https://github.com/rfjakob/gocryptfs/issues/585#issuecomment-887370065 FstypenameExfat = "exfat" ) @@ -31,9 +31,9 @@ func DetectQuirks(cipherdir string) (q uint64) { tlog.Debug.Printf("DetectQuirks: Fstypename=%q\n", fstypename) // On MacOS ExFAT, all empty files share inode number 1: - // https://github.com/rfjakob/gocryptfs/v2/issues/585 + // https://github.com/rfjakob/gocryptfs/issues/585 if fstypename == FstypenameExfat { - logQuirk("ExFAT detected, disabling hard links. See https://github.com/rfjakob/gocryptfs/v2/issues/585 for why.") + logQuirk("ExFAT detected, disabling hard links. See https://github.com/rfjakob/gocryptfs/issues/585 for why.") q |= QuirkDuplicateIno1 } diff --git a/internal/syscallcompat/quirks_linux.go b/internal/syscallcompat/quirks_linux.go index 418f010..bcdcf07 100644 --- a/internal/syscallcompat/quirks_linux.go +++ b/internal/syscallcompat/quirks_linux.go @@ -18,12 +18,12 @@ func DetectQuirks(cipherdir string) (q uint64) { return 0 } - // Preallocation on Btrfs is broken ( https://github.com/rfjakob/gocryptfs/v2/issues/395 ) - // and slow ( https://github.com/rfjakob/gocryptfs/v2/issues/63 ). + // Preallocation on Btrfs is broken ( https://github.com/rfjakob/gocryptfs/issues/395 ) + // and slow ( https://github.com/rfjakob/gocryptfs/issues/63 ). // // Cast to uint32 avoids compile error on arm: "constant 2435016766 overflows int32" if uint32(st.Type) == unix.BTRFS_SUPER_MAGIC { - logQuirk("Btrfs detected, forcing -noprealloc. See https://github.com/rfjakob/gocryptfs/v2/issues/395 for why.") + logQuirk("Btrfs detected, forcing -noprealloc. See https://github.com/rfjakob/gocryptfs/issues/395 for why.") q |= QuirkBrokenFalloc } diff --git a/internal/syscallcompat/sys_darwin.go b/internal/syscallcompat/sys_darwin.go index e35f213..075563f 100644 --- a/internal/syscallcompat/sys_darwin.go +++ b/internal/syscallcompat/sys_darwin.go @@ -59,7 +59,7 @@ func setattrlist(path *byte, list unsafe.Pointer, buf unsafe.Pointer, size uintp // Sorry, fallocate is not available on OSX at all and // fcntl F_PREALLOCATE is not accessible from Go. -// See https://github.com/rfjakob/gocryptfs/v2/issues/18 if you want to help. +// See https://github.com/rfjakob/gocryptfs/issues/18 if you want to help. func EnospcPrealloc(fd int, off int64, len int64) error { return nil } diff --git a/internal/syscallcompat/sys_linux.go b/internal/syscallcompat/sys_linux.go index b7f1dad..961d1c9 100644 --- a/internal/syscallcompat/sys_linux.go +++ b/internal/syscallcompat/sys_linux.go @@ -48,7 +48,7 @@ func EnospcPrealloc(fd int, off int64, len int64) (err error) { } if err == syscall.EOPNOTSUPP { // ZFS and ext3 do not support fallocate. Warn but continue anyway. - // https://github.com/rfjakob/gocryptfs/v2/issues/22 + // https://github.com/rfjakob/gocryptfs/issues/22 preallocWarn.Do(func() { tlog.Warn.Printf("Warning: The underlying filesystem " + "does not support fallocate(2). gocryptfs will continue working " + diff --git a/main.go b/main.go index 169bdeb..0ac7423 100644 --- a/main.go +++ b/main.go @@ -1,5 +1,5 @@ // gocryptfs is an encrypted overlay filesystem written in Go. -// See README.md ( https://github.com/rfjakob/gocryptfs/v2/blob/master/README.md ) +// See README.md ( https://github.com/rfjakob/gocryptfs/blob/master/README.md ) // and the official website ( https://nuetzlich.net/gocryptfs/ ) for details. package main diff --git a/mount.go b/mount.go index 60ff1ba..d7cd7db 100644 --- a/mount.go +++ b/mount.go @@ -75,7 +75,7 @@ func doMount(args *argContainer) { // and `drop_privileges` in `man mount.fuse3` for background. } else { err = isEmptyDir(args.mountpoint) - // OSXFuse will create the mountpoint for us ( https://github.com/rfjakob/gocryptfs/v2/issues/194 ) + // OSXFuse will create the mountpoint for us ( https://github.com/rfjakob/gocryptfs/issues/194 ) if runtime.GOOS == "darwin" && os.IsNotExist(err) { tlog.Info.Printf("Mountpoint %q does not exist, but should be created by OSXFuse", args.mountpoint) diff --git a/tests/cli/cli_test.go b/tests/cli/cli_test.go index 96a5b72..d63625b 100644 --- a/tests/cli/cli_test.go +++ b/tests/cli/cli_test.go @@ -60,9 +60,9 @@ func TestInitFilePerms(t *testing.T) { syscall.Stat(dir+"/gocryptfs.diriv", &st) perms = st.Mode & 0777 // From v1.7.1, these are created with 0440 permissions, see - // https://github.com/rfjakob/gocryptfs/v2/issues/387 . + // https://github.com/rfjakob/gocryptfs/issues/387 . // From v2.0, created with 0444 perms, see - // https://github.com/rfjakob/gocryptfs/v2/issues/539 . + // https://github.com/rfjakob/gocryptfs/issues/539 . if perms != 0444 { t.Errorf("Wrong permissions for gocryptfs.diriv: %#o", perms) } @@ -441,7 +441,7 @@ func TestPasswdPasswordIncorrect(t *testing.T) { // Check that we correctly background on mount and close stderr and stdout. // Something like // gocryptfs a b | cat -// must not hang ( https://github.com/rfjakob/gocryptfs/v2/issues/130 ). +// must not hang ( https://github.com/rfjakob/gocryptfs/issues/130 ). func TestMountBackground(t *testing.T) { dir := test_helpers.InitFS(t) mnt := dir + ".mnt" @@ -557,7 +557,7 @@ func TestExcludeForward(t *testing.T) { } // Check that the config file can be read from a named pipe. -// Make sure bug https://github.com/rfjakob/gocryptfs/v2/issues/258 does not come +// Make sure bug https://github.com/rfjakob/gocryptfs/issues/258 does not come // back. func TestConfigPipe(t *testing.T) { dir := test_helpers.InitFS(t) @@ -580,7 +580,7 @@ func TestConfigPipe(t *testing.T) { } // Ciphertext dir and mountpoint contains a comma -// https://github.com/rfjakob/gocryptfs/v2/issues/262 +// https://github.com/rfjakob/gocryptfs/issues/262 func TestComma(t *testing.T) { dir0 := test_helpers.InitFS(t) dir := dir0 + ",foo,bar" @@ -625,7 +625,7 @@ func TestIdle(t *testing.T) { } // Mount with idle timeout of 100ms read something every 10ms. The fs should -// NOT get unmounted. Regression test for https://github.com/rfjakob/gocryptfs/v2/issues/421 +// NOT get unmounted. Regression test for https://github.com/rfjakob/gocryptfs/issues/421 func TestNotIdle(t *testing.T) { dir := test_helpers.InitFS(t) mnt := dir + ".mnt" @@ -663,7 +663,7 @@ func TestNotIdle(t *testing.T) { // TestSymlinkedCipherdir checks that if CIPHERDIR itself is a symlink, it is // followed. -// https://github.com/rfjakob/gocryptfs/v2/issues/450 +// https://github.com/rfjakob/gocryptfs/issues/450 func TestSymlinkedCipherdir(t *testing.T) { dir := test_helpers.InitFS(t) dirSymlink := dir + ".symlink" @@ -909,7 +909,7 @@ func TestPassfileX2(t *testing.T) { } // TestInitNotEmpty checks that `gocryptfs -init` returns the right error code -// if CIPHERDIR is not empty. See https://github.com/rfjakob/gocryptfs/v2/pull/503 +// if CIPHERDIR is not empty. See https://github.com/rfjakob/gocryptfs/pull/503 func TestInitNotEmpty(t *testing.T) { dir := test_helpers.TmpDir + "/" + t.Name() if err := os.Mkdir(dir, 0700); err != nil { diff --git a/tests/defaults/acl_test.go b/tests/defaults/acl_test.go index 94d3c38..2ab5dc1 100644 --- a/tests/defaults/acl_test.go +++ b/tests/defaults/acl_test.go @@ -17,7 +17,7 @@ import ( "github.com/rfjakob/gocryptfs/v2/tests/test_helpers" ) -// https://github.com/rfjakob/gocryptfs/v2/issues/543 +// https://github.com/rfjakob/gocryptfs/issues/543 func TestCpA(t *testing.T) { fn1 := filepath.Join(test_helpers.TmpDir, t.Name()) fn2 := filepath.Join(test_helpers.DefaultPlainDir, t.Name()) @@ -77,7 +77,7 @@ func getfacl(fn string) (string, error) { return string(out), err } -// https://github.com/rfjakob/gocryptfs/v2/issues/543 +// https://github.com/rfjakob/gocryptfs/issues/543 func TestAcl543(t *testing.T) { fn1 := test_helpers.TmpDir + "/TestAcl543" fn2 := test_helpers.DefaultPlainDir + "/TestAcl543" diff --git a/tests/defaults/main_test.go b/tests/defaults/main_test.go index 0acab2d..0045981 100644 --- a/tests/defaults/main_test.go +++ b/tests/defaults/main_test.go @@ -239,7 +239,7 @@ func TestMvWarningSymlink(t *testing.T) { if err != nil { t.Log(string(out)) if runtime.GOOS == "darwin" { - t.Skip("mv on darwin chokes on broken symlinks, see https://github.com/rfjakob/gocryptfs/v2/issues/349") + t.Skip("mv on darwin chokes on broken symlinks, see https://github.com/rfjakob/gocryptfs/issues/349") } t.Fatal(err) } diff --git a/tests/matrix/concurrency_test.go b/tests/matrix/concurrency_test.go index fa713df..04b7c96 100644 --- a/tests/matrix/concurrency_test.go +++ b/tests/matrix/concurrency_test.go @@ -12,7 +12,7 @@ import ( "github.com/rfjakob/gocryptfs/v2/tests/test_helpers" ) -// https://github.com/rfjakob/gocryptfs/v2/issues/363 +// https://github.com/rfjakob/gocryptfs/issues/363 // // Note: this test calls log.Fatal() instead of t.Fatal() because apparently, // calling t.Fatal() from a goroutine hangs the test. @@ -73,7 +73,7 @@ func TestConcurrentReadWrite(t *testing.T) { wg.Wait() } -// https://github.com/rfjakob/gocryptfs/v2/issues/363 +// https://github.com/rfjakob/gocryptfs/issues/363 // // Note: this test calls log.Fatal() instead of t.Fatal() because apparently, // calling t.Fatal() from a goroutine hangs the test. diff --git a/tests/matrix/dir_test.go b/tests/matrix/dir_test.go index 52c97a7..13d6712 100644 --- a/tests/matrix/dir_test.go +++ b/tests/matrix/dir_test.go @@ -34,7 +34,7 @@ func TestDirOverwrite(t *testing.T) { } // Test that we can create and remove a directory regardless of the permission it has -// https://github.com/rfjakob/gocryptfs/v2/issues/354 +// https://github.com/rfjakob/gocryptfs/issues/354 func TestRmdirPerms(t *testing.T) { for _, perm := range []uint32{0000, 0100, 0200, 0300, 0400, 0500, 0600, 0700} { dir := fmt.Sprintf("TestRmdir%#o", perm) diff --git a/tests/matrix/fallocate_test.go b/tests/matrix/fallocate_test.go index 84eae7e..ade6415 100644 --- a/tests/matrix/fallocate_test.go +++ b/tests/matrix/fallocate_test.go @@ -149,7 +149,7 @@ func TestFallocate(t *testing.T) { } } // We used to allocate 18 bytes too much: - // https://github.com/rfjakob/gocryptfs/v2/issues/311 + // https://github.com/rfjakob/gocryptfs/issues/311 // // 8110 bytes of plaintext should get us exactly 8192 bytes of ciphertext. err = file.Truncate(0) diff --git a/tests/matrix/matrix_test.go b/tests/matrix/matrix_test.go index f34b5f2..abcb7e0 100644 --- a/tests/matrix/matrix_test.go +++ b/tests/matrix/matrix_test.go @@ -774,7 +774,7 @@ func TestMkfifo(t *testing.T) { } // TestMagicNames verifies that "magic" names are handled correctly -// https://github.com/rfjakob/gocryptfs/v2/issues/174 +// https://github.com/rfjakob/gocryptfs/issues/174 func TestMagicNames(t *testing.T) { names := []string{"warmup1", "warmup2", "gocryptfs.longname.QhUr5d9FHerwEs--muUs6_80cy6JRp89c1otLwp92Cs", "gocryptfs.diriv"} for _, n := range names { @@ -891,7 +891,7 @@ func TestStatfs(t *testing.T) { } // gocryptfs 2.0 reported the ciphertext size on symlink creation, causing -// confusion: https://github.com/rfjakob/gocryptfs/v2/issues/574 +// confusion: https://github.com/rfjakob/gocryptfs/issues/574 func TestSymlinkSize(t *testing.T) { p := filepath.Join(test_helpers.DefaultPlainDir, t.Name()) // SYMLINK reports the size to the kernel @@ -911,7 +911,7 @@ func TestSymlinkSize(t *testing.T) { // TestPwd check that /usr/bin/pwd works inside gocryptfs. // // This was broken in gocryptfs v2.0 with -sharedstorage: -// https://github.com/rfjakob/gocryptfs/v2/issues/584 +// https://github.com/rfjakob/gocryptfs/issues/584 func TestPwd(t *testing.T) { dir := test_helpers.DefaultPlainDir for i := 0; i < 3; i++ { diff --git a/tests/reverse/correctness_test.go b/tests/reverse/correctness_test.go index 3b25112..090a468 100644 --- a/tests/reverse/correctness_test.go +++ b/tests/reverse/correctness_test.go @@ -185,7 +185,7 @@ func TestEnoent(t *testing.T) { // If the symlink target gets too long due to base64 encoding, we should // return ENAMETOOLONG instead of having the kernel reject the data and // returning an I/O error to the user. -// https://github.com/rfjakob/gocryptfs/v2/issues/167 +// https://github.com/rfjakob/gocryptfs/issues/167 func TestTooLongSymlink(t *testing.T) { var err error var l int diff --git a/tests/reverse/exclude_test.go b/tests/reverse/exclude_test.go index e37050c..f24ff2c 100644 --- a/tests/reverse/exclude_test.go +++ b/tests/reverse/exclude_test.go @@ -128,7 +128,7 @@ func TestExcludeTestFs(t *testing.T) { } // Exclude everything using "/*", then selectively include only dir1 using "!/dir1" -// https://github.com/rfjakob/gocryptfs/v2/issues/588 +// https://github.com/rfjakob/gocryptfs/issues/588 func TestExcludeAllOnlyDir1(t *testing.T) { // --exclude-wildcard patterns, gitignore syntax patterns := []string{ From 4e3b7702aff72283fb9a999e6568bdfd41581fd5 Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Mon, 30 Aug 2021 11:39:44 +0200 Subject: [PATCH 07/59] fusefrontend: remove leftover Printf Commit b83ca9c921019fc3b790dabb6198bb77ef2f9a34 inadveredly added a leftover debug Printf. Delete it. --- internal/fusefrontend/file.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/fusefrontend/file.go b/internal/fusefrontend/file.go index 6f33a60..661c2b8 100644 --- a/internal/fusefrontend/file.go +++ b/internal/fusefrontend/file.go @@ -382,7 +382,6 @@ func (f *File) Write(ctx context.Context, data []byte, off int64) (uint32, sysca // But if the write directly follows an earlier write, it cannot create a // hole, and we can save one Stat() call. if !f.isConsecutiveWrite(off) { - fmt.Printf("isConsecutiveWrite=false, off=%d\n", off) errno := f.writePadHole(off) if errno != 0 { return 0, errno From c505e73a13d8e2083b53c5c1af1f937b46665abd Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Mon, 30 Aug 2021 20:00:00 +0200 Subject: [PATCH 08/59] README: explain where -xchacha makes sense --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 44e7896..7f47139 100644 --- a/README.md +++ b/README.md @@ -203,6 +203,7 @@ v2.2, IN PROGRESS Gives [better performance on embedded CPUs](https://gist.github.com/rfjakob/b28383f4c84263ac7c5388ccc262e38b) ([#452](https://github.com/rfjakob/gocryptfs/issues/452)) * New feature flag! You need gocryptfs v2.2 or higher to mount a filesystem that uses this flag. + * Test with `gocryptfs -speed` what is fastest for your CPU, or read [here](https://github.com/rfjakob/gocryptfs/issues/452#issuecomment-908559414) * `-serialize_reads`: get rid of delay logic by taking advantage of the kernel flag `FUSE_CAP_ASYNC_READ` ([go-fuse commit](https://github.com/hanwen/go-fuse/commit/15a8bb029a4e1a51e10043c370970596b1fbb737), From c63f7e9f64ee394b3311edb7f36f56fd786d145a Mon Sep 17 00:00:00 2001 From: a1346054 <36859588+a1346054@users.noreply.github.com> Date: Tue, 31 Aug 2021 17:01:47 +0000 Subject: [PATCH 09/59] shell scripts: fix shellcheck warnings --- Documentation/MANPAGE-render.bash | 10 ++--- benchmark-reverse.bash | 10 ++--- benchmark.bash | 2 +- contrib/gocryptfs-maybe.bash | 2 +- contrib/maxlen.bash | 8 ++-- contrib/mount-ext4-ramdisk.sh | 2 +- profiling/ls.bash | 18 ++++---- profiling/streaming-read.bash | 16 ++++---- profiling/streaming-write.bash | 10 ++--- profiling/tar-extract.bash | 10 ++--- profiling/write-trace.bash | 10 ++--- test.bash | 2 +- tests/canonical-benchmarks.bash | 2 +- tests/dl-linux-tarball.bash | 8 ++-- tests/fuse-unmount.bash | 2 +- tests/len2elen.sh | 6 +-- tests/reverse/linux-tarball-test.bash | 6 +-- tests/sshfs-benchmark.bash | 16 ++++---- tests/stress_tests/extractloop.bash | 48 +++++++++++----------- tests/stress_tests/fsstress-gocryptfs.bash | 40 +++++++++--------- tests/stress_tests/parallel_cp.sh | 16 ++++---- tests/stress_tests/pingpong.bash | 36 ++++++++-------- 22 files changed, 140 insertions(+), 140 deletions(-) diff --git a/Documentation/MANPAGE-render.bash b/Documentation/MANPAGE-render.bash index 74028ad..c141c1e 100755 --- a/Documentation/MANPAGE-render.bash +++ b/Documentation/MANPAGE-render.bash @@ -1,16 +1,16 @@ #!/bin/bash set -eu -cd $(dirname "$0") +cd "$(dirname "$0")" # Render Markdown to a proper man(1) manpage -function render { +render() { IN=$1 OUT=$2 echo "Rendering $IN to $OUT" - echo ".\\\" This man page was generated from $IN. View it using 'man ./$OUT'" > $OUT - echo ".\\\"" >> $OUT - pandoc "$IN" -s -t man >> $OUT + echo ".\\\" This man page was generated from $IN. View it using 'man ./$OUT'" > "$OUT" + echo ".\\\"" >> "$OUT" + pandoc "$IN" -s -t man >> "$OUT" } render MANPAGE.md gocryptfs.1 diff --git a/benchmark-reverse.bash b/benchmark-reverse.bash index be98fc1..fad6bfe 100755 --- a/benchmark-reverse.bash +++ b/benchmark-reverse.bash @@ -14,18 +14,18 @@ PLAIN=linux-3.0 SIZE=0 if [[ -d $PLAIN ]]; then - SIZE=$(du -s --apparent-size $PLAIN | cut -f1) + SIZE=$(du -s --apparent-size "$PLAIN" | cut -f1) fi if [[ $SIZE -ne 412334 ]] ; then echo "Extracting linux-3.0.tar.gz" - rm -Rf $PLAIN + rm -Rf "$PLAIN" tar xf linux-3.0.tar.gz fi -rm -f $PLAIN/.gocryptfs.reverse.conf -gocryptfs -q -init -reverse -extpass="echo test" -scryptn=10 $PLAIN +rm -f "$PLAIN/.gocryptfs.reverse.conf" +gocryptfs -q -init -reverse -extpass="echo test" -scryptn=10 "$PLAIN" MNT=$(mktemp -d /tmp/linux-3.0.reverse.mnt.XXX) @@ -37,7 +37,7 @@ gocryptfs -q -reverse -extpass="echo test" "$PLAIN" "$MNT" # Execute command, discard all stdout output, print elapsed time # (to stderr, unfortunately). -function etime { +etime() { # Make the bash builtin "time" print out only the elapse wall clock # seconds TIMEFORMAT=%R diff --git a/benchmark.bash b/benchmark.bash index dac575b..52c691f 100755 --- a/benchmark.bash +++ b/benchmark.bash @@ -7,7 +7,7 @@ cd "$(dirname "$0")" MYNAME=$(basename "$0") source tests/fuse-unmount.bash -function usage { +usage() { echo "Usage: $MYNAME [-encfs] [-openssl=true] [-openssl=false] [-dd] [DIR]" } diff --git a/contrib/gocryptfs-maybe.bash b/contrib/gocryptfs-maybe.bash index 78a6b2c..daf3e60 100755 --- a/contrib/gocryptfs-maybe.bash +++ b/contrib/gocryptfs-maybe.bash @@ -12,7 +12,7 @@ # Note that pam_mount ignores messages on stdout which is why printing # to stdout is ok. set -eu -MYNAME=$(basename $0) +MYNAME=$(basename "$0") if [[ $# -lt 2 || $1 == -* ]]; then echo "Usage: $MYNAME CIPHERDIR MOUNTPOINT [-o COMMA-SEPARATED-OPTIONS]" >&2 exit 1 diff --git a/contrib/maxlen.bash b/contrib/maxlen.bash index 99ffb1f..be5f7a6 100755 --- a/contrib/maxlen.bash +++ b/contrib/maxlen.bash @@ -32,8 +32,8 @@ while true ; do echo "error: file $PWD/$NEXT already exists" exit 1 fi - echo -n 2> /dev/null > $NEXT || break - rm $NEXT + echo -n 2> /dev/null > "$NEXT" || break + rm "$NEXT" NAME="$NEXT" done echo "${#NAME}" @@ -47,8 +47,8 @@ if [[ $QUICK -ne 1 ]]; then NAME="" while true ; do NEXT="${NAME}x" - mkdir $NEXT 2> /dev/null || break - rmdir $NEXT + mkdir "$NEXT" 2> /dev/null || break + rmdir "$NEXT" NAME="$NEXT" done MAX_DIRNAME=${#NAME} diff --git a/contrib/mount-ext4-ramdisk.sh b/contrib/mount-ext4-ramdisk.sh index 5ff7ef1..79f18a2 100755 --- a/contrib/mount-ext4-ramdisk.sh +++ b/contrib/mount-ext4-ramdisk.sh @@ -2,7 +2,7 @@ MNT=/mnt/ext4-ramdisk -if mountpoint $MNT ; then +if mountpoint "$MNT" ; then exit 1 fi diff --git a/profiling/ls.bash b/profiling/ls.bash index 5f736b6..d8d55f6 100755 --- a/profiling/ls.bash +++ b/profiling/ls.bash @@ -6,10 +6,10 @@ cd "$(dirname "$0")" ../tests/dl-linux-tarball.bash T=$(mktemp -d) -mkdir $T/a $T/b +mkdir "$T/a" "$T/b" -../gocryptfs -init -quiet -scryptn 10 -extpass "echo test" $T/a -../gocryptfs -quiet -nosyslog -extpass "echo test" $T/a $T/b +../gocryptfs -init -quiet -scryptn 10 -extpass "echo test" "$T/a" +../gocryptfs -quiet -nosyslog -extpass "echo test" "$T/a" "$T/b" # Cleanup trap trap "cd /; fusermount -u -z $T/b; rm -Rf $T/a" EXIT @@ -17,20 +17,20 @@ trap "cd /; fusermount -u -z $T/b; rm -Rf $T/a" EXIT echo "Creating 40000 empty files (linux-3.0.tar.gz contains 36782 files)..." SECONDS=0 for dir in $(seq -w 1 200); do - mkdir $T/b/$dir - ( cd $T/b/$dir ; touch $(seq -w 1 200) ) + mkdir "$T/b/$dir" + ( cd "$T/b/$dir" ; touch $(seq -w 1 200) ) done echo "done, $SECONDS seconds" echo "Remount..." -fusermount -u $T/b -../gocryptfs -quiet -nosyslog -extpass "echo test" -cpuprofile $T/cprof -memprofile $T/mprof \ - $T/a $T/b +fusermount -u "$T/b" +../gocryptfs -quiet -nosyslog -extpass "echo test" -cpuprofile "$T/cprof" -memprofile "$T/mprof" \ + "$T/a" "$T/b" echo "Running ls under profiler (3x)..." for i in 1 2 3; do SECONDS=0 -ls -lR $T/b > /dev/null +ls -lR "$T/b" > /dev/null echo "$i done, $SECONDS seconds" done diff --git a/profiling/streaming-read.bash b/profiling/streaming-read.bash index df75c68..ef55138 100755 --- a/profiling/streaming-read.bash +++ b/profiling/streaming-read.bash @@ -3,25 +3,25 @@ cd "$(dirname "$0")" T=$(mktemp -d) -mkdir $T/a $T/b +mkdir "$T/a" "$T/b" -../gocryptfs -init -quiet -scryptn 10 -extpass "echo test" $T/a -../gocryptfs -quiet -extpass "echo test" $T/a $T/b +../gocryptfs -init -quiet -scryptn 10 -extpass "echo test" "$T/a" +../gocryptfs -quiet -extpass "echo test" "$T/a" "$T/b" # Cleanup trap trap "cd /; fusermount -u -z $T/b; rm -Rf $T/a" EXIT # Write 100MB test file -dd if=/dev/zero of=$T/b/zero bs=1M count=100 status=none +dd if=/dev/zero of="$T/b/zero" bs=1M count=100 status=none # Remount with profiling -fusermount -u $T/b -../gocryptfs -quiet -extpass "echo test" -cpuprofile $T/cprof -memprofile $T/mprof \ - $T/a $T/b +fusermount -u "$T/b" +../gocryptfs -quiet -extpass "echo test" -cpuprofile "$T/cprof" -memprofile "$T/mprof" \ + "$T/a" "$T/b" # Read 10 x 100MB instead of 1 x 1GB to keep the used disk space low for i in $(seq 1 10); do - dd if=$T/b/zero of=/dev/null bs=1M count=100 + dd if="$T/b/zero" of=/dev/null bs=1M count=100 done echo diff --git a/profiling/streaming-write.bash b/profiling/streaming-write.bash index 7732cfb..2cd74ad 100755 --- a/profiling/streaming-write.bash +++ b/profiling/streaming-write.bash @@ -3,18 +3,18 @@ cd "$(dirname "$0")" T=$(mktemp -d) -mkdir $T/a $T/b +mkdir "$T/a" "$T/b" -../gocryptfs -init -quiet -scryptn 10 -extpass "echo test" $T/a -../gocryptfs -quiet -extpass "echo test" -cpuprofile $T/cprof -memprofile $T/mprof \ - $T/a $T/b +../gocryptfs -init -quiet -scryptn 10 -extpass "echo test" "$T/a" +../gocryptfs -quiet -extpass "echo test" -cpuprofile "$T/cprof" -memprofile "$T/mprof" \ + "$T/a" "$T/b" # Cleanup trap trap "cd /; fusermount -u -z $T/b; rm -Rf $T/a" EXIT # Write 10 x 100MB instead of 1 x 1GB to keep the used disk space low for i in $(seq 1 10); do - dd if=/dev/zero of=$T/b/zero bs=1M count=100 + dd if=/dev/zero of="$T/b/zero" bs=1M count=100 done echo diff --git a/profiling/tar-extract.bash b/profiling/tar-extract.bash index 21b2e2b..25f99a6 100755 --- a/profiling/tar-extract.bash +++ b/profiling/tar-extract.bash @@ -6,17 +6,17 @@ cd "$(dirname "$0")" ../tests/dl-linux-tarball.bash T=$(mktemp -d) -mkdir $T/a $T/b +mkdir "$T/a" "$T/b" -../gocryptfs -init -quiet -scryptn 10 -extpass "echo test" $T/a -../gocryptfs -quiet -extpass "echo test" -cpuprofile $T/cprof -memprofile $T/mprof \ - $T/a $T/b +../gocryptfs -init -quiet -scryptn 10 -extpass "echo test" "$T/a" +../gocryptfs -quiet -extpass "echo test" -cpuprofile "$T/cprof" -memprofile "$T/mprof" \ + "$T/a" "$T/b" # Cleanup trap trap "cd /; fusermount -u -z $T/b; rm -Rf $T/a" EXIT echo "Extracting..." -time tar xzf /tmp/linux-3.0.tar.gz -C $T/b +time tar xzf /tmp/linux-3.0.tar.gz -C "$T/b" echo echo "Hint: go tool pprof ../gocryptfs $T/cprof" diff --git a/profiling/write-trace.bash b/profiling/write-trace.bash index 3475140..bb1de1a 100755 --- a/profiling/write-trace.bash +++ b/profiling/write-trace.bash @@ -6,17 +6,17 @@ cd "$(dirname "$0")" T=$(mktemp -d) -mkdir $T/a $T/b +mkdir "$T/a" "$T/b" -../gocryptfs -init -quiet -scryptn 10 -extpass "echo test" $T/a -../gocryptfs -quiet -extpass "echo test" -trace $T/trace \ - $T/a $T/b +../gocryptfs -init -quiet -scryptn 10 -extpass "echo test" "$T/a" +../gocryptfs -quiet -extpass "echo test" -trace "$T/trace" \ + "$T/a" "$T/b" # Cleanup trap trap "cd /; fusermount -u -z $T/b; rm -Rf $T/a" EXIT # Write only 1x100MB, otherwise the trace gets too big. -dd if=/dev/zero of=$T/b/zero bs=1M count=100 +dd if=/dev/zero of="$T/b/zero" bs=1M count=100 echo echo "Hint: go tool trace $T/trace" diff --git a/test.bash b/test.bash index 1af1d43..7c60b8c 100755 --- a/test.bash +++ b/test.bash @@ -16,7 +16,7 @@ TESTDIR=$TMPDIR/gocryptfs-test-parent-$UID mkdir -p "$TESTDIR" LOCKFILE=$TESTDIR/$MYNAME.lock -function unmount_leftovers { +unmount_leftovers() { RET=0 for i in $(mount | grep "$TESTDIR" | cut -f3 -d" "); do echo "Warning: unmounting leftover filesystem: $i" diff --git a/tests/canonical-benchmarks.bash b/tests/canonical-benchmarks.bash index 7b37601..195effe 100755 --- a/tests/canonical-benchmarks.bash +++ b/tests/canonical-benchmarks.bash @@ -45,7 +45,7 @@ echo -n "UNTAR: " etime tar xzf /tmp/linux-3.0.tar.gz sleep 0.1 echo -n "MD5: " -etime md5sum --quiet -c $MD5 +etime md5sum --quiet -c "$MD5" sleep 0.1 echo -n "LS: " etime ls -lR linux-3.0 diff --git a/tests/dl-linux-tarball.bash b/tests/dl-linux-tarball.bash index fa27e37..dfff492 100755 --- a/tests/dl-linux-tarball.bash +++ b/tests/dl-linux-tarball.bash @@ -10,18 +10,18 @@ SIZE_WANT=96675825 SIZE_ACTUAL=0 if [[ -e $TGZ ]]; then if [[ $OSTYPE == linux* ]] ; then - SIZE_ACTUAL=$(stat -c %s $TGZ) + SIZE_ACTUAL=$(stat -c %s "$TGZ") else # Mac OS X - SIZE_ACTUAL=$(stat -f %z $TGZ) + SIZE_ACTUAL=$(stat -f %z "$TGZ") fi fi if [[ $SIZE_ACTUAL -ne $SIZE_WANT ]]; then echo "Downloading linux-3.0.tar.gz" if command -v wget > /dev/null ; then - wget -nv --show-progress -c -O $TGZ $URL + wget -nv --show-progress -c -O "$TGZ" "$URL" else - curl -o $TGZ $URL + curl -o "$TGZ" "$URL" fi fi diff --git a/tests/fuse-unmount.bash b/tests/fuse-unmount.bash index debae29..b36f28c 100755 --- a/tests/fuse-unmount.bash +++ b/tests/fuse-unmount.bash @@ -5,7 +5,7 @@ # # This script can be sourced or executed directly. # -function fuse-unmount { +fuse-unmount() { local MYNAME=$(basename "$BASH_SOURCE") if [[ $# -eq 0 ]] ; then echo "$MYNAME: missing argument" diff --git a/tests/len2elen.sh b/tests/len2elen.sh index 86f2119..3c8cb2a 100755 --- a/tests/len2elen.sh +++ b/tests/len2elen.sh @@ -20,10 +20,10 @@ fi rm -f b/* while [[ $LEN -le 255 ]]; do - touch b/$NAME || break + touch "b/$NAME" || break ELEN=$(ls a | wc -L) - echo $LEN $ELEN - rm b/$NAME + echo "$LEN $ELEN" + rm "b/$NAME" NAME="${NAME}x" LEN=${#NAME} done diff --git a/tests/reverse/linux-tarball-test.bash b/tests/reverse/linux-tarball-test.bash index 26bbb6c..4054c29 100755 --- a/tests/reverse/linux-tarball-test.bash +++ b/tests/reverse/linux-tarball-test.bash @@ -10,12 +10,12 @@ source ../fuse-unmount.bash # Setup dirs ../dl-linux-tarball.bash cd /tmp -WD=$(mktemp -d /tmp/$MYNAME.XXX) +WD=$(mktemp -d "/tmp/$MYNAME.XXX") # Cleanup trap trap "set +u; cd /; fuse-unmount -z $WD/c; fuse-unmount -z $WD/b; rm -rf $WD" EXIT -cd $WD +cd "$WD" mkdir a b c echo "Extracting tarball" tar -x -f /tmp/linux-3.0.tar.gz -C a @@ -30,4 +30,4 @@ gocryptfs -q -extpass="echo test" b c cd c echo "Checking md5 sums" set -o pipefail -md5sum -c $MD5 | pv -l -s 36782 -N "files checked" | (grep -v ": OK" || true) +md5sum -c "$MD5" | pv -l -s 36782 -N "files checked" | (grep -v ": OK" || true) diff --git a/tests/sshfs-benchmark.bash b/tests/sshfs-benchmark.bash index 13c7a0f..4695f8d 100755 --- a/tests/sshfs-benchmark.bash +++ b/tests/sshfs-benchmark.bash @@ -2,7 +2,7 @@ set -eu -function cleanup { +cleanup() { cd "$LOCAL_TMP" fusermount -u gocryptfs.mnt rm -Rf "$SSHFS_TMP" @@ -11,22 +11,22 @@ function cleanup { rm -Rf "$LOCAL_TMP" } -function prepare_mounts { +prepare_mounts() { LOCAL_TMP=$(mktemp -d -t "$MYNAME.XXX") - cd $LOCAL_TMP + cd "$LOCAL_TMP" echo "working directory: $PWD" mkdir sshfs.mnt gocryptfs.mnt - sshfs $HOST:/tmp sshfs.mnt + sshfs "$HOST:/tmp" sshfs.mnt echo "sshfs mounted: $HOST:/tmp -> sshfs.mnt" trap cleanup EXIT SSHFS_TMP=$(mktemp -d "sshfs.mnt/$MYNAME.XXX") - mkdir $SSHFS_TMP/gocryptfs.crypt - gocryptfs -q -init -extpass "echo test" -scryptn=10 $SSHFS_TMP/gocryptfs.crypt - gocryptfs -q -extpass "echo test" $SSHFS_TMP/gocryptfs.crypt gocryptfs.mnt + mkdir "$SSHFS_TMP/gocryptfs.crypt" + gocryptfs -q -init -extpass "echo test" -scryptn=10 "$SSHFS_TMP/gocryptfs.crypt" + gocryptfs -q -extpass "echo test" "$SSHFS_TMP/gocryptfs.crypt" gocryptfs.mnt echo "gocryptfs mounted: $SSHFS_TMP/gocryptfs.crypt -> gocryptfs.mnt" } -function etime { +etime() { T=$(/usr/bin/time -f %e -o /dev/stdout "$@") LC_ALL=C printf %20.2f "$T" } diff --git a/tests/stress_tests/extractloop.bash b/tests/stress_tests/extractloop.bash index cdd0c25..1f78a5e 100755 --- a/tests/stress_tests/extractloop.bash +++ b/tests/stress_tests/extractloop.bash @@ -28,17 +28,17 @@ source ../fuse-unmount.bash # Setup dirs ../dl-linux-tarball.bash -cd $TMPDIR +cd "$TMPDIR" EXTRACTLOOP_TMPDIR=$TMPDIR/extractloop_tmpdir -mkdir -p $EXTRACTLOOP_TMPDIR -CRYPT=$(mktemp -d $EXTRACTLOOP_TMPDIR/XXX) +mkdir -p "$EXTRACTLOOP_TMPDIR" +CRYPT=$(mktemp -d "$EXTRACTLOOP_TMPDIR/XXX") CSV=$CRYPT.csv MNT=$CRYPT.mnt -mkdir $MNT +mkdir "$MNT" -function check_md5sums { +check_md5sums() { if command -v md5sum > /dev/null ; then - md5sum --status -c $1 + md5sum --status -c "$1" else # MacOS / darwin which do not have the md5sum utility # installed by default @@ -52,50 +52,50 @@ FS="" if [ $# -eq 1 ] && [ "$1" == "-encfs" ]; then FS=encfs echo "Testing EncFS" - encfs --extpass="echo test" --standard $CRYPT $MNT > /dev/null + encfs --extpass="echo test" --standard "$CRYPT" "$MNT" > /dev/null elif [ $# -eq 1 ] && [ "$1" == "-loopback" ]; then FS=loopback echo "Testing go-fuse loopback" rm -f /tmp/loopback*.memprof - loopback -memprofile=/tmp/loopback $MNT $CRYPT & + loopback -memprofile=/tmp/loopback "$MNT" "$CRYPT" & FSPID=$(jobs -p) disown else FS=gocryptfs echo "Testing gocryptfs" - gocryptfs -q -init -extpass="echo test" -scryptn=10 $CRYPT - gocryptfs -q -extpass="echo test" -nosyslog -fg $CRYPT $MNT & + gocryptfs -q -init -extpass="echo test" -scryptn=10 "$CRYPT" + gocryptfs -q -extpass="echo test" -nosyslog -fg "$CRYPT" "$MNT" & FSPID=$(jobs -p) disown - #gocryptfs -q -extpass="echo test" -nosyslog -memprofile /tmp/extractloop-mem $CRYPT $MNT + #gocryptfs -q -extpass="echo test" -nosyslog -memprofile /tmp/extractloop-mem "$CRYPT" "$MNT" fi echo "Test dir: $CRYPT" # Sleep to make sure the FS is already mounted on MNT sleep 1 -cd $MNT +cd "$MNT" -ln -v -sTf $CSV /tmp/extractloop.csv 2> /dev/null || true # fails on MacOS, ignore +ln -v -sTf "$CSV" /tmp/extractloop.csv 2> /dev/null || true # fails on MacOS, ignore # Cleanup trap # Note: gocryptfs may have already umounted itself because bash relays SIGINT # Just ignore unmount errors. trap cleanup_exit EXIT -function cleanup_exit { +cleanup_exit() { if [[ $FS == loopback ]]; then # SIGUSR1 causes loopback to write the memory profile to disk kill -USR1 $FSPID fi cd / - rm -Rf $CRYPT - fuse-unmount -z $MNT || true - rmdir $MNT + rm -Rf "$CRYPT" + fuse-unmount -z "$MNT" || true + rmdir "$MNT" } -function loop { +loop() { ID=$1 - mkdir $ID - cd $ID + mkdir "$ID" + cd "$ID" echo "[looper $ID] Starting" @@ -107,20 +107,20 @@ function loop { tar xf /tmp/linux-3.0.tar.gz --exclude linux-3.0/arch/microblaze/boot/dts/system.dts # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ # Exclude the one symlink in the tarball - causes problems on MacOS: "Can't set permissions to 0755" - check_md5sums $MD5 + check_md5sums "$MD5" rm -R linux-3.0 t2=$SECONDS delta=$((t2-t1)) if [[ $FSPID -gt 0 && -d /proc ]]; then RSS=$(grep VmRSS /proc/$FSPID/status | tr -s ' ' | cut -f2 -d ' ') - echo "$N,$SECONDS,$RSS,$delta" >> $CSV + echo "$N,$SECONDS,$RSS,$delta" >> "$CSV" fi echo "[looper $ID] Iteration $N done, $delta seconds, RSS $RSS kiB" - let N=$N+1 + N=$((N+1)) done } -function memprof { +memprof() { while true; do kill -USR1 $FSPID sleep 60 diff --git a/tests/stress_tests/fsstress-gocryptfs.bash b/tests/stress_tests/fsstress-gocryptfs.bash index a7aa811..08cdd45 100755 --- a/tests/stress_tests/fsstress-gocryptfs.bash +++ b/tests/stress_tests/fsstress-gocryptfs.bash @@ -19,7 +19,7 @@ export TMPDIR=${TMPDIR:-/var/tmp} DEBUG=${DEBUG:-0} cd "$(dirname "$0")" -MYNAME=$(basename $0) +MYNAME=$(basename "$0") source ../fuse-unmount.bash # fsstress binary @@ -32,23 +32,23 @@ then fi # Backing directory -DIR=$(mktemp -d $TMPDIR/$MYNAME.XXX) +DIR=$(mktemp -d "$TMPDIR/$MYNAME.XXX") # Mountpoint MNT="$DIR.mnt" -mkdir $MNT +mkdir "$MNT" # Set the GOPATH variable to the default if it is empty GOPATH=$(go env GOPATH) # Clean up old mounts -for i in $(mount | cut -d" " -f3 | grep $TMPDIR/$MYNAME) ; do - fusermount -u $i +for i in $(mount | cut -d" " -f3 | grep "$TMPDIR/$MYNAME") ; do + fusermount -u "$i" done # FS-specific compile and mount if [[ $MYNAME = fsstress-loopback.bash ]]; then echo -n "Recompile go-fuse loopback: " - cd $GOPATH/src/github.com/hanwen/go-fuse/example/loopback + cd "$GOPATH/src/github.com/hanwen/go-fuse/example/loopback" git describe go build && go install OPTS="-q" @@ -59,20 +59,20 @@ if [[ $MYNAME = fsstress-loopback.bash ]]; then disown elif [[ $MYNAME = fsstress-gocryptfs.bash ]]; then echo "Recompile gocryptfs" - cd $GOPATH/src/github.com/rfjakob/gocryptfs + cd "$GOPATH/src/github.com/rfjakob/gocryptfs" ./build.bash # also prints the version - $GOPATH/bin/gocryptfs -q -init -extpass "echo test" -scryptn=10 $DIR - $GOPATH/bin/gocryptfs -q -extpass "echo test" -nosyslog -fusedebug=$DEBUG $DIR $MNT + $GOPATH/bin/gocryptfs -q -init -extpass "echo test" -scryptn=10 "$DIR" + $GOPATH/bin/gocryptfs -q -extpass "echo test" -nosyslog -fusedebug="$DEBUG" "$DIR" "$MNT" elif [[ $MYNAME = fsstress-encfs.bash ]]; then - encfs --extpass "echo test" --standard $DIR $MNT + encfs --extpass "echo test" --standard "$DIR" "$MNT" else - echo Unknown mode: $MYNAME + echo "Unknown mode: $MYNAME" exit 1 fi sleep 0.5 echo -n "Waiting for mount: " -while ! grep "$(basename $MNT) fuse" /proc/self/mounts > /dev/null +while ! grep "$(basename "$MNT") fuse" /proc/self/mounts > /dev/null do sleep 1 echo -n x @@ -87,26 +87,26 @@ N=1 while true do echo "$N ................................. $(date)" - mkdir $MNT/fsstress.1 + mkdir "$MNT/fsstress.1" echo -n " fsstress.1 " - $FSSTRESS -r -m 8 -n 1000 -d $MNT/fsstress.1 & + $FSSTRESS -r -m 8 -n 1000 -d "$MNT/fsstress.1" & wait - mkdir $MNT/fsstress.2 + mkdir "$MNT/fsstress.2" echo -n " fsstress.2 " - $FSSTRESS -p 20 -r -m 8 -n 1000 -d $MNT/fsstress.2 & + $FSSTRESS -p 20 -r -m 8 -n 1000 -d "$MNT/fsstress.2" & wait - mkdir $MNT/fsstress.3 + mkdir "$MNT/fsstress.3" echo -n " fsstress.3 " $FSSTRESS -p 4 -z -f rmdir=10 -f link=10 -f creat=10 -f mkdir=10 \ -f rename=30 -f stat=30 -f unlink=30 -f truncate=20 -m 8 \ - -n 1000 -d $MNT/fsstress.3 & + -n 1000 -d "$MNT/fsstress.3" & wait echo " rm" - rm -Rf $MNT/* + rm -Rf "$MNT"/* - let N=$N+1 + N=$((N+1)) done diff --git a/tests/stress_tests/parallel_cp.sh b/tests/stress_tests/parallel_cp.sh index ad98e5e..cd08d31 100755 --- a/tests/stress_tests/parallel_cp.sh +++ b/tests/stress_tests/parallel_cp.sh @@ -18,32 +18,32 @@ if [[ -z $TMPDIR ]]; then fi cd "$(dirname "$0")" -MYNAME=$(basename $0) +MYNAME=$(basename "$0") source ../fuse-unmount.bash # Set the GOPATH variable to the default if it is empty GOPATH=$(go env GOPATH) # Backing directory -DIR=$(mktemp -d $TMPDIR/$MYNAME.XXX) -$GOPATH/bin/gocryptfs -q -init -extpass "echo test" -scryptn=10 $DIR +DIR=$(mktemp -d "$TMPDIR/$MYNAME.XXX") +$GOPATH/bin/gocryptfs -q -init -extpass "echo test" -scryptn=10 "$DIR" # Mountpoint MNT="$DIR.mnt" -mkdir $MNT -$GOPATH/bin/gocryptfs -q -extpass "echo test" -nosyslog $DIR $MNT +mkdir "$MNT" +$GOPATH/bin/gocryptfs -q -extpass "echo test" -nosyslog "$DIR" "$MNT" echo "Mounted gocryptfs $DIR at $MNT" # Cleanup trap trap "cd / ; fuse-unmount -z $MNT ; rm -rf $DIR $MNT" EXIT -cd $MNT +cd "$MNT" SECONDS=0 echo "creating files with dd" mkdir -p origin for i in $(seq 1 778) ; do - dd if=/dev/zero of=origin/file_$i bs=8192 count=1 status=none + dd if=/dev/zero of="origin/file_$i" bs=8192 count=1 status=none done # Perform the shell expansion only once and store the list ORIGIN_FILES=origin/* @@ -51,7 +51,7 @@ ORIGIN_FILES=origin/* echo -n "cp starting: " for i in $(seq 1 100) ; do echo -n "$i " - (mkdir sub_$i && cp $ORIGIN_FILES sub_$i ; echo -n "$i ") & + (mkdir "sub_$i" && cp $ORIGIN_FILES "sub_$i" ; echo -n "$i ") & done echo diff --git a/tests/stress_tests/pingpong.bash b/tests/stress_tests/pingpong.bash index 218b6e8..4b8346e 100755 --- a/tests/stress_tests/pingpong.bash +++ b/tests/stress_tests/pingpong.bash @@ -13,7 +13,7 @@ renice 19 $$ cd "$(dirname "$0")" MD5="$PWD/linux-3.0.md5sums" -MYNAME=$(basename $0) +MYNAME=$(basename "$0") source ../fuse-unmount.bash # Setup @@ -22,48 +22,48 @@ cd /tmp PING=$(mktemp -d ping.XXX) PONG=$(mktemp -d pong.XXX) -mkdir $PING.mnt $PONG.mnt +mkdir "$PING.mnt" "$PONG.mnt" # Cleanup trap # Note: gocryptfs may have already umounted itself because bash relays SIGINT # Just ignore unmount errors. trap "set +e ; cd /tmp; fuse-unmount -z $PING.mnt ; fuse-unmount -z $PONG.mnt ; rm -rf $PING $PONG $PING.mnt $PONG.mnt" EXIT -gocryptfs -q -init -extpass="echo test" -scryptn=10 $PING -gocryptfs -q -init -extpass="echo test" -scryptn=10 $PONG +gocryptfs -q -init -extpass="echo test" -scryptn=10 "$PING" +gocryptfs -q -init -extpass="echo test" -scryptn=10 "$PONG" -gocryptfs -q -extpass="echo test" -nosyslog $PING $PING.mnt -gocryptfs -q -extpass="echo test" -nosyslog $PONG $PONG.mnt +gocryptfs -q -extpass="echo test" -nosyslog "$PING" "$PING.mnt" +gocryptfs -q -extpass="echo test" -nosyslog "$PONG" "$PONG.mnt" echo "Initial extract" -tar xf /tmp/linux-3.0.tar.gz -C $PING.mnt +tar xf /tmp/linux-3.0.tar.gz -C "$PING.mnt" -function move_and_md5 { - if [ $MYNAME = pingpong-rsync.bash ]; then +move_and_md5() { + if [ "$MYNAME" = "pingpong-rsync.bash" ]; then echo -n "rsync " - rsync -a --remove-source-files $1 $2 - find $1 -type d -delete + rsync -a --remove-source-files "$1" "$2" + find "$1" -type d -delete else echo -n "mv " - mv $1 $2 + mv "$1" "$2" fi - if [ -e $1 ]; then + if [ -e "$1" ]; then echo "error: source directory $1 was not removed" exit 1 fi - cd $2 + cd "$2" echo -n "md5 " - md5sum --status -c $MD5 + md5sum --status -c "$MD5" cd .. } N=1 while true; do echo -n "$N: " - move_and_md5 $PING.mnt/linux-3.0 $PONG.mnt - move_and_md5 $PONG.mnt/linux-3.0 $PING.mnt + move_and_md5 "$PING.mnt/linux-3.0" "$PONG.mnt" + move_and_md5 "$PONG.mnt/linux-3.0" "$PING.mnt" date +%H:%M:%S - let N=$N+1 + N=$((N+1)) done wait From 6cb03b54fe4c13a4534793c4c900b7894ae2b387 Mon Sep 17 00:00:00 2001 From: a1346054 <36859588+a1346054@users.noreply.github.com> Date: Tue, 31 Aug 2021 17:05:29 +0000 Subject: [PATCH 10/59] *: fix spelling --- Documentation/MANPAGE.md | 2 +- test.bash | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/MANPAGE.md b/Documentation/MANPAGE.md index acb0ff0..bc7a798 100644 --- a/Documentation/MANPAGE.md +++ b/Documentation/MANPAGE.md @@ -368,7 +368,7 @@ Mount the filesystem read-write (`-rw`, default) or read-only (`-ro`). If both are specified, `-ro` takes precedence. #### -reverse -See the `-reverse` section in INIT FLAGS. You need to specifiy the +See the `-reverse` section in INIT FLAGS. You need to specify the `-reverse` option both at `-init` and at mount. #### -serialize_reads diff --git a/test.bash b/test.bash index 7c60b8c..99884a8 100755 --- a/test.bash +++ b/test.bash @@ -28,7 +28,7 @@ unmount_leftovers() { ( # Prevent multiple parallel test.bash instances as this causes -# all kinds of mayham +# all kinds of mayhem if ! command -v flock > /dev/null ; then echo "flock is not available, skipping" elif ! flock -n 200 ; then From 7c2255be9027551a1df54b3cb58ea9493857d391 Mon Sep 17 00:00:00 2001 From: a1346054 <36859588+a1346054@users.noreply.github.com> Date: Tue, 31 Aug 2021 17:05:38 +0000 Subject: [PATCH 11/59] *: trim trailing whitespace --- Documentation/MANPAGE.md | 4 ++-- Documentation/XFSTESTS.md | 14 +++++++------- README.md | 8 ++++---- benchmark.bash | 1 - package-release-tarballs.bash | 2 +- profiling/write-trace.bash | 1 - tests/stress_tests/fsstress-gocryptfs.bash | 1 - 7 files changed, 14 insertions(+), 17 deletions(-) diff --git a/Documentation/MANPAGE.md b/Documentation/MANPAGE.md index bc7a798..7371fc2 100644 --- a/Documentation/MANPAGE.md +++ b/Documentation/MANPAGE.md @@ -191,7 +191,7 @@ Show all invalid filenames: Create a control socket at the specified location. The socket can be used to decrypt and encrypt paths inside the filesystem. When using this option, make sure that the directory you place the socket in is -not world-accessible. For example, `/run/user/UID/my.socket` would +not world-accessible. For example, `/run/user/UID/my.socket` would be suitable. #### -dev, -nodev @@ -257,7 +257,7 @@ filesystem that doesn't properly support UNIX ownership and permissions. #### -forcedecode Force decode of encrypted files even if the integrity check fails, instead of -failing with an IO error. Warning messages are still printed to syslog if corrupted +failing with an IO error. Warning messages are still printed to syslog if corrupted files are encountered. It can be useful to recover files from disks with bad sectors or other corrupted media. It shall not be used if the origin of corruption is unknown, specially diff --git a/Documentation/XFSTESTS.md b/Documentation/XFSTESTS.md index bbf19e7..d5421cc 100644 --- a/Documentation/XFSTESTS.md +++ b/Documentation/XFSTESTS.md @@ -61,7 +61,7 @@ Needs further analysis: `Too many open files` ## Full Test Output ``` -0 jakob@brikett:~/code/fuse-xfstests$ sudo ./check-gocryptfs +0 jakob@brikett:~/code/fuse-xfstests$ sudo ./check-gocryptfs gocryptfs v1.7.1; go-fuse v2.0.2-4-g8458b8a; 2019-10-10 go1.12.9 linux/amd64 fuse-xfstests nlink0/dff383ab Thu 10 Oct 2019 08:31:43 PM UTC @@ -146,12 +146,12 @@ generic/062 - output mismatch (see /home/jakob.donotbackup/code/fuse-xfstests/re --- tests/generic/062.out 2018-01-20 14:29:39.067451950 +0100 +++ /home/jakob.donotbackup/code/fuse-xfstests/results//generic/062.out.bad 2019-10-10 22:34:34.290196880 +0200 @@ -13,7 +13,7 @@ - + *** set/get one initially empty attribute # file: SCRATCH_MNT/reg -user.name +user.name="" - + *** overwrite empty, set several new attributes ... (Run 'diff -u tests/generic/062.out /home/jakob.donotbackup/code/fuse-xfstests/results//generic/062.out.bad' to see the entire diff) @@ -190,7 +190,7 @@ generic/093 - output mismatch (see /home/jakob.donotbackup/code/fuse-xfstests/re +++ /home/jakob.donotbackup/code/fuse-xfstests/results//generic/093.out.bad 2019-10-10 22:45:22.194059446 +0200 @@ -1,15 +1,22 @@ QA output created by 093 - + **** Verifying that appending to file clears capabilities **** -file = cap_chown+ep +Failed to set capabilities on file '/var/tmp/check-gocryptfs/testdir/093.file' (Operation not supported) @@ -206,12 +206,12 @@ generic/097 - output mismatch (see /home/jakob.donotbackup/code/fuse-xfstests/re +++ /home/jakob.donotbackup/code/fuse-xfstests/results//generic/097.out.bad 2019-10-10 22:45:23.382064979 +0200 @@ -110,18 +110,16 @@ *** Test out the trusted namespace *** - + set EA : +setfattr: TEST_DIR/foo: Operation not supported - + set EA : - + ... (Run 'diff -u tests/generic/097.out /home/jakob.donotbackup/code/fuse-xfstests/results//generic/097.out.bad' to see the entire diff) generic/098 1s ... 0s diff --git a/README.md b/README.md index 7f47139..e1104b8 100644 --- a/README.md +++ b/README.md @@ -169,9 +169,9 @@ Example for a CPU with AES-NI: ``` $ ./gocryptfs -speed gocryptfs v2.0; go-fuse v2.1.1-0.20210508151621-62c5aa1919a7; 2021-06-06 go1.16.5 linux/amd64 -AES-GCM-256-OpenSSL 536.63 MB/s +AES-GCM-256-OpenSSL 536.63 MB/s AES-GCM-256-Go 831.84 MB/s (selected in auto mode) -AES-SIV-512-Go 155.85 MB/s +AES-SIV-512-Go 155.85 MB/s XChaCha20-Poly1305-Go 700.02 MB/s (benchmark only, not selectable yet) ``` @@ -181,7 +181,7 @@ tarball, recursively listing and finally deleting it. The output will look like this: ``` -$ ./benchmark.bash +$ ./benchmark.bash Testing gocryptfs at /tmp/benchmark.bash.xFD: gocryptfs v2.0; go-fuse v2.1.1-0.20210508151621-62c5aa1919a7; 2021-06-06 go1.16.5 linux/amd64 WRITE: 262144000 bytes (262 MB, 250 MiB) copied, 0,698174 s, 375 MB/s READ: 262144000 bytes (262 MB, 250 MiB) copied, 0,268916 s, 975 MB/s @@ -314,7 +314,7 @@ v1.7.1, 2019-10-06 * Create `gocryptfs.diriv` files with 0440 permissions to make it easier to share an encrypted folder via a network drive ([#387](https://github.com/rfjakob/gocryptfs/issues/387)). - Note: as a security precaution, the owner must still manually + Note: as a security precaution, the owner must still manually `chmod gocryptfs.conf 0440` to allow mounting. * Allow the `nofail` option in `/etc/fstab` * `-passwd` can now change the `-scryptn` parameter for existing filesystems diff --git a/benchmark.bash b/benchmark.bash index 52c691f..fb99c65 100755 --- a/benchmark.bash +++ b/benchmark.bash @@ -104,4 +104,3 @@ if [[ $DD_ONLY -eq 1 ]]; then else ./tests/canonical-benchmarks.bash "$MNT" fi - diff --git a/package-release-tarballs.bash b/package-release-tarballs.bash index ac651bf..fde214e 100755 --- a/package-release-tarballs.bash +++ b/package-release-tarballs.bash @@ -69,7 +69,7 @@ package_static_binary() { local TARGZ TARGZ=$TARBALL.gz - tar --owner=root --group=root --create -vf "$TARBALL" gocryptfs + tar --owner=root --group=root --create -vf "$TARBALL" gocryptfs tar --owner=root --group=root --append -vf "$TARBALL" -C gocryptfs-xray gocryptfs-xray tar --owner=root --group=root --append -vf "$TARBALL" -C Documentation gocryptfs.1 gocryptfs-xray.1 diff --git a/profiling/write-trace.bash b/profiling/write-trace.bash index bb1de1a..707e1d3 100755 --- a/profiling/write-trace.bash +++ b/profiling/write-trace.bash @@ -20,4 +20,3 @@ dd if=/dev/zero of="$T/b/zero" bs=1M count=100 echo echo "Hint: go tool trace $T/trace" - diff --git a/tests/stress_tests/fsstress-gocryptfs.bash b/tests/stress_tests/fsstress-gocryptfs.bash index 08cdd45..03352f2 100755 --- a/tests/stress_tests/fsstress-gocryptfs.bash +++ b/tests/stress_tests/fsstress-gocryptfs.bash @@ -109,4 +109,3 @@ do N=$((N+1)) done - From cbf282861b450db3a475f77686e3658c95c2a5a0 Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Wed, 1 Sep 2021 10:28:33 +0200 Subject: [PATCH 12/59] tests/matrix: don't leak fds in TestConcurrentReadCreate We leaked a file descriptor for each empty file we encountered. --- tests/matrix/concurrency_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/matrix/concurrency_test.go b/tests/matrix/concurrency_test.go index 04b7c96..d133c45 100644 --- a/tests/matrix/concurrency_test.go +++ b/tests/matrix/concurrency_test.go @@ -110,6 +110,7 @@ func TestConcurrentReadCreate(t *testing.T) { continue } n, err := f.Read(buf0) + f.Close() if err == io.EOF { i++ continue @@ -122,7 +123,6 @@ func TestConcurrentReadCreate(t *testing.T) { // Calling t.Fatal() from a goroutine hangs the test so we use log.Fatal log.Fatalf("%s: content mismatch: have=%q want=%q", t.Name(), string(buf), string(content)) } - f.Close() } wg.Done() }() From c9b090770ac457090224c407aefa47c55394fb70 Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Thu, 2 Sep 2021 09:30:28 +0200 Subject: [PATCH 13/59] stupidgcm: add chacha20poly1305 via openssl "stupidChacha20poly1305". XChaCha will build upon this. --- internal/stupidgcm/stupidchacha.go | 222 ++++++++++++++++++++++++ internal/stupidgcm/stupidchacha_test.go | 171 ++++++++++++++++++ 2 files changed, 393 insertions(+) create mode 100644 internal/stupidgcm/stupidchacha.go create mode 100644 internal/stupidgcm/stupidchacha_test.go diff --git a/internal/stupidgcm/stupidchacha.go b/internal/stupidgcm/stupidchacha.go new file mode 100644 index 0000000..e2f6407 --- /dev/null +++ b/internal/stupidgcm/stupidchacha.go @@ -0,0 +1,222 @@ +// +build !without_openssl + +package stupidgcm + +// #include +// #cgo pkg-config: libcrypto +import "C" + +import ( + "crypto/cipher" + "fmt" + "log" + "unsafe" + + "golang.org/x/crypto/chacha20poly1305" +) + +type stupidChacha20poly1305 struct { + key []byte +} + +// Verify that we satisfy the cipher.AEAD interface +var _ cipher.AEAD = &stupidChacha20poly1305{} + +func newChacha20poly1305(keyIn []byte) cipher.AEAD { + if len(keyIn) != chacha20poly1305.KeySize { + log.Panicf("Only %d-byte keys are supported, you passed %d bytes", chacha20poly1305.KeySize, len(keyIn)) + } + // Create a private copy of the key + key := append([]byte{}, keyIn...) + return &stupidChacha20poly1305{key: key} +} + +// NonceSize returns the required size of the nonce / IV. +func (g *stupidChacha20poly1305) NonceSize() int { + return chacha20poly1305.NonceSize +} + +// Overhead returns the number of bytes that are added for authentication. +func (g *stupidChacha20poly1305) Overhead() int { + return tagLen +} + +// Seal encrypts "in" using "iv" and "authData" and append the result to "dst" +func (g *stupidChacha20poly1305) Seal(dst, iv, in, authData []byte) []byte { + if len(iv) != g.NonceSize() { + log.Panicf("Only %d-byte IVs are supported, you passed %d bytes", g.NonceSize(), len(iv)) + } + if len(in) == 0 { + log.Panic("Zero-length input data is not supported") + } + if len(g.key) != chacha20poly1305.KeySize { + log.Panicf("Wrong key length: %d. Key has been wiped?", len(g.key)) + } + + // If the "dst" slice is large enough we can use it as our output buffer + outLen := len(in) + tagLen + var buf []byte + inplace := false + if cap(dst)-len(dst) >= outLen { + inplace = true + buf = dst[len(dst) : len(dst)+outLen] + } else { + buf = make([]byte, outLen) + } + + // https://wiki.openssl.org/index.php/EVP_Authenticated_Encryption_and_Decryption#Authenticated_Encryption_using_GCM_mode + + // Create scratch space "context" + ctx := C.EVP_CIPHER_CTX_new() + if ctx == nil { + log.Panic("EVP_CIPHER_CTX_new failed") + } + + // Set cipher + if C.EVP_EncryptInit_ex(ctx, C.EVP_chacha20_poly1305(), nil, nil, nil) != 1 { + log.Panic("EVP_EncryptInit_ex I failed") + } + + // Set key and IV + if C.EVP_EncryptInit_ex(ctx, nil, nil, (*C.uchar)(&g.key[0]), (*C.uchar)(&iv[0])) != 1 { + log.Panic("EVP_EncryptInit_ex II failed") + } + + // Provide authentication data + var resultLen C.int + if C.EVP_EncryptUpdate(ctx, nil, &resultLen, (*C.uchar)(&authData[0]), C.int(len(authData))) != 1 { + log.Panic("EVP_EncryptUpdate authData failed") + } + if int(resultLen) != len(authData) { + log.Panicf("Unexpected length %d", resultLen) + } + + // Encrypt "in" into "buf" + if C.EVP_EncryptUpdate(ctx, (*C.uchar)(&buf[0]), &resultLen, (*C.uchar)(&in[0]), C.int(len(in))) != 1 { + log.Panic("EVP_EncryptUpdate failed") + } + if int(resultLen) != len(in) { + log.Panicf("Unexpected length %d", resultLen) + } + + // Finalise encryption + // Because GCM is a stream encryption, this will not write out any data. + dummy := make([]byte, 16) + if C.EVP_EncryptFinal_ex(ctx, (*C.uchar)(&dummy[0]), &resultLen) != 1 { + log.Panic("EVP_EncryptFinal_ex failed") + } + if resultLen != 0 { + log.Panicf("Unexpected length %d", resultLen) + } + + // Get MAC tag and append it to the ciphertext in "buf" + if C.EVP_CIPHER_CTX_ctrl(ctx, C.EVP_CTRL_AEAD_GET_TAG, tagLen, (unsafe.Pointer)(&buf[len(in)])) != 1 { + log.Panic("EVP_CIPHER_CTX_ctrl EVP_CTRL_AEAD_GET_TAG failed") + } + + // Free scratch space + C.EVP_CIPHER_CTX_free(ctx) + + if inplace { + return dst[:len(dst)+outLen] + } + return append(dst, buf...) +} + +// Open decrypts "in" using "iv" and "authData" and append the result to "dst" +func (g *stupidChacha20poly1305) Open(dst, iv, in, authData []byte) ([]byte, error) { + if len(iv) != g.NonceSize() { + log.Panicf("Only %d-byte IVs are supported", g.NonceSize()) + } + if len(g.key) != chacha20poly1305.KeySize { + log.Panicf("Wrong key length: %d. Key has been wiped?", len(g.key)) + } + if len(in) <= tagLen { + return nil, fmt.Errorf("stupidChacha20poly1305: input data too short (%d bytes)", len(in)) + } + + // If the "dst" slice is large enough we can use it as our output buffer + outLen := len(in) - tagLen + var buf []byte + inplace := false + if cap(dst)-len(dst) >= outLen { + inplace = true + buf = dst[len(dst) : len(dst)+outLen] + } else { + buf = make([]byte, len(in)-tagLen) + } + + ciphertext := in[:len(in)-tagLen] + tag := in[len(in)-tagLen:] + + // https://wiki.openssl.org/index.php/EVP_Authenticated_Encryption_and_Decryption#Authenticated_Encryption_using_GCM_mode + + // Create scratch space "context" + ctx := C.EVP_CIPHER_CTX_new() + if ctx == nil { + log.Panic("EVP_CIPHER_CTX_new failed") + } + + // Set cipher to AES-256 + if C.EVP_DecryptInit_ex(ctx, C.EVP_chacha20_poly1305(), nil, nil, nil) != 1 { + log.Panic("EVP_DecryptInit_ex I failed") + } + + // Set key and IV + if C.EVP_DecryptInit_ex(ctx, nil, nil, (*C.uchar)(&g.key[0]), (*C.uchar)(&iv[0])) != 1 { + log.Panic("EVP_DecryptInit_ex II failed") + } + + // Set expected MAC tag + if C.EVP_CIPHER_CTX_ctrl(ctx, C.EVP_CTRL_AEAD_SET_TAG, tagLen, (unsafe.Pointer)(&tag[0])) != 1 { + log.Panic("EVP_CIPHER_CTX_ctrl EVP_CTRL_AEAD_SET_TAG failed") + } + + // Provide authentication data + var resultLen C.int + if C.EVP_DecryptUpdate(ctx, nil, &resultLen, (*C.uchar)(&authData[0]), C.int(len(authData))) != 1 { + log.Panic("EVP_DecryptUpdate authData failed") + } + if int(resultLen) != len(authData) { + log.Panicf("Unexpected length %d", resultLen) + } + + // Decrypt "ciphertext" into "buf" + if C.EVP_DecryptUpdate(ctx, (*C.uchar)(&buf[0]), &resultLen, (*C.uchar)(&ciphertext[0]), C.int(len(ciphertext))) != 1 { + log.Panic("EVP_DecryptUpdate failed") + } + if int(resultLen) != len(ciphertext) { + log.Panicf("Unexpected length %d", resultLen) + } + + // Check MAC + dummy := make([]byte, 16) + res := C.EVP_DecryptFinal_ex(ctx, (*C.uchar)(&dummy[0]), &resultLen) + if resultLen != 0 { + log.Panicf("Unexpected length %d", resultLen) + } + + // Free scratch space + C.EVP_CIPHER_CTX_free(ctx) + + if res != 1 { + return nil, ErrAuth + } + + if inplace { + return dst[:len(dst)+outLen], nil + } + return append(dst, buf...), nil +} + +// Wipe tries to wipe the AES key from memory by overwriting it with zeros +// and setting the reference to nil. +// +// This is not bulletproof due to possible GC copies, but +// still raises to bar for extracting the key. +func (g *stupidChacha20poly1305) Wipe() { + for i := range g.key { + g.key[i] = 0 + } + g.key = nil +} diff --git a/internal/stupidgcm/stupidchacha_test.go b/internal/stupidgcm/stupidchacha_test.go new file mode 100644 index 0000000..010055f --- /dev/null +++ b/internal/stupidgcm/stupidchacha_test.go @@ -0,0 +1,171 @@ +// +build !without_openssl + +// We compare against Go's built-in GCM implementation. Since stupidgcm only +// supports 128-bit IVs and Go only supports that from 1.5 onward, we cannot +// run these tests on older Go versions. +package stupidgcm + +import ( + "bytes" + "encoding/hex" + "testing" + + "golang.org/x/crypto/chacha20poly1305" +) + +// TestEncryptDecrypt encrypts and decrypts using both stupidgcm and Go's built-in +// GCM implementation and verifies that the results are identical. +func TestEncryptDecryptChacha(t *testing.T) { + key := randBytes(32) + sGCM := newChacha20poly1305(key) + authData := randBytes(24) + iv := randBytes(sGCM.NonceSize()) + dst := make([]byte, 71) // 71 = random length + + gGCM, err := chacha20poly1305.New(key) + if err != nil { + t.Fatal(err) + } + + // Check all block sizes from 1 to 5000 + for i := 1; i < 5000; i++ { + in := make([]byte, i) + + sOut := sGCM.Seal(dst, iv, in, authData) + gOut := gGCM.Seal(dst, iv, in, authData) + + // Ciphertext must be identical to Go GCM + if !bytes.Equal(sOut, gOut) { + t.Fatalf("Compare failed for encryption, size %d", i) + t.Log("sOut:") + t.Log("\n" + hex.Dump(sOut)) + t.Log("gOut:") + t.Log("\n" + hex.Dump(gOut)) + } + + sOut2, sErr := sGCM.Open(dst, iv, sOut[len(dst):], authData) + if sErr != nil { + t.Fatal(sErr) + } + gOut2, gErr := gGCM.Open(dst, iv, gOut[len(dst):], authData) + if gErr != nil { + t.Fatal(gErr) + } + + // Plaintext must be identical to Go GCM + if !bytes.Equal(sOut2, gOut2) { + t.Fatalf("Compare failed for decryption, size %d", i) + } + } +} + +// Seal re-uses the "dst" buffer it is large enough. +// Check that this works correctly by testing different "dst" capacities from +// 5000 to 16 and "in" lengths from 1 to 5000. +func TestInplaceSealChacha(t *testing.T) { + key := randBytes(32) + sGCM := newChacha20poly1305(key) + authData := randBytes(24) + iv := randBytes(sGCM.NonceSize()) + + gGCM, err := chacha20poly1305.New(key) + if err != nil { + t.Fatal(err) + } + max := 5016 + // Check all block sizes from 1 to 5000 + for i := 1; i < max-len(iv); i++ { + in := make([]byte, i) + dst := make([]byte, max-i) + dst = dst[:len(iv)] + + sOut := sGCM.Seal(dst, iv, in, authData) + dst2 := make([]byte, len(iv)) + gOut := gGCM.Seal(dst2, iv, in, authData) + + // Ciphertext must be identical to Go GCM + if !bytes.Equal(sOut, gOut) { + t.Fatalf("Compare failed for encryption, size %d", i) + t.Log("sOut:") + t.Log("\n" + hex.Dump(sOut)) + t.Log("gOut:") + t.Log("\n" + hex.Dump(gOut)) + } + } +} + +// Open re-uses the "dst" buffer it is large enough. +// Check that this works correctly by testing different "dst" capacities from +// 5000 to 16 and "in" lengths from 1 to 5000. +func TestInplaceOpenChacha(t *testing.T) { + key := randBytes(32) + sGCM := newChacha20poly1305(key) + authData := randBytes(24) + iv := randBytes(sGCM.NonceSize()) + + gGCM, err := chacha20poly1305.New(key) + if err != nil { + t.Fatal(err) + } + max := 5016 + // Check all block sizes from 1 to 5000 + for i := 1; i < max-len(iv); i++ { + in := make([]byte, i) + + gCiphertext := gGCM.Seal(iv, iv, in, authData) + + dst := make([]byte, max-i) + // sPlaintext ... stupidgcm plaintext + sPlaintext, err := sGCM.Open(dst[:0], iv, gCiphertext[len(iv):], authData) + if err != nil { + t.Fatal(err) + } + + // Plaintext must be identical to Go GCM + if !bytes.Equal(in, sPlaintext) { + t.Fatalf("Compare failed, i=%d", i) + } + } +} + +// TestCorruption verifies that changes in the ciphertext result in a decryption +// error +func TestCorruptionChacha(t *testing.T) { + key := randBytes(32) + sGCM := newChacha20poly1305(key) + authData := randBytes(24) + iv := randBytes(sGCM.NonceSize()) + + in := make([]byte, 354) + sOut := sGCM.Seal(nil, iv, in, authData) + sOut2, sErr := sGCM.Open(nil, iv, sOut, authData) + if sErr != nil { + t.Fatal(sErr) + } + if !bytes.Equal(in, sOut2) { + t.Fatalf("Compare failed") + } + + // Corrupt first byte + sOut[0]++ + sOut2, sErr = sGCM.Open(nil, iv, sOut, authData) + if sErr == nil || sOut2 != nil { + t.Fatalf("Should have gotten error") + } + sOut[0]-- + + // Corrupt last byte + sOut[len(sOut)-1]++ + sOut2, sErr = sGCM.Open(nil, iv, sOut, authData) + if sErr == nil || sOut2 != nil { + t.Fatalf("Should have gotten error") + } + sOut[len(sOut)-1]-- + + // Append one byte + sOut = append(sOut, 0) + sOut2, sErr = sGCM.Open(nil, iv, sOut, authData) + if sErr == nil || sOut2 != nil { + t.Fatalf("Should have gotten error") + } +} From 676a4ceb87e8c8e0811b4312ce4b3b74f53b4368 Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Thu, 2 Sep 2021 09:57:06 +0200 Subject: [PATCH 14/59] stupidgcm: deduplicate tests 1/2 Pull the code shared between chacha and gcm into generic functions. --- internal/stupidgcm/common_test.go | 152 ++++++++++++++++++++++++ internal/stupidgcm/stupidchacha_test.go | 131 ++------------------ internal/stupidgcm/stupidgcm_test.go | 114 +----------------- 3 files changed, 167 insertions(+), 230 deletions(-) create mode 100644 internal/stupidgcm/common_test.go diff --git a/internal/stupidgcm/common_test.go b/internal/stupidgcm/common_test.go new file mode 100644 index 0000000..28f3308 --- /dev/null +++ b/internal/stupidgcm/common_test.go @@ -0,0 +1,152 @@ +package stupidgcm + +import ( + "bytes" + "crypto/cipher" + "encoding/hex" + "testing" +) + +// testEncryptDecrypt encrypts and decrypts using both stupidgcm and Go's built-in +// GCM implementation and verifies that the results are identical. +func testEncryptDecrypt(t *testing.T, c1 cipher.AEAD, c2 cipher.AEAD) { + if c1.NonceSize() != c2.NonceSize() { + t.Fatal("different NonceSize") + } + if c1.Overhead() != c2.Overhead() { + t.Fatal("different Overhead") + } + + authData := randBytes(24) + iv := randBytes(c1.NonceSize()) + + dst := make([]byte, 71) // 71 = arbitrary length + + // Check all block sizes from 1 to 5000 + for i := 1; i < 5000; i++ { + in := make([]byte, i) + + c1out := c1.Seal(dst, iv, in, authData) + c2out := c2.Seal(dst, iv, in, authData) + + // Ciphertext must be identical to Go GCM + if !bytes.Equal(c1out, c2out) { + t.Fatalf("Compare failed for encryption, size %d", i) + t.Log("c1out:") + t.Log("\n" + hex.Dump(c1out)) + t.Log("c2out:") + t.Log("\n" + hex.Dump(c2out)) + } + + c1out2, sErr := c1.Open(dst, iv, c1out[len(dst):], authData) + if sErr != nil { + t.Fatal(sErr) + } + c2out2, gErr := c2.Open(dst, iv, c2out[len(dst):], authData) + if gErr != nil { + t.Fatal(gErr) + } + + // Plaintext must be identical to Go GCM + if !bytes.Equal(c1out2, c2out2) { + t.Fatalf("Compare failed for decryption, size %d", i) + } + } +} + +// Seal re-uses the "dst" buffer it is large enough. +// Check that this works correctly by testing different "dst" capacities from +// 5000 to 16 and "in" lengths from 1 to 5000. +func testInplaceSeal(t *testing.T, c1 cipher.AEAD, c2 cipher.AEAD) { + authData := randBytes(24) + iv := randBytes(c1.NonceSize()) + + max := 5016 + // Check all block sizes from 1 to 5000 + for i := 1; i < max-16; i++ { + in := make([]byte, i) + dst := make([]byte, max-i) + dst = dst[:16] + + c1out := c1.Seal(dst, iv, in, authData) + dst2 := make([]byte, 16) + c2out := c2.Seal(dst2, iv, in, authData) + + // Ciphertext must be identical to Go GCM + if !bytes.Equal(c1out, c2out) { + t.Fatalf("Compare failed for encryption, size %d", i) + t.Log("sOut:") + t.Log("\n" + hex.Dump(c1out)) + t.Log("gOut:") + t.Log("\n" + hex.Dump(c2out)) + } + } +} + +// testInplaceOpen - Open re-uses the "dst" buffer it is large enough. +// Check that this works correctly by testing different "dst" capacities from +// 5000 to 16 and "in" lengths from 1 to 5000. +func testInplaceOpen(t *testing.T, c1 cipher.AEAD, c2 cipher.AEAD) { + authData := randBytes(24) + iv := randBytes(c1.NonceSize()) + + max := 5016 + // Check all block sizes from 1 to 5000 + for i := 1; i < max-c1.NonceSize(); i++ { + in := make([]byte, i) + + c2ciphertext := c2.Seal(iv, iv, in, authData) + + dst := make([]byte, max-i) + // sPlaintext ... stupidgcm plaintext + c1plaintext, err := c1.Open(dst[:0], iv, c2ciphertext[c1.NonceSize():], authData) + if err != nil { + t.Fatal(err) + } + + // Plaintext must be identical to Go GCM + if !bytes.Equal(in, c1plaintext) { + t.Fatalf("Compare failed, i=%d", i) + } + } +} + +// testCorruption verifies that changes in the ciphertext result in a decryption +// error +func testCorruption(t *testing.T, c cipher.AEAD) { + authData := randBytes(24) + iv := randBytes(c.NonceSize()) + + in := make([]byte, 354) + out := c.Seal(nil, iv, in, authData) + out2, sErr := c.Open(nil, iv, out, authData) + if sErr != nil { + t.Fatal(sErr) + } + if !bytes.Equal(in, out2) { + t.Fatalf("Compare failed") + } + + // Corrupt first byte + out[0]++ + out2, sErr = c.Open(nil, iv, out, authData) + if sErr == nil || out2 != nil { + t.Fatalf("Should have gotten error") + } + out[0]-- + + // Corrupt last byte + out[len(out)-1]++ + out2, sErr = c.Open(nil, iv, out, authData) + if sErr == nil || out2 != nil { + t.Fatalf("Should have gotten error") + } + out[len(out)-1]-- + + // Append one byte + out = append(out, 0) + out2, sErr = c.Open(nil, iv, out, authData) + if sErr == nil || out2 != nil { + t.Fatalf("Should have gotten error") + } +} diff --git a/internal/stupidgcm/stupidchacha_test.go b/internal/stupidgcm/stupidchacha_test.go index 010055f..c1086d1 100644 --- a/internal/stupidgcm/stupidchacha_test.go +++ b/internal/stupidgcm/stupidchacha_test.go @@ -6,8 +6,6 @@ package stupidgcm import ( - "bytes" - "encoding/hex" "testing" "golang.org/x/crypto/chacha20poly1305" @@ -17,46 +15,13 @@ import ( // GCM implementation and verifies that the results are identical. func TestEncryptDecryptChacha(t *testing.T) { key := randBytes(32) - sGCM := newChacha20poly1305(key) - authData := randBytes(24) - iv := randBytes(sGCM.NonceSize()) - dst := make([]byte, 71) // 71 = random length - - gGCM, err := chacha20poly1305.New(key) + c := newChacha20poly1305(key) + ref, err := chacha20poly1305.New(key) if err != nil { t.Fatal(err) } - // Check all block sizes from 1 to 5000 - for i := 1; i < 5000; i++ { - in := make([]byte, i) - - sOut := sGCM.Seal(dst, iv, in, authData) - gOut := gGCM.Seal(dst, iv, in, authData) - - // Ciphertext must be identical to Go GCM - if !bytes.Equal(sOut, gOut) { - t.Fatalf("Compare failed for encryption, size %d", i) - t.Log("sOut:") - t.Log("\n" + hex.Dump(sOut)) - t.Log("gOut:") - t.Log("\n" + hex.Dump(gOut)) - } - - sOut2, sErr := sGCM.Open(dst, iv, sOut[len(dst):], authData) - if sErr != nil { - t.Fatal(sErr) - } - gOut2, gErr := gGCM.Open(dst, iv, gOut[len(dst):], authData) - if gErr != nil { - t.Fatal(gErr) - } - - // Plaintext must be identical to Go GCM - if !bytes.Equal(sOut2, gOut2) { - t.Fatalf("Compare failed for decryption, size %d", i) - } - } + testEncryptDecrypt(t, c, ref) } // Seal re-uses the "dst" buffer it is large enough. @@ -64,34 +29,13 @@ func TestEncryptDecryptChacha(t *testing.T) { // 5000 to 16 and "in" lengths from 1 to 5000. func TestInplaceSealChacha(t *testing.T) { key := randBytes(32) - sGCM := newChacha20poly1305(key) - authData := randBytes(24) - iv := randBytes(sGCM.NonceSize()) - - gGCM, err := chacha20poly1305.New(key) + c := newChacha20poly1305(key) + ref, err := chacha20poly1305.New(key) if err != nil { t.Fatal(err) } - max := 5016 - // Check all block sizes from 1 to 5000 - for i := 1; i < max-len(iv); i++ { - in := make([]byte, i) - dst := make([]byte, max-i) - dst = dst[:len(iv)] - sOut := sGCM.Seal(dst, iv, in, authData) - dst2 := make([]byte, len(iv)) - gOut := gGCM.Seal(dst2, iv, in, authData) - - // Ciphertext must be identical to Go GCM - if !bytes.Equal(sOut, gOut) { - t.Fatalf("Compare failed for encryption, size %d", i) - t.Log("sOut:") - t.Log("\n" + hex.Dump(sOut)) - t.Log("gOut:") - t.Log("\n" + hex.Dump(gOut)) - } - } + testInplaceSeal(t, c, ref) } // Open re-uses the "dst" buffer it is large enough. @@ -99,73 +43,20 @@ func TestInplaceSealChacha(t *testing.T) { // 5000 to 16 and "in" lengths from 1 to 5000. func TestInplaceOpenChacha(t *testing.T) { key := randBytes(32) - sGCM := newChacha20poly1305(key) - authData := randBytes(24) - iv := randBytes(sGCM.NonceSize()) - - gGCM, err := chacha20poly1305.New(key) + c := newChacha20poly1305(key) + ref, err := chacha20poly1305.New(key) if err != nil { t.Fatal(err) } - max := 5016 - // Check all block sizes from 1 to 5000 - for i := 1; i < max-len(iv); i++ { - in := make([]byte, i) - gCiphertext := gGCM.Seal(iv, iv, in, authData) - - dst := make([]byte, max-i) - // sPlaintext ... stupidgcm plaintext - sPlaintext, err := sGCM.Open(dst[:0], iv, gCiphertext[len(iv):], authData) - if err != nil { - t.Fatal(err) - } - - // Plaintext must be identical to Go GCM - if !bytes.Equal(in, sPlaintext) { - t.Fatalf("Compare failed, i=%d", i) - } - } + testInplaceOpen(t, c, ref) } // TestCorruption verifies that changes in the ciphertext result in a decryption // error func TestCorruptionChacha(t *testing.T) { key := randBytes(32) - sGCM := newChacha20poly1305(key) - authData := randBytes(24) - iv := randBytes(sGCM.NonceSize()) + c := newChacha20poly1305(key) - in := make([]byte, 354) - sOut := sGCM.Seal(nil, iv, in, authData) - sOut2, sErr := sGCM.Open(nil, iv, sOut, authData) - if sErr != nil { - t.Fatal(sErr) - } - if !bytes.Equal(in, sOut2) { - t.Fatalf("Compare failed") - } - - // Corrupt first byte - sOut[0]++ - sOut2, sErr = sGCM.Open(nil, iv, sOut, authData) - if sErr == nil || sOut2 != nil { - t.Fatalf("Should have gotten error") - } - sOut[0]-- - - // Corrupt last byte - sOut[len(sOut)-1]++ - sOut2, sErr = sGCM.Open(nil, iv, sOut, authData) - if sErr == nil || sOut2 != nil { - t.Fatalf("Should have gotten error") - } - sOut[len(sOut)-1]-- - - // Append one byte - sOut = append(sOut, 0) - sOut2, sErr = sGCM.Open(nil, iv, sOut, authData) - if sErr == nil || sOut2 != nil { - t.Fatalf("Should have gotten error") - } + testCorruption(t, c) } diff --git a/internal/stupidgcm/stupidgcm_test.go b/internal/stupidgcm/stupidgcm_test.go index 18732df..968034e 100644 --- a/internal/stupidgcm/stupidgcm_test.go +++ b/internal/stupidgcm/stupidgcm_test.go @@ -6,11 +6,9 @@ package stupidgcm import ( - "bytes" "crypto/aes" "crypto/cipher" "crypto/rand" - "encoding/hex" "log" "testing" ) @@ -30,9 +28,6 @@ func randBytes(n int) []byte { func TestEncryptDecrypt(t *testing.T) { key := randBytes(32) sGCM := New(key, false) - authData := randBytes(24) - iv := randBytes(16) - dst := make([]byte, 71) // 71 = random length gAES, err := aes.NewCipher(key) if err != nil { @@ -43,36 +38,7 @@ func TestEncryptDecrypt(t *testing.T) { t.Fatal(err) } - // Check all block sizes from 1 to 5000 - for i := 1; i < 5000; i++ { - in := make([]byte, i) - - sOut := sGCM.Seal(dst, iv, in, authData) - gOut := gGCM.Seal(dst, iv, in, authData) - - // Ciphertext must be identical to Go GCM - if !bytes.Equal(sOut, gOut) { - t.Fatalf("Compare failed for encryption, size %d", i) - t.Log("sOut:") - t.Log("\n" + hex.Dump(sOut)) - t.Log("gOut:") - t.Log("\n" + hex.Dump(gOut)) - } - - sOut2, sErr := sGCM.Open(dst, iv, sOut[len(dst):], authData) - if sErr != nil { - t.Fatal(sErr) - } - gOut2, gErr := gGCM.Open(dst, iv, gOut[len(dst):], authData) - if gErr != nil { - t.Fatal(gErr) - } - - // Plaintext must be identical to Go GCM - if !bytes.Equal(sOut2, gOut2) { - t.Fatalf("Compare failed for decryption, size %d", i) - } - } + testEncryptDecrypt(t, sGCM, gGCM) } // Seal re-uses the "dst" buffer it is large enough. @@ -81,8 +47,6 @@ func TestEncryptDecrypt(t *testing.T) { func TestInplaceSeal(t *testing.T) { key := randBytes(32) sGCM := New(key, false) - authData := randBytes(24) - iv := randBytes(16) gAES, err := aes.NewCipher(key) if err != nil { @@ -92,26 +56,8 @@ func TestInplaceSeal(t *testing.T) { if err != nil { t.Fatal(err) } - max := 5016 - // Check all block sizes from 1 to 5000 - for i := 1; i < max-16; i++ { - in := make([]byte, i) - dst := make([]byte, max-i) - dst = dst[:16] - sOut := sGCM.Seal(dst, iv, in, authData) - dst2 := make([]byte, 16) - gOut := gGCM.Seal(dst2, iv, in, authData) - - // Ciphertext must be identical to Go GCM - if !bytes.Equal(sOut, gOut) { - t.Fatalf("Compare failed for encryption, size %d", i) - t.Log("sOut:") - t.Log("\n" + hex.Dump(sOut)) - t.Log("gOut:") - t.Log("\n" + hex.Dump(gOut)) - } - } + testInplaceSeal(t, sGCM, gGCM) } // Open re-uses the "dst" buffer it is large enough. @@ -120,8 +66,6 @@ func TestInplaceSeal(t *testing.T) { func TestInplaceOpen(t *testing.T) { key := randBytes(32) sGCM := New(key, false) - authData := randBytes(24) - iv := randBytes(16) gAES, err := aes.NewCipher(key) if err != nil { @@ -131,25 +75,8 @@ func TestInplaceOpen(t *testing.T) { if err != nil { t.Fatal(err) } - max := 5016 - // Check all block sizes from 1 to 5000 - for i := 1; i < max-16; i++ { - in := make([]byte, i) - gCiphertext := gGCM.Seal(iv, iv, in, authData) - - dst := make([]byte, max-i) - // sPlaintext ... stupidgcm plaintext - sPlaintext, err := sGCM.Open(dst[:0], iv, gCiphertext[16:], authData) - if err != nil { - t.Fatal(err) - } - - // Plaintext must be identical to Go GCM - if !bytes.Equal(in, sPlaintext) { - t.Fatalf("Compare failed, i=%d", i) - } - } + testInplaceOpen(t, sGCM, gGCM) } // TestCorruption verifies that changes in the ciphertext result in a decryption @@ -157,39 +84,6 @@ func TestInplaceOpen(t *testing.T) { func TestCorruption(t *testing.T) { key := randBytes(32) sGCM := New(key, false) - authData := randBytes(24) - iv := randBytes(16) - in := make([]byte, 354) - sOut := sGCM.Seal(nil, iv, in, authData) - sOut2, sErr := sGCM.Open(nil, iv, sOut, authData) - if sErr != nil { - t.Fatal(sErr) - } - if !bytes.Equal(in, sOut2) { - t.Fatalf("Compare failed") - } - - // Corrupt first byte - sOut[0]++ - sOut2, sErr = sGCM.Open(nil, iv, sOut, authData) - if sErr == nil || sOut2 != nil { - t.Fatalf("Should have gotten error") - } - sOut[0]-- - - // Corrupt last byte - sOut[len(sOut)-1]++ - sOut2, sErr = sGCM.Open(nil, iv, sOut, authData) - if sErr == nil || sOut2 != nil { - t.Fatalf("Should have gotten error") - } - sOut[len(sOut)-1]-- - - // Append one byte - sOut = append(sOut, 0) - sOut2, sErr = sGCM.Open(nil, iv, sOut, authData) - if sErr == nil || sOut2 != nil { - t.Fatalf("Should have gotten error") - } + testCorruption(t, sGCM) } From 961b8ca438361b01f2f232d8735c236ef94b4d03 Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Thu, 2 Sep 2021 10:04:38 +0200 Subject: [PATCH 15/59] stupidgcm: deduplicate tests 2/2 Deduplicate the cipher setup that was identical for all tests for each cipher. --- internal/stupidgcm/common_test.go | 20 ++++++++ internal/stupidgcm/stupidchacha_test.go | 43 +--------------- internal/stupidgcm/stupidgcm_test.go | 65 +------------------------ 3 files changed, 24 insertions(+), 104 deletions(-) diff --git a/internal/stupidgcm/common_test.go b/internal/stupidgcm/common_test.go index 28f3308..27ca7cc 100644 --- a/internal/stupidgcm/common_test.go +++ b/internal/stupidgcm/common_test.go @@ -3,10 +3,20 @@ package stupidgcm import ( "bytes" "crypto/cipher" + "crypto/rand" "encoding/hex" + "log" "testing" ) +func testCiphers(t *testing.T, c1 cipher.AEAD, c2 cipher.AEAD) { + t.Run("testEncryptDecrypt", func(t *testing.T) { testEncryptDecrypt(t, c1, c2) }) + t.Run("testInplaceSeal", func(t *testing.T) { testInplaceSeal(t, c1, c2) }) + t.Run("testInplaceOpen", func(t *testing.T) { testInplaceOpen(t, c1, c2) }) + t.Run("testCorruption_c1", func(t *testing.T) { testCorruption(t, c1) }) + t.Run("testCorruption_c2", func(t *testing.T) { testCorruption(t, c2) }) +} + // testEncryptDecrypt encrypts and decrypts using both stupidgcm and Go's built-in // GCM implementation and verifies that the results are identical. func testEncryptDecrypt(t *testing.T, c1 cipher.AEAD, c2 cipher.AEAD) { @@ -150,3 +160,13 @@ func testCorruption(t *testing.T, c cipher.AEAD) { t.Fatalf("Should have gotten error") } } + +// Get "n" random bytes from /dev/urandom or panic +func randBytes(n int) []byte { + b := make([]byte, n) + _, err := rand.Read(b) + if err != nil { + log.Panic("Failed to read random bytes: " + err.Error()) + } + return b +} diff --git a/internal/stupidgcm/stupidchacha_test.go b/internal/stupidgcm/stupidchacha_test.go index c1086d1..2690f85 100644 --- a/internal/stupidgcm/stupidchacha_test.go +++ b/internal/stupidgcm/stupidchacha_test.go @@ -11,9 +11,7 @@ import ( "golang.org/x/crypto/chacha20poly1305" ) -// TestEncryptDecrypt encrypts and decrypts using both stupidgcm and Go's built-in -// GCM implementation and verifies that the results are identical. -func TestEncryptDecryptChacha(t *testing.T) { +func TestStupidChacha20poly1305(t *testing.T) { key := randBytes(32) c := newChacha20poly1305(key) ref, err := chacha20poly1305.New(key) @@ -21,42 +19,5 @@ func TestEncryptDecryptChacha(t *testing.T) { t.Fatal(err) } - testEncryptDecrypt(t, c, ref) -} - -// Seal re-uses the "dst" buffer it is large enough. -// Check that this works correctly by testing different "dst" capacities from -// 5000 to 16 and "in" lengths from 1 to 5000. -func TestInplaceSealChacha(t *testing.T) { - key := randBytes(32) - c := newChacha20poly1305(key) - ref, err := chacha20poly1305.New(key) - if err != nil { - t.Fatal(err) - } - - testInplaceSeal(t, c, ref) -} - -// Open re-uses the "dst" buffer it is large enough. -// Check that this works correctly by testing different "dst" capacities from -// 5000 to 16 and "in" lengths from 1 to 5000. -func TestInplaceOpenChacha(t *testing.T) { - key := randBytes(32) - c := newChacha20poly1305(key) - ref, err := chacha20poly1305.New(key) - if err != nil { - t.Fatal(err) - } - - testInplaceOpen(t, c, ref) -} - -// TestCorruption verifies that changes in the ciphertext result in a decryption -// error -func TestCorruptionChacha(t *testing.T) { - key := randBytes(32) - c := newChacha20poly1305(key) - - testCorruption(t, c) + testCiphers(t, c, ref) } diff --git a/internal/stupidgcm/stupidgcm_test.go b/internal/stupidgcm/stupidgcm_test.go index 968034e..5323afa 100644 --- a/internal/stupidgcm/stupidgcm_test.go +++ b/internal/stupidgcm/stupidgcm_test.go @@ -8,24 +8,10 @@ package stupidgcm import ( "crypto/aes" "crypto/cipher" - "crypto/rand" - "log" "testing" ) -// Get "n" random bytes from /dev/urandom or panic -func randBytes(n int) []byte { - b := make([]byte, n) - _, err := rand.Read(b) - if err != nil { - log.Panic("Failed to read random bytes: " + err.Error()) - } - return b -} - -// TestEncryptDecrypt encrypts and decrypts using both stupidgcm and Go's built-in -// GCM implementation and verifies that the results are identical. -func TestEncryptDecrypt(t *testing.T) { +func TestStupidGCM(t *testing.T) { key := randBytes(32) sGCM := New(key, false) @@ -38,52 +24,5 @@ func TestEncryptDecrypt(t *testing.T) { t.Fatal(err) } - testEncryptDecrypt(t, sGCM, gGCM) -} - -// Seal re-uses the "dst" buffer it is large enough. -// Check that this works correctly by testing different "dst" capacities from -// 5000 to 16 and "in" lengths from 1 to 5000. -func TestInplaceSeal(t *testing.T) { - key := randBytes(32) - sGCM := New(key, false) - - gAES, err := aes.NewCipher(key) - if err != nil { - t.Fatal(err) - } - gGCM, err := cipher.NewGCMWithNonceSize(gAES, 16) - if err != nil { - t.Fatal(err) - } - - testInplaceSeal(t, sGCM, gGCM) -} - -// Open re-uses the "dst" buffer it is large enough. -// Check that this works correctly by testing different "dst" capacities from -// 5000 to 16 and "in" lengths from 1 to 5000. -func TestInplaceOpen(t *testing.T) { - key := randBytes(32) - sGCM := New(key, false) - - gAES, err := aes.NewCipher(key) - if err != nil { - t.Fatal(err) - } - gGCM, err := cipher.NewGCMWithNonceSize(gAES, 16) - if err != nil { - t.Fatal(err) - } - - testInplaceOpen(t, sGCM, gGCM) -} - -// TestCorruption verifies that changes in the ciphertext result in a decryption -// error -func TestCorruption(t *testing.T) { - key := randBytes(32) - sGCM := New(key, false) - - testCorruption(t, sGCM) + testCiphers(t, sGCM, gGCM) } From 3ba74ac4fcb8ad5c7bfa73d63059805318b8682e Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Thu, 2 Sep 2021 10:17:01 +0200 Subject: [PATCH 16/59] stupidgcm: add testWipe test After looking at the cover profile, this was the only untested code except panic cases. --- internal/stupidgcm/common_test.go | 34 ++++++++++++++++++++++++------ internal/stupidgcm/stupidchacha.go | 2 +- internal/stupidgcm/stupidgcm.go | 2 +- 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/internal/stupidgcm/common_test.go b/internal/stupidgcm/common_test.go index 27ca7cc..ded6273 100644 --- a/internal/stupidgcm/common_test.go +++ b/internal/stupidgcm/common_test.go @@ -9,12 +9,13 @@ import ( "testing" ) -func testCiphers(t *testing.T, c1 cipher.AEAD, c2 cipher.AEAD) { - t.Run("testEncryptDecrypt", func(t *testing.T) { testEncryptDecrypt(t, c1, c2) }) - t.Run("testInplaceSeal", func(t *testing.T) { testInplaceSeal(t, c1, c2) }) - t.Run("testInplaceOpen", func(t *testing.T) { testInplaceOpen(t, c1, c2) }) - t.Run("testCorruption_c1", func(t *testing.T) { testCorruption(t, c1) }) - t.Run("testCorruption_c2", func(t *testing.T) { testCorruption(t, c2) }) +func testCiphers(t *testing.T, our cipher.AEAD, ref cipher.AEAD) { + t.Run("testEncryptDecrypt", func(t *testing.T) { testEncryptDecrypt(t, our, ref) }) + t.Run("testInplaceSeal", func(t *testing.T) { testInplaceSeal(t, our, ref) }) + t.Run("testInplaceOpen", func(t *testing.T) { testInplaceOpen(t, our, ref) }) + t.Run("testCorruption_c1", func(t *testing.T) { testCorruption(t, our) }) + t.Run("testCorruption_c2", func(t *testing.T) { testCorruption(t, ref) }) + t.Run("testWipe", func(t *testing.T) { testWipe(t, our) }) } // testEncryptDecrypt encrypts and decrypts using both stupidgcm and Go's built-in @@ -161,6 +162,27 @@ func testCorruption(t *testing.T, c cipher.AEAD) { } } +type Wiper interface { + Wipe() +} + +func testWipe(t *testing.T, c cipher.AEAD) { + var key []byte + switch c2 := c.(type) { + case *StupidGCM: + c2.Wipe() + key = c2.key + case *stupidChacha20poly1305: + c2.Wipe() + key = c2.key + default: + t.Fatalf("BUG: unhandled type %t", c2) + } + if key != nil { + t.Fatal("key is not nil") + } +} + // Get "n" random bytes from /dev/urandom or panic func randBytes(n int) []byte { b := make([]byte, n) diff --git a/internal/stupidgcm/stupidchacha.go b/internal/stupidgcm/stupidchacha.go index e2f6407..2b31e0f 100644 --- a/internal/stupidgcm/stupidchacha.go +++ b/internal/stupidgcm/stupidchacha.go @@ -213,7 +213,7 @@ func (g *stupidChacha20poly1305) Open(dst, iv, in, authData []byte) ([]byte, err // and setting the reference to nil. // // This is not bulletproof due to possible GC copies, but -// still raises to bar for extracting the key. +// still raises the bar for extracting the key. func (g *stupidChacha20poly1305) Wipe() { for i := range g.key { g.key[i] = 0 diff --git a/internal/stupidgcm/stupidgcm.go b/internal/stupidgcm/stupidgcm.go index 01db41b..3499c85 100644 --- a/internal/stupidgcm/stupidgcm.go +++ b/internal/stupidgcm/stupidgcm.go @@ -240,7 +240,7 @@ func (g *StupidGCM) Open(dst, iv, in, authData []byte) ([]byte, error) { // and setting the reference to nil. // // This is not bulletproof due to possible GC copies, but -// still raises to bar for extracting the key. +// still raises the bar for extracting the key. func (g *StupidGCM) Wipe() { for i := range g.key { g.key[i] = 0 From 5df7ee815dcd91d33e6167c20cebcbd5c51c2c7a Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Thu, 2 Sep 2021 10:37:44 +0200 Subject: [PATCH 17/59] stupidgcm: stupidChacha20poly1305: use byte array for key Follow what golang.org/x/crypto/chacha20poly1305 does for easier integration in the next commit. --- internal/stupidgcm/common_test.go | 17 +++++++++++------ internal/stupidgcm/stupidchacha.go | 23 +++++++++++++++-------- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/internal/stupidgcm/common_test.go b/internal/stupidgcm/common_test.go index ded6273..cf555b0 100644 --- a/internal/stupidgcm/common_test.go +++ b/internal/stupidgcm/common_test.go @@ -167,20 +167,25 @@ type Wiper interface { } func testWipe(t *testing.T, c cipher.AEAD) { - var key []byte switch c2 := c.(type) { case *StupidGCM: c2.Wipe() - key = c2.key + if c2.key != nil { + t.Fatal("key is not nil") + } case *stupidChacha20poly1305: c2.Wipe() - key = c2.key + if !c2.wiped { + t.Error("c2.wiped is not set") + } + for _, v := range c2.key { + if v != 0 { + t.Fatal("c2.key is not zeroed") + } + } default: t.Fatalf("BUG: unhandled type %t", c2) } - if key != nil { - t.Fatal("key is not nil") - } } // Get "n" random bytes from /dev/urandom or panic diff --git a/internal/stupidgcm/stupidchacha.go b/internal/stupidgcm/stupidchacha.go index 2b31e0f..be8accb 100644 --- a/internal/stupidgcm/stupidchacha.go +++ b/internal/stupidgcm/stupidchacha.go @@ -16,19 +16,20 @@ import ( ) type stupidChacha20poly1305 struct { - key []byte + key [chacha20poly1305.KeySize]byte + wiped bool } // Verify that we satisfy the cipher.AEAD interface var _ cipher.AEAD = &stupidChacha20poly1305{} -func newChacha20poly1305(keyIn []byte) cipher.AEAD { - if len(keyIn) != chacha20poly1305.KeySize { - log.Panicf("Only %d-byte keys are supported, you passed %d bytes", chacha20poly1305.KeySize, len(keyIn)) +func newChacha20poly1305(key []byte) cipher.AEAD { + if len(key) != chacha20poly1305.KeySize { + log.Panicf("Only %d-byte keys are supported, you passed %d bytes", chacha20poly1305.KeySize, len(key)) } - // Create a private copy of the key - key := append([]byte{}, keyIn...) - return &stupidChacha20poly1305{key: key} + ret := new(stupidChacha20poly1305) + copy(ret.key[:], key) + return ret } // NonceSize returns the required size of the nonce / IV. @@ -43,6 +44,9 @@ func (g *stupidChacha20poly1305) Overhead() int { // Seal encrypts "in" using "iv" and "authData" and append the result to "dst" func (g *stupidChacha20poly1305) Seal(dst, iv, in, authData []byte) []byte { + if g.wiped { + panic("BUG: tried to use wiped stupidChacha20poly1305") + } if len(iv) != g.NonceSize() { log.Panicf("Only %d-byte IVs are supported, you passed %d bytes", g.NonceSize(), len(iv)) } @@ -125,6 +129,9 @@ func (g *stupidChacha20poly1305) Seal(dst, iv, in, authData []byte) []byte { // Open decrypts "in" using "iv" and "authData" and append the result to "dst" func (g *stupidChacha20poly1305) Open(dst, iv, in, authData []byte) ([]byte, error) { + if g.wiped { + panic("BUG: tried to use wiped stupidChacha20poly1305") + } if len(iv) != g.NonceSize() { log.Panicf("Only %d-byte IVs are supported", g.NonceSize()) } @@ -215,8 +222,8 @@ func (g *stupidChacha20poly1305) Open(dst, iv, in, authData []byte) ([]byte, err // This is not bulletproof due to possible GC copies, but // still raises the bar for extracting the key. func (g *stupidChacha20poly1305) Wipe() { + g.wiped = true for i := range g.key { g.key[i] = 0 } - g.key = nil } From 591a56e7ae06a5766747eb91cb1e7fade7f3a704 Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Thu, 2 Sep 2021 10:50:45 +0200 Subject: [PATCH 18/59] stupidgcm: stupidChacha20poly1305: normalize panic messages --- internal/stupidgcm/stupidchacha.go | 7 +++---- internal/stupidgcm/stupidchacha_test.go | 3 --- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/internal/stupidgcm/stupidchacha.go b/internal/stupidgcm/stupidchacha.go index be8accb..a6fe318 100644 --- a/internal/stupidgcm/stupidchacha.go +++ b/internal/stupidgcm/stupidchacha.go @@ -45,7 +45,7 @@ func (g *stupidChacha20poly1305) Overhead() int { // Seal encrypts "in" using "iv" and "authData" and append the result to "dst" func (g *stupidChacha20poly1305) Seal(dst, iv, in, authData []byte) []byte { if g.wiped { - panic("BUG: tried to use wiped stupidChacha20poly1305") + panic("BUG: tried to use wiped key") } if len(iv) != g.NonceSize() { log.Panicf("Only %d-byte IVs are supported, you passed %d bytes", g.NonceSize(), len(iv)) @@ -130,7 +130,7 @@ func (g *stupidChacha20poly1305) Seal(dst, iv, in, authData []byte) []byte { // Open decrypts "in" using "iv" and "authData" and append the result to "dst" func (g *stupidChacha20poly1305) Open(dst, iv, in, authData []byte) ([]byte, error) { if g.wiped { - panic("BUG: tried to use wiped stupidChacha20poly1305") + panic("BUG: tried to use wiped key") } if len(iv) != g.NonceSize() { log.Panicf("Only %d-byte IVs are supported", g.NonceSize()) @@ -216,8 +216,7 @@ func (g *stupidChacha20poly1305) Open(dst, iv, in, authData []byte) ([]byte, err return append(dst, buf...), nil } -// Wipe tries to wipe the AES key from memory by overwriting it with zeros -// and setting the reference to nil. +// Wipe tries to wipe the key from memory by overwriting it with zeros. // // This is not bulletproof due to possible GC copies, but // still raises the bar for extracting the key. diff --git a/internal/stupidgcm/stupidchacha_test.go b/internal/stupidgcm/stupidchacha_test.go index 2690f85..513b68f 100644 --- a/internal/stupidgcm/stupidchacha_test.go +++ b/internal/stupidgcm/stupidchacha_test.go @@ -1,8 +1,5 @@ // +build !without_openssl -// We compare against Go's built-in GCM implementation. Since stupidgcm only -// supports 128-bit IVs and Go only supports that from 1.5 onward, we cannot -// run these tests on older Go versions. package stupidgcm import ( From 4017e4b22c2fe1c31dc67163affc49c28fd8c391 Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Thu, 2 Sep 2021 10:51:51 +0200 Subject: [PATCH 19/59] stupidgcm: add stupidXchacha20poly1305 Implementation copied from https://github.com/golang/crypto/blob/32db794688a5a24a23a43f2a984cecd5b3d8da58/chacha20poly1305/xchacha20poly1305.go --- internal/stupidgcm/common_test.go | 12 ++- internal/stupidgcm/stupidxchacha.go | 111 +++++++++++++++++++++++ internal/stupidgcm/stupidxchacha_test.go | 20 ++++ 3 files changed, 142 insertions(+), 1 deletion(-) create mode 100644 internal/stupidgcm/stupidxchacha.go create mode 100644 internal/stupidgcm/stupidxchacha_test.go diff --git a/internal/stupidgcm/common_test.go b/internal/stupidgcm/common_test.go index cf555b0..8123ce2 100644 --- a/internal/stupidgcm/common_test.go +++ b/internal/stupidgcm/common_test.go @@ -183,8 +183,18 @@ func testWipe(t *testing.T, c cipher.AEAD) { t.Fatal("c2.key is not zeroed") } } + case *stupidXchacha20poly1305: + c2.Wipe() + if !c2.wiped { + t.Error("c2.wiped is not set") + } + for _, v := range c2.key { + if v != 0 { + t.Fatal("c2.key is not zeroed") + } + } default: - t.Fatalf("BUG: unhandled type %t", c2) + t.Fatalf("BUG: unhandled type %T", c2) } } diff --git a/internal/stupidgcm/stupidxchacha.go b/internal/stupidgcm/stupidxchacha.go new file mode 100644 index 0000000..9f2ac2f --- /dev/null +++ b/internal/stupidgcm/stupidxchacha.go @@ -0,0 +1,111 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +// +// Copied from +// https://github.com/golang/crypto/blob/32db794688a5a24a23a43f2a984cecd5b3d8da58/chacha20poly1305/xchacha20poly1305.go +// and adapted for stupidgcm by @rfjakob. + +package stupidgcm + +import ( + "crypto/cipher" + "errors" + + "golang.org/x/crypto/chacha20" + "golang.org/x/crypto/chacha20poly1305" +) + +type stupidXchacha20poly1305 struct { + key [chacha20poly1305.KeySize]byte + wiped bool +} + +// NewXchacha20poly1305 returns a XChaCha20-Poly1305 AEAD that uses the given 256-bit key. +// +// XChaCha20-Poly1305 is a ChaCha20-Poly1305 variant that takes a longer nonce, +// suitable to be generated randomly without risk of collisions. It should be +// preferred when nonce uniqueness cannot be trivially ensured, or whenever +// nonces are randomly generated. +func NewXchacha20poly1305(key []byte) cipher.AEAD { + if len(key) != chacha20poly1305.KeySize { + panic("bad key length") + } + ret := new(stupidXchacha20poly1305) + copy(ret.key[:], key) + return ret +} + +func (*stupidXchacha20poly1305) NonceSize() int { + return chacha20poly1305.NonceSizeX +} + +func (*stupidXchacha20poly1305) Overhead() int { + return 16 +} + +func (x *stupidXchacha20poly1305) Seal(dst, nonce, plaintext, additionalData []byte) []byte { + if x.wiped { + panic("BUG: tried to use wiped key") + } + if len(nonce) != chacha20poly1305.NonceSizeX { + panic("bad nonce length passed to Seal") + } + + // XChaCha20-Poly1305 technically supports a 64-bit counter, so there is no + // size limit. However, since we reuse the ChaCha20-Poly1305 implementation, + // the second half of the counter is not available. This is unlikely to be + // an issue because the cipher.AEAD API requires the entire message to be in + // memory, and the counter overflows at 256 GB. + if uint64(len(plaintext)) > (1<<38)-64 { + panic("plaintext too large") + } + + c := new(stupidChacha20poly1305) + hKey, _ := chacha20.HChaCha20(x.key[:], nonce[0:16]) + copy(c.key[:], hKey) + defer c.Wipe() + + // The first 4 bytes of the final nonce are unused counter space. + cNonce := make([]byte, chacha20poly1305.NonceSize) + copy(cNonce[4:12], nonce[16:24]) + + return c.Seal(dst, cNonce[:], plaintext, additionalData) +} + +func (x *stupidXchacha20poly1305) Open(dst, nonce, ciphertext, additionalData []byte) ([]byte, error) { + if x.wiped { + panic("BUG: tried to use wiped key") + } + if len(nonce) != chacha20poly1305.NonceSizeX { + panic("bad nonce length passed to Open") + } + if len(ciphertext) < 16 { + return nil, errors.New("message too short") + } + if uint64(len(ciphertext)) > (1<<38)-48 { + panic("ciphertext too large") + } + + c := new(stupidChacha20poly1305) + hKey, _ := chacha20.HChaCha20(x.key[:], nonce[0:16]) + copy(c.key[:], hKey) + defer c.Wipe() + + // The first 4 bytes of the final nonce are unused counter space. + cNonce := make([]byte, chacha20poly1305.NonceSize) + copy(cNonce[4:12], nonce[16:24]) + + return c.Open(dst, cNonce[:], ciphertext, additionalData) +} + +// Wipe tries to wipe the key from memory by overwriting it with zeros. +// +// This is not bulletproof due to possible GC copies, but +// still raises the bar for extracting the key. +func (g *stupidXchacha20poly1305) Wipe() { + g.wiped = true + for i := range g.key { + g.key[i] = 0 + } +} diff --git a/internal/stupidgcm/stupidxchacha_test.go b/internal/stupidgcm/stupidxchacha_test.go new file mode 100644 index 0000000..fdea8b5 --- /dev/null +++ b/internal/stupidgcm/stupidxchacha_test.go @@ -0,0 +1,20 @@ +// +build !without_openssl + +package stupidgcm + +import ( + "testing" + + "golang.org/x/crypto/chacha20poly1305" +) + +func TestStupidXchacha20poly1305(t *testing.T) { + key := randBytes(32) + c := NewXchacha20poly1305(key) + ref, err := chacha20poly1305.NewX(key) + if err != nil { + t.Fatal(err) + } + + testCiphers(t, c, ref) +} From 9e1dd73e55e095b8e8b4264897c4d21fcdcdb2ae Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Thu, 2 Sep 2021 11:33:06 +0200 Subject: [PATCH 20/59] -speed: add XChaCha20-Poly1305-OpenSSL $ ./gocryptfs -speed gocryptfs v2.1-56-gdb1466f-dirty.stupidchacha; go-fuse v2.1.1-0.20210825171523-3ab5d95a30ae; 2021-09-02 go1.17 linux/amd64 AES-GCM-256-OpenSSL 529.53 MB/s AES-GCM-256-Go 833.85 MB/s (selected in auto mode) AES-SIV-512-Go 155.27 MB/s XChaCha20-Poly1305-Go 715.33 MB/s (use via -xchacha flag) XChaCha20-Poly1305-OpenSSL 468.94 MB/s https://github.com/rfjakob/gocryptfs/issues/452 --- internal/cryptocore/cryptocore.go | 3 +++ internal/speed/speed.go | 28 ++++++++++++++++++++++++---- internal/speed/speed_test.go | 8 ++++++++ 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/internal/cryptocore/cryptocore.go b/internal/cryptocore/cryptocore.go index 8fb7936..1b692ff 100644 --- a/internal/cryptocore/cryptocore.go +++ b/internal/cryptocore/cryptocore.go @@ -48,6 +48,9 @@ var BackendAESSIV AEADTypeEnum = AEADTypeEnum{"AES-SIV-512-Go", siv_aead.NonceSi // "XChaCha20-Poly1305-Go" in gocryptfs -speed. var BackendXChaCha20Poly1305 AEADTypeEnum = AEADTypeEnum{"XChaCha20-Poly1305-Go", chacha20poly1305.NonceSizeX} +// BackendXChaCha20Poly1305OpenSSL specifies XChaCha20-Poly1305-OpenSSL. +var BackendXChaCha20Poly1305OpenSSL AEADTypeEnum = AEADTypeEnum{"XChaCha20-Poly1305-OpenSSL", chacha20poly1305.NonceSizeX} + // CryptoCore is the low level crypto implementation. type CryptoCore struct { // EME is used for filename encryption. diff --git a/internal/speed/speed.go b/internal/speed/speed.go index 46e138d..d53e402 100644 --- a/internal/speed/speed.go +++ b/internal/speed/speed.go @@ -35,10 +35,11 @@ func Run() { {name: cryptocore.BackendOpenSSL.Name, f: bStupidGCM, preferred: stupidgcm.PreferOpenSSL()}, {name: cryptocore.BackendGoGCM.Name, f: bGoGCM, preferred: !stupidgcm.PreferOpenSSL()}, {name: cryptocore.BackendAESSIV.Name, f: bAESSIV, preferred: false}, - {name: cryptocore.BackendXChaCha20Poly1305.Name, f: bChacha20poly1305, preferred: false}, + {name: cryptocore.BackendXChaCha20Poly1305.Name, f: bXchacha20poly1305, preferred: false}, + {name: cryptocore.BackendXChaCha20Poly1305OpenSSL.Name, f: bStupidXchacha, preferred: false}, } for _, b := range bTable { - fmt.Printf("%-20s\t", b.name) + fmt.Printf("%-26s\t", b.name) mbs := mbPerSec(testing.Benchmark(b.f)) if mbs > 0 { fmt.Printf("%7.2f MB/s", mbs) @@ -132,8 +133,8 @@ func bAESSIV(b *testing.B) { } } -// bChacha20poly1305 benchmarks XChaCha20 from golang.org/x/crypto/chacha20poly1305 -func bChacha20poly1305(b *testing.B) { +// bXchacha20poly1305 benchmarks XChaCha20 from golang.org/x/crypto/chacha20poly1305 +func bXchacha20poly1305(b *testing.B) { key := randBytes(32) authData := randBytes(adLen) iv := randBytes(chacha20poly1305.NonceSizeX) @@ -147,3 +148,22 @@ func bChacha20poly1305(b *testing.B) { c.Seal(iv, iv, in, authData) } } + +// bStupidXchacha benchmarks OpenSSL XChaCha20 +func bStupidXchacha(b *testing.B) { + if stupidgcm.BuiltWithoutOpenssl { + b.Skip("openssl has been disabled at compile-time") + } + key := randBytes(32) + authData := randBytes(adLen) + iv := randBytes(chacha20poly1305.NonceSizeX) + in := make([]byte, blockSize) + b.SetBytes(int64(len(in))) + c := stupidgcm.NewXchacha20poly1305(key) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + // Encrypt and append to nonce + c.Seal(iv, iv, in, authData) + } +} diff --git a/internal/speed/speed_test.go b/internal/speed/speed_test.go index f3ec66c..a6f3f30 100644 --- a/internal/speed/speed_test.go +++ b/internal/speed/speed_test.go @@ -27,3 +27,11 @@ func BenchmarkGoGCM(b *testing.B) { func BenchmarkAESSIV(b *testing.B) { bAESSIV(b) } + +func BenchmarkXchacha(b *testing.B) { + bXchacha20poly1305(b) +} + +func BenchmarkStupidXchacha(b *testing.B) { + bStupidXchacha(b) +} From a3f5a8492a8bc62d8e2a639bd449b425efa66ce2 Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Fri, 3 Sep 2021 16:44:13 +0200 Subject: [PATCH 21/59] stupidgcm: batch C calls in chacha20poly1305_seal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Go has a high overhead for each C call, so batch all openssl operations in the new C function chacha20poly1305_seal. Benchmark results: internal/speed$ go test -bench BenchmarkStupidXchacha -count 10 > old.txt internal/speed$ go test -bench BenchmarkStupidXchacha -count 10 > new.txt internal/speed$ benchstat old.txt new.txt name old time/op new time/op delta StupidXchacha-4 8.79µs ± 1% 7.25µs ± 1% -17.54% (p=0.000 n=10+10) name old speed new speed delta StupidXchacha-4 466MB/s ± 1% 565MB/s ± 1% +21.27% (p=0.000 n=10+10) --- internal/stupidgcm/.gitignore | 1 + internal/stupidgcm/Makefile | 7 +++ internal/stupidgcm/chacha.c | 98 ++++++++++++++++++++++++++++++ internal/stupidgcm/stupidchacha.go | 77 +++++++---------------- 4 files changed, 127 insertions(+), 56 deletions(-) create mode 100644 internal/stupidgcm/.gitignore create mode 100644 internal/stupidgcm/Makefile create mode 100644 internal/stupidgcm/chacha.c diff --git a/internal/stupidgcm/.gitignore b/internal/stupidgcm/.gitignore new file mode 100644 index 0000000..5761abc --- /dev/null +++ b/internal/stupidgcm/.gitignore @@ -0,0 +1 @@ +*.o diff --git a/internal/stupidgcm/Makefile b/internal/stupidgcm/Makefile new file mode 100644 index 0000000..19f9914 --- /dev/null +++ b/internal/stupidgcm/Makefile @@ -0,0 +1,7 @@ +.PHONY: gcc +gcc: + gcc -Wall -Wextra -Wformat-security -Wconversion -lcrypto -c *.c + +.PHONY: format +format: + clang-format --style=WebKit -i *.c diff --git a/internal/stupidgcm/chacha.c b/internal/stupidgcm/chacha.c new file mode 100644 index 0000000..c85cf78 --- /dev/null +++ b/internal/stupidgcm/chacha.c @@ -0,0 +1,98 @@ +#include +#include +//#cgo pkg-config: libcrypto + +extern void panic1(void); + +static void panic(const char* const msg) +{ + fprintf(stderr, "panic in C code: %s\n", msg); + __builtin_trap(); +} + +// https://wiki.openssl.org/index.php/EVP_Authenticated_Encryption_and_Decryption#Authenticated_Encryption_using_GCM_mode +int chacha20poly1305_seal( + const unsigned char* const plaintext, + const int plaintextLen, + const unsigned char* const authData, + const int authDataLen, + const unsigned char* const key, + const int keyLen, + const unsigned char* const iv, + const int ivLen, + unsigned char* const ciphertext, + const int ciphertextBufLen) +{ + // Create scratch space "context" + EVP_CIPHER_CTX* ctx = EVP_CIPHER_CTX_new(); + if (!ctx) { + panic("EVP_CIPHER_CTX_new failed"); + } + + // Set cipher + if (EVP_EncryptInit_ex(ctx, EVP_chacha20_poly1305(), NULL, NULL, NULL) != 1) { + panic("EVP_EncryptInit_ex set cipher failed"); + } + + // Check keyLen by trying to set it (fails if keyLen != 32) + if (EVP_CIPHER_CTX_set_key_length(ctx, keyLen) != 1) { + panic("keyLen mismatch"); + } + + // Set IV length so we do not depend on the default + if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_IVLEN, ivLen, NULL) != 1) { + panic("EVP_CTRL_AEAD_SET_IVLEN failed"); + } + + // Set key and IV + if (EVP_EncryptInit_ex(ctx, NULL, NULL, key, iv) != 1) { + panic("EVP_EncryptInit_ex set key & iv failed"); + } + + // Provide authentication data + int outLen = 0; + if (EVP_EncryptUpdate(ctx, NULL, &outLen, authData, authDataLen) != 1) { + panic("EVP_EncryptUpdate authData failed"); + } + if (outLen != authDataLen) { + panic("EVP_EncryptUpdate authData: unexpected length"); + } + + // Encrypt "plaintext" into "ciphertext" + if (plaintextLen > ciphertextBufLen) { + panic("plaintext overflows output buffer"); + } + if (EVP_EncryptUpdate(ctx, ciphertext, &outLen, plaintext, plaintextLen) != 1) { + panic("EVP_EncryptUpdate ciphertext failed"); + } + if (outLen != plaintextLen) { + panic("EVP_EncryptUpdate ciphertext: unexpected length"); + } + int ciphertextLen = outLen; + + // Finalise encryption + // Normally ciphertext bytes may be written at this stage, but this does not occur in GCM mode + if (EVP_EncryptFinal_ex(ctx, ciphertext + plaintextLen, &outLen) != 1) { + panic("EVP_EncryptFinal_ex failed"); + } + if (outLen != 0) { + panic("EVP_EncryptFinal_ex: unexpected length"); + } + + // We only support 16-byte tags + const int tagLen = 16; + + // Get MAC tag and append it to the ciphertext + if (ciphertextLen + tagLen > ciphertextBufLen) { + panic("tag overflows output buffer"); + } + if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_GET_TAG, tagLen, ciphertext + plaintextLen) != 1) { + panic("EVP_CTRL_AEAD_GET_TAG failed"); + } + ciphertextLen += tagLen; + + // Free scratch space + EVP_CIPHER_CTX_free(ctx); + + return ciphertextLen; +} diff --git a/internal/stupidgcm/stupidchacha.go b/internal/stupidgcm/stupidchacha.go index a6fe318..1f16a5e 100644 --- a/internal/stupidgcm/stupidchacha.go +++ b/internal/stupidgcm/stupidchacha.go @@ -2,10 +2,6 @@ package stupidgcm -// #include -// #cgo pkg-config: libcrypto -import "C" - import ( "crypto/cipher" "fmt" @@ -15,6 +11,17 @@ import ( "golang.org/x/crypto/chacha20poly1305" ) +/* +#include +#cgo pkg-config: libcrypto +int chacha20poly1305_seal(const unsigned char * const plaintext, const int plaintextLen, + const unsigned char * const authData, const int authDataLen, + const unsigned char * const key, const int keyLen, + const unsigned char * const iv, const int ivLen, + unsigned char * const ciphertext, const int ciphertextBufLen); +*/ +import "C" + type stupidChacha20poly1305 struct { key [chacha20poly1305.KeySize]byte wiped bool @@ -68,58 +75,16 @@ func (g *stupidChacha20poly1305) Seal(dst, iv, in, authData []byte) []byte { buf = make([]byte, outLen) } - // https://wiki.openssl.org/index.php/EVP_Authenticated_Encryption_and_Decryption#Authenticated_Encryption_using_GCM_mode - - // Create scratch space "context" - ctx := C.EVP_CIPHER_CTX_new() - if ctx == nil { - log.Panic("EVP_CIPHER_CTX_new failed") - } - - // Set cipher - if C.EVP_EncryptInit_ex(ctx, C.EVP_chacha20_poly1305(), nil, nil, nil) != 1 { - log.Panic("EVP_EncryptInit_ex I failed") - } - - // Set key and IV - if C.EVP_EncryptInit_ex(ctx, nil, nil, (*C.uchar)(&g.key[0]), (*C.uchar)(&iv[0])) != 1 { - log.Panic("EVP_EncryptInit_ex II failed") - } - - // Provide authentication data - var resultLen C.int - if C.EVP_EncryptUpdate(ctx, nil, &resultLen, (*C.uchar)(&authData[0]), C.int(len(authData))) != 1 { - log.Panic("EVP_EncryptUpdate authData failed") - } - if int(resultLen) != len(authData) { - log.Panicf("Unexpected length %d", resultLen) - } - - // Encrypt "in" into "buf" - if C.EVP_EncryptUpdate(ctx, (*C.uchar)(&buf[0]), &resultLen, (*C.uchar)(&in[0]), C.int(len(in))) != 1 { - log.Panic("EVP_EncryptUpdate failed") - } - if int(resultLen) != len(in) { - log.Panicf("Unexpected length %d", resultLen) - } - - // Finalise encryption - // Because GCM is a stream encryption, this will not write out any data. - dummy := make([]byte, 16) - if C.EVP_EncryptFinal_ex(ctx, (*C.uchar)(&dummy[0]), &resultLen) != 1 { - log.Panic("EVP_EncryptFinal_ex failed") - } - if resultLen != 0 { - log.Panicf("Unexpected length %d", resultLen) - } - - // Get MAC tag and append it to the ciphertext in "buf" - if C.EVP_CIPHER_CTX_ctrl(ctx, C.EVP_CTRL_AEAD_GET_TAG, tagLen, (unsafe.Pointer)(&buf[len(in)])) != 1 { - log.Panic("EVP_CIPHER_CTX_ctrl EVP_CTRL_AEAD_GET_TAG failed") - } - - // Free scratch space - C.EVP_CIPHER_CTX_free(ctx) + C.chacha20poly1305_seal((*C.uchar)(&in[0]), + C.int(len(in)), + (*C.uchar)(&authData[0]), + C.int(len(authData)), + (*C.uchar)(&g.key[0]), + C.int(len(g.key)), + (*C.uchar)(&iv[0]), + C.int(len(iv)), + (*C.uchar)(&buf[0]), + C.int(len(buf))) if inplace { return dst[:len(dst)+outLen] From 69d626b26f5a3f55c752b59af44710b992e2ab76 Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Fri, 3 Sep 2021 17:11:57 +0200 Subject: [PATCH 22/59] stupidgcm: replace chacha20poly1305_seal with generic aead_seal --- internal/stupidgcm/Makefile | 2 +- internal/stupidgcm/chacha.c | 20 ++++++++++++++++---- internal/stupidgcm/chacha.h | 17 +++++++++++++++++ internal/stupidgcm/stupidchacha.go | 9 +++------ 4 files changed, 37 insertions(+), 11 deletions(-) create mode 100644 internal/stupidgcm/chacha.h diff --git a/internal/stupidgcm/Makefile b/internal/stupidgcm/Makefile index 19f9914..22e3b2e 100644 --- a/internal/stupidgcm/Makefile +++ b/internal/stupidgcm/Makefile @@ -4,4 +4,4 @@ gcc: .PHONY: format format: - clang-format --style=WebKit -i *.c + clang-format --style=WebKit -i *.c *.h diff --git a/internal/stupidgcm/chacha.c b/internal/stupidgcm/chacha.c index c85cf78..e188bfc 100644 --- a/internal/stupidgcm/chacha.c +++ b/internal/stupidgcm/chacha.c @@ -1,9 +1,8 @@ +#include "chacha.h" #include #include //#cgo pkg-config: libcrypto -extern void panic1(void); - static void panic(const char* const msg) { fprintf(stderr, "panic in C code: %s\n", msg); @@ -11,7 +10,8 @@ static void panic(const char* const msg) } // https://wiki.openssl.org/index.php/EVP_Authenticated_Encryption_and_Decryption#Authenticated_Encryption_using_GCM_mode -int chacha20poly1305_seal( +int aead_seal( + const enum aeadType cipherId, const unsigned char* const plaintext, const int plaintextLen, const unsigned char* const authData, @@ -23,6 +23,18 @@ int chacha20poly1305_seal( unsigned char* const ciphertext, const int ciphertextBufLen) { + const EVP_CIPHER* evpCipher = NULL; + switch (cipherId) { + case aeadTypeChacha: + evpCipher = EVP_chacha20_poly1305(); + break; + case aeadTypeGcm: + evpCipher = EVP_aes_256_gcm(); + break; + default: + panic("unknown cipherId"); + } + // Create scratch space "context" EVP_CIPHER_CTX* ctx = EVP_CIPHER_CTX_new(); if (!ctx) { @@ -30,7 +42,7 @@ int chacha20poly1305_seal( } // Set cipher - if (EVP_EncryptInit_ex(ctx, EVP_chacha20_poly1305(), NULL, NULL, NULL) != 1) { + if (EVP_EncryptInit_ex(ctx, evpCipher, NULL, NULL, NULL) != 1) { panic("EVP_EncryptInit_ex set cipher failed"); } diff --git a/internal/stupidgcm/chacha.h b/internal/stupidgcm/chacha.h new file mode 100644 index 0000000..780350a --- /dev/null +++ b/internal/stupidgcm/chacha.h @@ -0,0 +1,17 @@ +enum aeadType { + aeadTypeChacha = 1, + aeadTypeGcm = 2, +}; + +int aead_seal( + const enum aeadType cipherId, + const unsigned char* const plaintext, + const int plaintextLen, + const unsigned char* const authData, + const int authDataLen, + const unsigned char* const key, + const int keyLen, + const unsigned char* const iv, + const int ivLen, + unsigned char* const ciphertext, + const int ciphertextBufLen); diff --git a/internal/stupidgcm/stupidchacha.go b/internal/stupidgcm/stupidchacha.go index 1f16a5e..18037c6 100644 --- a/internal/stupidgcm/stupidchacha.go +++ b/internal/stupidgcm/stupidchacha.go @@ -13,12 +13,8 @@ import ( /* #include +#include "chacha.h" #cgo pkg-config: libcrypto -int chacha20poly1305_seal(const unsigned char * const plaintext, const int plaintextLen, - const unsigned char * const authData, const int authDataLen, - const unsigned char * const key, const int keyLen, - const unsigned char * const iv, const int ivLen, - unsigned char * const ciphertext, const int ciphertextBufLen); */ import "C" @@ -75,7 +71,8 @@ func (g *stupidChacha20poly1305) Seal(dst, iv, in, authData []byte) []byte { buf = make([]byte, outLen) } - C.chacha20poly1305_seal((*C.uchar)(&in[0]), + C.aead_seal(C.aeadTypeChacha, + (*C.uchar)(&in[0]), C.int(len(in)), (*C.uchar)(&authData[0]), C.int(len(authData)), From d9e89cd0210b1d8a1c9cbb33c77013211dfc1ffd Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Fri, 3 Sep 2021 17:19:12 +0200 Subject: [PATCH 23/59] stupidgcm: use aead_seal for gcm as well MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit $ benchstat old.txt new.txt name old time/op new time/op delta StupidGCM-4 7.87µs ± 1% 6.64µs ± 2% -15.65% (p=0.000 n=10+10) name old speed new speed delta StupidGCM-4 520MB/s ± 1% 617MB/s ± 2% +18.56% (p=0.000 n=10+10) --- internal/stupidgcm/stupidgcm.go | 69 ++++++--------------------------- 1 file changed, 12 insertions(+), 57 deletions(-) diff --git a/internal/stupidgcm/stupidgcm.go b/internal/stupidgcm/stupidgcm.go index 3499c85..46b6b86 100644 --- a/internal/stupidgcm/stupidgcm.go +++ b/internal/stupidgcm/stupidgcm.go @@ -5,6 +5,7 @@ package stupidgcm // #include +// #include "chacha.h" // #cgo pkg-config: libcrypto import "C" @@ -76,63 +77,17 @@ func (g *StupidGCM) Seal(dst, iv, in, authData []byte) []byte { buf = make([]byte, outLen) } - // https://wiki.openssl.org/index.php/EVP_Authenticated_Encryption_and_Decryption#Authenticated_Encryption_using_GCM_mode - - // Create scratch space "context" - ctx := C.EVP_CIPHER_CTX_new() - if ctx == nil { - log.Panic("EVP_CIPHER_CTX_new failed") - } - - // Set cipher to AES-256 - if C.EVP_EncryptInit_ex(ctx, C.EVP_aes_256_gcm(), nil, nil, nil) != 1 { - log.Panic("EVP_EncryptInit_ex I failed") - } - - // Use 16-byte IV - if C.EVP_CIPHER_CTX_ctrl(ctx, C.EVP_CTRL_GCM_SET_IVLEN, ivLen, nil) != 1 { - log.Panic("EVP_CIPHER_CTX_ctrl EVP_CTRL_GCM_SET_IVLEN failed") - } - - // Set key and IV - if C.EVP_EncryptInit_ex(ctx, nil, nil, (*C.uchar)(&g.key[0]), (*C.uchar)(&iv[0])) != 1 { - log.Panic("EVP_EncryptInit_ex II failed") - } - - // Provide authentication data - var resultLen C.int - if C.EVP_EncryptUpdate(ctx, nil, &resultLen, (*C.uchar)(&authData[0]), C.int(len(authData))) != 1 { - log.Panic("EVP_EncryptUpdate authData failed") - } - if int(resultLen) != len(authData) { - log.Panicf("Unexpected length %d", resultLen) - } - - // Encrypt "in" into "buf" - if C.EVP_EncryptUpdate(ctx, (*C.uchar)(&buf[0]), &resultLen, (*C.uchar)(&in[0]), C.int(len(in))) != 1 { - log.Panic("EVP_EncryptUpdate failed") - } - if int(resultLen) != len(in) { - log.Panicf("Unexpected length %d", resultLen) - } - - // Finalise encryption - // Because GCM is a stream encryption, this will not write out any data. - dummy := make([]byte, 16) - if C.EVP_EncryptFinal_ex(ctx, (*C.uchar)(&dummy[0]), &resultLen) != 1 { - log.Panic("EVP_EncryptFinal_ex failed") - } - if resultLen != 0 { - log.Panicf("Unexpected length %d", resultLen) - } - - // Get GMAC tag and append it to the ciphertext in "buf" - if C.EVP_CIPHER_CTX_ctrl(ctx, C.EVP_CTRL_GCM_GET_TAG, tagLen, (unsafe.Pointer)(&buf[len(in)])) != 1 { - log.Panic("EVP_CIPHER_CTX_ctrl EVP_CTRL_GCM_GET_TAG failed") - } - - // Free scratch space - C.EVP_CIPHER_CTX_free(ctx) + C.aead_seal(C.aeadTypeGcm, + (*C.uchar)(&in[0]), + C.int(len(in)), + (*C.uchar)(&authData[0]), + C.int(len(authData)), + (*C.uchar)(&g.key[0]), + C.int(len(g.key)), + (*C.uchar)(&iv[0]), + C.int(len(iv)), + (*C.uchar)(&buf[0]), + C.int(len(buf))) if inplace { return dst[:len(dst)+outLen] From 5046962634d83a01341585f9f96090add2396e82 Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Fri, 3 Sep 2021 17:40:29 +0200 Subject: [PATCH 24/59] speed: add bEncrypt helper, reuse dst buffer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bEncrypt helper massively deduplicates the code, and reusing the dst buffer gives higher performance, and that's what gocryptfs does in normal operation via sync.Pool. $ benchstat old.txt new.txt name old time/op new time/op delta StupidGCM-4 6.24µs ± 1% 4.65µs ± 0% -25.47% (p=0.008 n=5+5) GoGCM-4 4.90µs ± 0% 4.10µs ± 0% -16.44% (p=0.008 n=5+5) AESSIV-4 26.4µs ± 0% 25.6µs ± 0% -2.90% (p=0.008 n=5+5) Xchacha-4 5.76µs ± 0% 4.91µs ± 0% -14.79% (p=0.008 n=5+5) StupidXchacha-4 7.24µs ± 1% 5.48µs ± 0% -24.33% (p=0.008 n=5+5) name old speed new speed delta StupidGCM-4 656MB/s ± 1% 880MB/s ± 0% +34.15% (p=0.008 n=5+5) GoGCM-4 835MB/s ± 0% 1000MB/s ± 0% +19.68% (p=0.008 n=5+5) AESSIV-4 155MB/s ± 0% 160MB/s ± 0% +2.99% (p=0.008 n=5+5) Xchacha-4 711MB/s ± 0% 834MB/s ± 0% +17.35% (p=0.008 n=5+5) StupidXchacha-4 565MB/s ± 1% 747MB/s ± 0% +32.15% (p=0.008 n=5+5) --- internal/speed/speed.go | 89 +++++++++++++---------------------------- 1 file changed, 27 insertions(+), 62 deletions(-) diff --git a/internal/speed/speed.go b/internal/speed/speed.go index d53e402..37b9daf 100644 --- a/internal/speed/speed.go +++ b/internal/speed/speed.go @@ -73,35 +73,36 @@ func randBytes(n int) []byte { return b } +// bEncrypt benchmarks the encryption speed of cipher "c" +func bEncrypt(b *testing.B, c cipher.AEAD) { + authData := randBytes(adLen) + iv := randBytes(c.NonceSize()) + in := make([]byte, blockSize) + dst := make([]byte, len(in)+len(iv)+c.Overhead()) + copy(dst, iv) + + b.SetBytes(int64(len(in))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + // Reset dst buffer + dst = dst[:len(iv)] + // Encrypt and append to nonce + c.Seal(dst, iv, in, authData) + } + +} + // bStupidGCM benchmarks stupidgcm's openssl GCM func bStupidGCM(b *testing.B) { if stupidgcm.BuiltWithoutOpenssl { b.Skip("openssl has been disabled at compile-time") } - key := randBytes(32) - authData := randBytes(adLen) - iv := randBytes(16) - in := make([]byte, blockSize) - b.SetBytes(int64(len(in))) - - sGCM := stupidgcm.New(key, false) - - b.ResetTimer() - for i := 0; i < b.N; i++ { - // Encrypt and append to nonce - sGCM.Seal(iv, iv, in, authData) - } + bEncrypt(b, stupidgcm.New(randBytes(32), false)) } // bGoGCM benchmarks Go stdlib GCM func bGoGCM(b *testing.B) { - key := randBytes(32) - authData := randBytes(adLen) - iv := randBytes(16) - in := make([]byte, blockSize) - b.SetBytes(int64(len(in))) - - gAES, err := aes.NewCipher(key) + gAES, err := aes.NewCipher(randBytes(32)) if err != nil { b.Fatal(err) } @@ -109,44 +110,19 @@ func bGoGCM(b *testing.B) { if err != nil { b.Fatal(err) } - - b.ResetTimer() - for i := 0; i < b.N; i++ { - // Encrypt and append to nonce - gGCM.Seal(iv, iv, in, authData) - } + bEncrypt(b, gGCM) } // bAESSIV benchmarks AES-SIV from github.com/jacobsa/crypto/siv func bAESSIV(b *testing.B) { - key := randBytes(64) - authData := randBytes(adLen) - iv := randBytes(16) - in := make([]byte, blockSize) - b.SetBytes(int64(len(in))) - gGCM := siv_aead.New(key) - - b.ResetTimer() - for i := 0; i < b.N; i++ { - // Encrypt and append to nonce - gGCM.Seal(iv, iv, in, authData) - } + c := siv_aead.New(randBytes(64)) + bEncrypt(b, c) } // bXchacha20poly1305 benchmarks XChaCha20 from golang.org/x/crypto/chacha20poly1305 func bXchacha20poly1305(b *testing.B) { - key := randBytes(32) - authData := randBytes(adLen) - iv := randBytes(chacha20poly1305.NonceSizeX) - in := make([]byte, blockSize) - b.SetBytes(int64(len(in))) - c, _ := chacha20poly1305.NewX(key) - - b.ResetTimer() - for i := 0; i < b.N; i++ { - // Encrypt and append to nonce - c.Seal(iv, iv, in, authData) - } + c, _ := chacha20poly1305.NewX(randBytes(32)) + bEncrypt(b, c) } // bStupidXchacha benchmarks OpenSSL XChaCha20 @@ -154,16 +130,5 @@ func bStupidXchacha(b *testing.B) { if stupidgcm.BuiltWithoutOpenssl { b.Skip("openssl has been disabled at compile-time") } - key := randBytes(32) - authData := randBytes(adLen) - iv := randBytes(chacha20poly1305.NonceSizeX) - in := make([]byte, blockSize) - b.SetBytes(int64(len(in))) - c := stupidgcm.NewXchacha20poly1305(key) - - b.ResetTimer() - for i := 0; i < b.N; i++ { - // Encrypt and append to nonce - c.Seal(iv, iv, in, authData) - } + bEncrypt(b, stupidgcm.NewXchacha20poly1305(randBytes(32))) } From 3e27acb989614f82a25d50ab19a0a4419f19aa27 Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Fri, 3 Sep 2021 18:30:42 +0200 Subject: [PATCH 25/59] speed: add decryption benchmarks gocryptfs/internal/speed$ go test -bench . goos: linux goarch: amd64 pkg: github.com/rfjakob/gocryptfs/v2/internal/speed cpu: Intel(R) Core(TM) i5-3470 CPU @ 3.20GHz BenchmarkStupidGCM-4 263742 4523 ns/op 905.61 MB/s BenchmarkStupidGCMDecrypt-4 204858 5779 ns/op 708.76 MB/s BenchmarkGoGCM-4 291259 4095 ns/op 1000.25 MB/s BenchmarkGoGCMDecrypt-4 293886 4061 ns/op 1008.53 MB/s BenchmarkAESSIV-4 46537 25538 ns/op 160.39 MB/s BenchmarkAESSIVDecrypt-4 46770 25627 ns/op 159.83 MB/s BenchmarkXchacha-4 243619 4893 ns/op 837.03 MB/s BenchmarkXchachaDecrypt-4 248857 4793 ns/op 854.51 MB/s BenchmarkStupidXchacha-4 213717 5558 ns/op 736.99 MB/s BenchmarkStupidXchachaDecrypt-4 176635 6782 ns/op 603.96 MB/s PASS ok github.com/rfjakob/gocryptfs/v2/internal/speed 12.871s --- internal/speed/speed.go | 19 +++++++++++++++ internal/speed/speed_test.go | 47 +++++++++++++++++++++++++++++++++--- 2 files changed, 62 insertions(+), 4 deletions(-) diff --git a/internal/speed/speed.go b/internal/speed/speed.go index 37b9daf..a696703 100644 --- a/internal/speed/speed.go +++ b/internal/speed/speed.go @@ -92,6 +92,25 @@ func bEncrypt(b *testing.B, c cipher.AEAD) { } +func bDecrypt(b *testing.B, c cipher.AEAD) { + authData := randBytes(adLen) + iv := randBytes(c.NonceSize()) + plain := randBytes(blockSize) + ciphertext := c.Seal(iv, iv, plain, authData) + + b.SetBytes(int64(len(plain))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + // Reset plain buffer + plain = plain[:0] + // Decrypt + _, err := c.Open(plain, iv, ciphertext[c.NonceSize():], authData) + if err != nil { + b.Fatal(err) + } + } +} + // bStupidGCM benchmarks stupidgcm's openssl GCM func bStupidGCM(b *testing.B) { if stupidgcm.BuiltWithoutOpenssl { diff --git a/internal/speed/speed_test.go b/internal/speed/speed_test.go index a6f3f30..4d09148 100644 --- a/internal/speed/speed_test.go +++ b/internal/speed/speed_test.go @@ -1,5 +1,16 @@ package speed +import ( + "crypto/aes" + "crypto/cipher" + "testing" + + "golang.org/x/crypto/chacha20poly1305" + + "github.com/rfjakob/gocryptfs/v2/internal/siv_aead" + "github.com/rfjakob/gocryptfs/v2/internal/stupidgcm" +) + /* Make the "-speed" benchmarks also accessible to the standard test system. Example run: @@ -12,26 +23,54 @@ PASS ok github.com/rfjakob/gocryptfs/v2/internal/speed 6.022s */ -import ( - "testing" -) - func BenchmarkStupidGCM(b *testing.B) { bStupidGCM(b) } +func BenchmarkStupidGCMDecrypt(b *testing.B) { + if stupidgcm.BuiltWithoutOpenssl { + b.Skip("openssl has been disabled at compile-time") + } + bDecrypt(b, stupidgcm.New(randBytes(32), false)) +} + func BenchmarkGoGCM(b *testing.B) { bGoGCM(b) } +func BenchmarkGoGCMDecrypt(b *testing.B) { + gAES, err := aes.NewCipher(randBytes(32)) + if err != nil { + b.Fatal(err) + } + gGCM, err := cipher.NewGCMWithNonceSize(gAES, 16) + if err != nil { + b.Fatal(err) + } + bDecrypt(b, gGCM) +} + func BenchmarkAESSIV(b *testing.B) { bAESSIV(b) } +func BenchmarkAESSIVDecrypt(b *testing.B) { + bEncrypt(b, siv_aead.New(randBytes(64))) +} + func BenchmarkXchacha(b *testing.B) { bXchacha20poly1305(b) } +func BenchmarkXchachaDecrypt(b *testing.B) { + c, _ := chacha20poly1305.NewX(randBytes(32)) + bDecrypt(b, c) +} + func BenchmarkStupidXchacha(b *testing.B) { bStupidXchacha(b) } + +func BenchmarkStupidXchachaDecrypt(b *testing.B) { + bDecrypt(b, stupidgcm.NewXchacha20poly1305(randBytes(32))) +} From bf572aef88963732849b8e5ae679e63c6be4aa46 Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Fri, 3 Sep 2021 18:44:41 +0200 Subject: [PATCH 26/59] stupidgcm: stupidChacha20poly1305.Open: batch C calls in aead_open MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gets the decryption speed to the same level as the encryption speed. internal/speed$ benchstat old.txt new.txt name old time/op new time/op delta StupidXchacha-4 732MB/s ± 0% 740MB/s ± 0% ~ (p=1.000 n=1+1) StupidXchachaDecrypt-4 602MB/s ± 0% 741MB/s ± 0% ~ (p=1.000 n=1+1) --- internal/stupidgcm/chacha.c | 120 ++++++++++++++++++++++++----- internal/stupidgcm/chacha.h | 15 ++++ internal/stupidgcm/stupidchacha.go | 68 ++++------------ 3 files changed, 134 insertions(+), 69 deletions(-) diff --git a/internal/stupidgcm/chacha.c b/internal/stupidgcm/chacha.c index e188bfc..05d68af 100644 --- a/internal/stupidgcm/chacha.c +++ b/internal/stupidgcm/chacha.c @@ -9,6 +9,21 @@ static void panic(const char* const msg) __builtin_trap(); } +static const EVP_CIPHER* getEvpCipher(enum aeadType cipherId) +{ + switch (cipherId) { + case aeadTypeChacha: + return EVP_chacha20_poly1305(); + case aeadTypeGcm: + return EVP_aes_256_gcm(); + } + panic("unknown cipherId"); + return NULL; +} + +// We only support 16-byte tags +static const int supportedTagLen = 16; + // https://wiki.openssl.org/index.php/EVP_Authenticated_Encryption_and_Decryption#Authenticated_Encryption_using_GCM_mode int aead_seal( const enum aeadType cipherId, @@ -23,19 +38,9 @@ int aead_seal( unsigned char* const ciphertext, const int ciphertextBufLen) { - const EVP_CIPHER* evpCipher = NULL; - switch (cipherId) { - case aeadTypeChacha: - evpCipher = EVP_chacha20_poly1305(); - break; - case aeadTypeGcm: - evpCipher = EVP_aes_256_gcm(); - break; - default: - panic("unknown cipherId"); - } + const EVP_CIPHER* evpCipher = getEvpCipher(cipherId); - // Create scratch space "context" + // Create scratch space "ctx" EVP_CIPHER_CTX* ctx = EVP_CIPHER_CTX_new(); if (!ctx) { panic("EVP_CIPHER_CTX_new failed"); @@ -91,20 +96,99 @@ int aead_seal( panic("EVP_EncryptFinal_ex: unexpected length"); } - // We only support 16-byte tags - const int tagLen = 16; - // Get MAC tag and append it to the ciphertext - if (ciphertextLen + tagLen > ciphertextBufLen) { + if (ciphertextLen + supportedTagLen > ciphertextBufLen) { panic("tag overflows output buffer"); } - if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_GET_TAG, tagLen, ciphertext + plaintextLen) != 1) { + if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_GET_TAG, supportedTagLen, ciphertext + plaintextLen) != 1) { panic("EVP_CTRL_AEAD_GET_TAG failed"); } - ciphertextLen += tagLen; + ciphertextLen += supportedTagLen; // Free scratch space EVP_CIPHER_CTX_free(ctx); return ciphertextLen; } + +int aead_open( + const enum aeadType cipherId, + const unsigned char* const ciphertext, + const int ciphertextLen, + const unsigned char* const authData, + const int authDataLen, + unsigned char* const tag, + const int tagLen, + const unsigned char* const key, + const int keyLen, + const unsigned char* const iv, + const int ivLen, + unsigned char* const plaintext, + const int plaintextBufLen) +{ + const EVP_CIPHER* evpCipher = getEvpCipher(cipherId); + + // Create scratch space "ctx" + EVP_CIPHER_CTX* ctx = EVP_CIPHER_CTX_new(); + if (!ctx) { + panic("EVP_CIPHER_CTX_new failed"); + } + + // Set cipher + if (EVP_DecryptInit_ex(ctx, evpCipher, NULL, NULL, NULL) != 1) { + panic("EVP_DecryptInit_ex set cipher failed"); + } + + // Check keyLen by trying to set it (fails if keyLen != 32) + if (EVP_CIPHER_CTX_set_key_length(ctx, keyLen) != 1) { + panic("keyLen mismatch"); + } + + // Set IV length so we do not depend on the default + if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_IVLEN, ivLen, NULL) != 1) { + panic("EVP_CTRL_AEAD_SET_IVLEN failed"); + } + + // Set key and IV + if (EVP_DecryptInit_ex(ctx, NULL, NULL, key, iv) != 1) { + panic("EVP_DecryptInit_ex set key & iv failed"); + } + + // Provide authentication data + int outLen = 0; + if (EVP_DecryptUpdate(ctx, NULL, &outLen, authData, authDataLen) != 1) { + panic("EVP_DecryptUpdate authData failed"); + } + if (outLen != authDataLen) { + panic("EVP_DecryptUpdate authData: unexpected length"); + } + + // Decrypt "ciphertext" into "plaintext" + if (ciphertextLen > plaintextBufLen) { + panic("ciphertextLen overflows output buffer"); + } + if (EVP_DecryptUpdate(ctx, plaintext, &outLen, ciphertext, ciphertextLen) != 1) { + panic("EVP_DecryptUpdate failed"); + } + int plaintextLen = outLen; + + // Check tag + if (tagLen != supportedTagLen) { + panic("unsupported tag length"); + } + if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_TAG, tagLen, tag) != 1) { + panic("EVP_CTRL_AEAD_SET_TAG failed"); + } + if (EVP_DecryptFinal_ex(ctx, plaintext + plaintextLen, &outLen) != 1) { + // authentication failed + return -1; + } + if (outLen != 0) { + panic("EVP_EncryptFinal_ex: unexpected length"); + } + + /* Clean up */ + EVP_CIPHER_CTX_free(ctx); + + return plaintextLen; +} diff --git a/internal/stupidgcm/chacha.h b/internal/stupidgcm/chacha.h index 780350a..a5eac04 100644 --- a/internal/stupidgcm/chacha.h +++ b/internal/stupidgcm/chacha.h @@ -15,3 +15,18 @@ int aead_seal( const int ivLen, unsigned char* const ciphertext, const int ciphertextBufLen); + +int aead_open( + const enum aeadType cipherId, + const unsigned char* const ciphertext, + const int ciphertextLen, + const unsigned char* const authData, + const int authDataLen, + unsigned char* const tag, + const int tagLen, + const unsigned char* const key, + const int keyLen, + const unsigned char* const iv, + const int ivLen, + unsigned char* const plaintext, + const int plaintextBufLen); diff --git a/internal/stupidgcm/stupidchacha.go b/internal/stupidgcm/stupidchacha.go index 18037c6..5073aa3 100644 --- a/internal/stupidgcm/stupidchacha.go +++ b/internal/stupidgcm/stupidchacha.go @@ -6,7 +6,6 @@ import ( "crypto/cipher" "fmt" "log" - "unsafe" "golang.org/x/crypto/chacha20poly1305" ) @@ -118,59 +117,26 @@ func (g *stupidChacha20poly1305) Open(dst, iv, in, authData []byte) ([]byte, err ciphertext := in[:len(in)-tagLen] tag := in[len(in)-tagLen:] - // https://wiki.openssl.org/index.php/EVP_Authenticated_Encryption_and_Decryption#Authenticated_Encryption_using_GCM_mode + res := int(C.aead_open(C.aeadTypeChacha, + (*C.uchar)(&ciphertext[0]), + C.int(len(ciphertext)), + (*C.uchar)(&authData[0]), + C.int(len(authData)), + (*C.uchar)(&tag[0]), + C.int(len(tag)), + (*C.uchar)(&g.key[0]), + C.int(len(g.key)), + (*C.uchar)(&iv[0]), + C.int(len(iv)), + (*C.uchar)(&buf[0]), + C.int(len(buf)))) - // Create scratch space "context" - ctx := C.EVP_CIPHER_CTX_new() - if ctx == nil { - log.Panic("EVP_CIPHER_CTX_new failed") - } - - // Set cipher to AES-256 - if C.EVP_DecryptInit_ex(ctx, C.EVP_chacha20_poly1305(), nil, nil, nil) != 1 { - log.Panic("EVP_DecryptInit_ex I failed") - } - - // Set key and IV - if C.EVP_DecryptInit_ex(ctx, nil, nil, (*C.uchar)(&g.key[0]), (*C.uchar)(&iv[0])) != 1 { - log.Panic("EVP_DecryptInit_ex II failed") - } - - // Set expected MAC tag - if C.EVP_CIPHER_CTX_ctrl(ctx, C.EVP_CTRL_AEAD_SET_TAG, tagLen, (unsafe.Pointer)(&tag[0])) != 1 { - log.Panic("EVP_CIPHER_CTX_ctrl EVP_CTRL_AEAD_SET_TAG failed") - } - - // Provide authentication data - var resultLen C.int - if C.EVP_DecryptUpdate(ctx, nil, &resultLen, (*C.uchar)(&authData[0]), C.int(len(authData))) != 1 { - log.Panic("EVP_DecryptUpdate authData failed") - } - if int(resultLen) != len(authData) { - log.Panicf("Unexpected length %d", resultLen) - } - - // Decrypt "ciphertext" into "buf" - if C.EVP_DecryptUpdate(ctx, (*C.uchar)(&buf[0]), &resultLen, (*C.uchar)(&ciphertext[0]), C.int(len(ciphertext))) != 1 { - log.Panic("EVP_DecryptUpdate failed") - } - if int(resultLen) != len(ciphertext) { - log.Panicf("Unexpected length %d", resultLen) - } - - // Check MAC - dummy := make([]byte, 16) - res := C.EVP_DecryptFinal_ex(ctx, (*C.uchar)(&dummy[0]), &resultLen) - if resultLen != 0 { - log.Panicf("Unexpected length %d", resultLen) - } - - // Free scratch space - C.EVP_CIPHER_CTX_free(ctx) - - if res != 1 { + if res < 0 { return nil, ErrAuth } + if res != outLen { + log.Panicf("unexpected length %d", res) + } if inplace { return dst[:len(dst)+outLen], nil From e2ec048a09889b2bf71e8bbfef9f0584ff7d69db Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Sat, 4 Sep 2021 11:41:56 +0200 Subject: [PATCH 27/59] stupidgcm: introduce stupidAEADCommon and use for both chacha & gcm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nice deduplication and brings the GCM decrypt speed up to par. internal/speed$ benchstat old new name old time/op new time/op delta StupidGCM-4 4.71µs ± 0% 4.66µs ± 0% -0.99% (p=0.008 n=5+5) StupidGCMDecrypt-4 5.77µs ± 1% 4.51µs ± 0% -21.80% (p=0.008 n=5+5) name old speed new speed delta StupidGCM-4 870MB/s ± 0% 879MB/s ± 0% +1.01% (p=0.008 n=5+5) StupidGCMDecrypt-4 710MB/s ± 1% 908MB/s ± 0% +27.87% (p=0.008 n=5+5) --- internal/stupidgcm/chacha.go | 35 +++ .../{stupidchacha_test.go => chacha_test.go} | 0 internal/stupidgcm/common.go | 68 ++++++ internal/stupidgcm/common_test.go | 23 +- internal/stupidgcm/gcm.go | 45 ++++ .../{stupidgcm_test.go => gcm_test.go} | 0 internal/stupidgcm/openssl.go | 108 ++++++++++ .../stupidgcm/{chacha.c => openssl_aead.c} | 26 +-- .../stupidgcm/{chacha.h => openssl_aead.h} | 13 +- internal/stupidgcm/stupidchacha.go | 156 -------------- internal/stupidgcm/stupidgcm.go | 204 ------------------ .../{stupidxchacha.go => xchacha.go} | 10 +- ...{stupidxchacha_test.go => xchacha_test.go} | 0 13 files changed, 283 insertions(+), 405 deletions(-) create mode 100644 internal/stupidgcm/chacha.go rename internal/stupidgcm/{stupidchacha_test.go => chacha_test.go} (100%) create mode 100644 internal/stupidgcm/common.go create mode 100644 internal/stupidgcm/gcm.go rename internal/stupidgcm/{stupidgcm_test.go => gcm_test.go} (100%) create mode 100644 internal/stupidgcm/openssl.go rename internal/stupidgcm/{chacha.c => openssl_aead.c} (91%) rename internal/stupidgcm/{chacha.h => openssl_aead.h} (81%) delete mode 100644 internal/stupidgcm/stupidchacha.go delete mode 100644 internal/stupidgcm/stupidgcm.go rename internal/stupidgcm/{stupidxchacha.go => xchacha.go} (94%) rename internal/stupidgcm/{stupidxchacha_test.go => xchacha_test.go} (100%) diff --git a/internal/stupidgcm/chacha.go b/internal/stupidgcm/chacha.go new file mode 100644 index 0000000..37f7e1f --- /dev/null +++ b/internal/stupidgcm/chacha.go @@ -0,0 +1,35 @@ +// +build !without_openssl + +package stupidgcm + +import ( + "crypto/cipher" + "log" + + "golang.org/x/crypto/chacha20poly1305" +) + +/* +#include +*/ +import "C" + +type stupidChacha20poly1305 struct { + stupidAEADCommon +} + +// Verify that we satisfy the cipher.AEAD interface +var _ cipher.AEAD = &stupidChacha20poly1305{} + +func newChacha20poly1305(key []byte) *stupidChacha20poly1305 { + if len(key) != chacha20poly1305.KeySize { + log.Panicf("Only %d-byte keys are supported, you passed %d bytes", chacha20poly1305.KeySize, len(key)) + } + return &stupidChacha20poly1305{ + stupidAEADCommon{ + key: append([]byte{}, key...), // private copy + openSSLEVPCipher: C.EVP_chacha20_poly1305(), + nonceSize: chacha20poly1305.NonceSize, + }, + } +} diff --git a/internal/stupidgcm/stupidchacha_test.go b/internal/stupidgcm/chacha_test.go similarity index 100% rename from internal/stupidgcm/stupidchacha_test.go rename to internal/stupidgcm/chacha_test.go diff --git a/internal/stupidgcm/common.go b/internal/stupidgcm/common.go new file mode 100644 index 0000000..3788315 --- /dev/null +++ b/internal/stupidgcm/common.go @@ -0,0 +1,68 @@ +package stupidgcm + +import ( + "log" +) + +/* +#include +*/ +import "C" + +type stupidAEADCommon struct { + wiped bool + key []byte + openSSLEVPCipher *C.EVP_CIPHER + nonceSize int +} + +// Overhead returns the number of bytes that are added for authentication. +// +// Part of the cipher.AEAD interface. +func (c *stupidAEADCommon) Overhead() int { + return tagLen +} + +// NonceSize returns the required size of the nonce / IV +// +// Part of the cipher.AEAD interface. +func (c *stupidAEADCommon) NonceSize() int { + return c.nonceSize +} + +// Seal encrypts "in" using "iv" and "authData" and append the result to "dst" +// +// Part of the cipher.AEAD interface. +func (c *stupidAEADCommon) Seal(dst, iv, in, authData []byte) []byte { + return openSSLSeal(c, dst, iv, in, authData) +} + +// Open decrypts "in" using "iv" and "authData" and append the result to "dst" +// +// Part of the cipher.AEAD interface. +func (c *stupidAEADCommon) Open(dst, iv, in, authData []byte) ([]byte, error) { + return openSSLOpen(c, dst, iv, in, authData) +} + +// Wipe tries to wipe the key from memory by overwriting it with zeros. +// +// This is not bulletproof due to possible GC copies, but +// still raises the bar for extracting the key. +func (c *stupidAEADCommon) Wipe() { + key := c.key + c.wiped = true + c.key = nil + for i := range key { + key[i] = 0 + } +} + +func (c *stupidAEADCommon) Wiped() bool { + if c.wiped { + return true + } + if len(c.key) != keyLen { + log.Panicf("wrong key length %d", len(c.key)) + } + return false +} diff --git a/internal/stupidgcm/common_test.go b/internal/stupidgcm/common_test.go index 8123ce2..a8080ca 100644 --- a/internal/stupidgcm/common_test.go +++ b/internal/stupidgcm/common_test.go @@ -162,25 +162,26 @@ func testCorruption(t *testing.T, c cipher.AEAD) { } } -type Wiper interface { - Wipe() -} - func testWipe(t *testing.T, c cipher.AEAD) { switch c2 := c.(type) { case *StupidGCM: c2.Wipe() - if c2.key != nil { - t.Fatal("key is not nil") - } - case *stupidChacha20poly1305: - c2.Wipe() - if !c2.wiped { + if !c2.Wiped() { t.Error("c2.wiped is not set") } for _, v := range c2.key { if v != 0 { - t.Fatal("c2.key is not zeroed") + t.Fatal("c2._key is not zeroed") + } + } + case *stupidChacha20poly1305: + c2.Wipe() + if !c2.Wiped() { + t.Error("c2.wiped is not set") + } + for _, v := range c2.key { + if v != 0 { + t.Fatal("c2._key is not zeroed") } } case *stupidXchacha20poly1305: diff --git a/internal/stupidgcm/gcm.go b/internal/stupidgcm/gcm.go new file mode 100644 index 0000000..439e7a7 --- /dev/null +++ b/internal/stupidgcm/gcm.go @@ -0,0 +1,45 @@ +// +build !without_openssl + +// Package stupidgcm is a thin wrapper for OpenSSL's GCM encryption and +// decryption functions. It only support 32-byte keys and 16-bit IVs. +package stupidgcm + +// #include +import "C" + +import ( + "crypto/cipher" + "log" +) + +const ( + // BuiltWithoutOpenssl indicates if openssl been disabled at compile-time + BuiltWithoutOpenssl = false + + keyLen = 32 + ivLen = 16 + tagLen = 16 +) + +// StupidGCM implements the cipher.AEAD interface +type StupidGCM struct { + stupidAEADCommon +} + +// Verify that we satisfy the interface +var _ cipher.AEAD = &StupidGCM{} + +// New returns a new cipher.AEAD implementation.. +func New(keyIn []byte, forceDecode bool) cipher.AEAD { + if len(keyIn) != keyLen { + log.Panicf("Only %d-byte keys are supported", keyLen) + } + return &StupidGCM{ + stupidAEADCommon{ + // Create a private copy of the key + key: append([]byte{}, keyIn...), + openSSLEVPCipher: C.EVP_aes_256_gcm(), + nonceSize: ivLen, + }, + } +} diff --git a/internal/stupidgcm/stupidgcm_test.go b/internal/stupidgcm/gcm_test.go similarity index 100% rename from internal/stupidgcm/stupidgcm_test.go rename to internal/stupidgcm/gcm_test.go diff --git a/internal/stupidgcm/openssl.go b/internal/stupidgcm/openssl.go new file mode 100644 index 0000000..d57d100 --- /dev/null +++ b/internal/stupidgcm/openssl.go @@ -0,0 +1,108 @@ +package stupidgcm + +import ( + "fmt" + "log" +) + +/* +#include "openssl_aead.h" +#cgo pkg-config: libcrypto +*/ +import "C" + +func openSSLSeal(a *stupidAEADCommon, dst, iv, in, authData []byte) []byte { + if a.Wiped() { + panic("BUG: tried to use wiped key") + } + if len(iv) != a.NonceSize() { + log.Panicf("Only %d-byte IVs are supported, you passed %d bytes", a.NonceSize(), len(iv)) + } + if len(in) == 0 { + log.Panic("Zero-length input data is not supported") + } + + // If the "dst" slice is large enough we can use it as our output buffer + outLen := len(in) + tagLen + var buf []byte + inplace := false + if cap(dst)-len(dst) >= outLen { + inplace = true + buf = dst[len(dst) : len(dst)+outLen] + } else { + buf = make([]byte, outLen) + } + + res := int(C.openssl_aead_seal(a.openSSLEVPCipher, + (*C.uchar)(&in[0]), + C.int(len(in)), + (*C.uchar)(&authData[0]), + C.int(len(authData)), + (*C.uchar)(&a.key[0]), + C.int(len(a.key)), + (*C.uchar)(&iv[0]), + C.int(len(iv)), + (*C.uchar)(&buf[0]), + C.int(len(buf)))) + + if res != outLen { + log.Panicf("expected length %d, got %d", outLen, res) + } + + if inplace { + return dst[:len(dst)+outLen] + } + return append(dst, buf...) +} + +func openSSLOpen(a *stupidAEADCommon, dst, iv, in, authData []byte) ([]byte, error) { + if a.Wiped() { + panic("BUG: tried to use wiped key") + } + if len(iv) != a.NonceSize() { + log.Panicf("Only %d-byte IVs are supported, you passed %d bytes", a.NonceSize(), len(iv)) + } + if len(in) <= tagLen { + return nil, fmt.Errorf("stupidChacha20poly1305: input data too short (%d bytes)", len(in)) + } + + // If the "dst" slice is large enough we can use it as our output buffer + outLen := len(in) - tagLen + var buf []byte + inplace := false + if cap(dst)-len(dst) >= outLen { + inplace = true + buf = dst[len(dst) : len(dst)+outLen] + } else { + buf = make([]byte, len(in)-tagLen) + } + + ciphertext := in[:len(in)-tagLen] + tag := in[len(in)-tagLen:] + + res := int(C.openssl_aead_open(a.openSSLEVPCipher, + (*C.uchar)(&ciphertext[0]), + C.int(len(ciphertext)), + (*C.uchar)(&authData[0]), + C.int(len(authData)), + (*C.uchar)(&tag[0]), + C.int(len(tag)), + (*C.uchar)(&a.key[0]), + C.int(len(a.key)), + (*C.uchar)(&iv[0]), + C.int(len(iv)), + (*C.uchar)(&buf[0]), + C.int(len(buf)))) + + if res < 0 { + return nil, ErrAuth + } + if res != outLen { + log.Panicf("unexpected length %d", res) + } + + if inplace { + return dst[:len(dst)+outLen], nil + } + return append(dst, buf...), nil +} diff --git a/internal/stupidgcm/chacha.c b/internal/stupidgcm/openssl_aead.c similarity index 91% rename from internal/stupidgcm/chacha.c rename to internal/stupidgcm/openssl_aead.c index 05d68af..9dc6866 100644 --- a/internal/stupidgcm/chacha.c +++ b/internal/stupidgcm/openssl_aead.c @@ -1,4 +1,4 @@ -#include "chacha.h" +#include "openssl_aead.h" #include #include //#cgo pkg-config: libcrypto @@ -9,24 +9,12 @@ static void panic(const char* const msg) __builtin_trap(); } -static const EVP_CIPHER* getEvpCipher(enum aeadType cipherId) -{ - switch (cipherId) { - case aeadTypeChacha: - return EVP_chacha20_poly1305(); - case aeadTypeGcm: - return EVP_aes_256_gcm(); - } - panic("unknown cipherId"); - return NULL; -} - // We only support 16-byte tags static const int supportedTagLen = 16; // https://wiki.openssl.org/index.php/EVP_Authenticated_Encryption_and_Decryption#Authenticated_Encryption_using_GCM_mode -int aead_seal( - const enum aeadType cipherId, +int openssl_aead_seal( + const EVP_CIPHER* evpCipher, const unsigned char* const plaintext, const int plaintextLen, const unsigned char* const authData, @@ -38,8 +26,6 @@ int aead_seal( unsigned char* const ciphertext, const int ciphertextBufLen) { - const EVP_CIPHER* evpCipher = getEvpCipher(cipherId); - // Create scratch space "ctx" EVP_CIPHER_CTX* ctx = EVP_CIPHER_CTX_new(); if (!ctx) { @@ -111,8 +97,8 @@ int aead_seal( return ciphertextLen; } -int aead_open( - const enum aeadType cipherId, +int openssl_aead_open( + const EVP_CIPHER* evpCipher, const unsigned char* const ciphertext, const int ciphertextLen, const unsigned char* const authData, @@ -126,8 +112,6 @@ int aead_open( unsigned char* const plaintext, const int plaintextBufLen) { - const EVP_CIPHER* evpCipher = getEvpCipher(cipherId); - // Create scratch space "ctx" EVP_CIPHER_CTX* ctx = EVP_CIPHER_CTX_new(); if (!ctx) { diff --git a/internal/stupidgcm/chacha.h b/internal/stupidgcm/openssl_aead.h similarity index 81% rename from internal/stupidgcm/chacha.h rename to internal/stupidgcm/openssl_aead.h index a5eac04..6a818b6 100644 --- a/internal/stupidgcm/chacha.h +++ b/internal/stupidgcm/openssl_aead.h @@ -1,10 +1,7 @@ -enum aeadType { - aeadTypeChacha = 1, - aeadTypeGcm = 2, -}; +#include -int aead_seal( - const enum aeadType cipherId, +int openssl_aead_seal( + const EVP_CIPHER* evpCipher, const unsigned char* const plaintext, const int plaintextLen, const unsigned char* const authData, @@ -16,8 +13,8 @@ int aead_seal( unsigned char* const ciphertext, const int ciphertextBufLen); -int aead_open( - const enum aeadType cipherId, +int openssl_aead_open( + const EVP_CIPHER* evpCipher, const unsigned char* const ciphertext, const int ciphertextLen, const unsigned char* const authData, diff --git a/internal/stupidgcm/stupidchacha.go b/internal/stupidgcm/stupidchacha.go deleted file mode 100644 index 5073aa3..0000000 --- a/internal/stupidgcm/stupidchacha.go +++ /dev/null @@ -1,156 +0,0 @@ -// +build !without_openssl - -package stupidgcm - -import ( - "crypto/cipher" - "fmt" - "log" - - "golang.org/x/crypto/chacha20poly1305" -) - -/* -#include -#include "chacha.h" -#cgo pkg-config: libcrypto -*/ -import "C" - -type stupidChacha20poly1305 struct { - key [chacha20poly1305.KeySize]byte - wiped bool -} - -// Verify that we satisfy the cipher.AEAD interface -var _ cipher.AEAD = &stupidChacha20poly1305{} - -func newChacha20poly1305(key []byte) cipher.AEAD { - if len(key) != chacha20poly1305.KeySize { - log.Panicf("Only %d-byte keys are supported, you passed %d bytes", chacha20poly1305.KeySize, len(key)) - } - ret := new(stupidChacha20poly1305) - copy(ret.key[:], key) - return ret -} - -// NonceSize returns the required size of the nonce / IV. -func (g *stupidChacha20poly1305) NonceSize() int { - return chacha20poly1305.NonceSize -} - -// Overhead returns the number of bytes that are added for authentication. -func (g *stupidChacha20poly1305) Overhead() int { - return tagLen -} - -// Seal encrypts "in" using "iv" and "authData" and append the result to "dst" -func (g *stupidChacha20poly1305) Seal(dst, iv, in, authData []byte) []byte { - if g.wiped { - panic("BUG: tried to use wiped key") - } - if len(iv) != g.NonceSize() { - log.Panicf("Only %d-byte IVs are supported, you passed %d bytes", g.NonceSize(), len(iv)) - } - if len(in) == 0 { - log.Panic("Zero-length input data is not supported") - } - if len(g.key) != chacha20poly1305.KeySize { - log.Panicf("Wrong key length: %d. Key has been wiped?", len(g.key)) - } - - // If the "dst" slice is large enough we can use it as our output buffer - outLen := len(in) + tagLen - var buf []byte - inplace := false - if cap(dst)-len(dst) >= outLen { - inplace = true - buf = dst[len(dst) : len(dst)+outLen] - } else { - buf = make([]byte, outLen) - } - - C.aead_seal(C.aeadTypeChacha, - (*C.uchar)(&in[0]), - C.int(len(in)), - (*C.uchar)(&authData[0]), - C.int(len(authData)), - (*C.uchar)(&g.key[0]), - C.int(len(g.key)), - (*C.uchar)(&iv[0]), - C.int(len(iv)), - (*C.uchar)(&buf[0]), - C.int(len(buf))) - - if inplace { - return dst[:len(dst)+outLen] - } - return append(dst, buf...) -} - -// Open decrypts "in" using "iv" and "authData" and append the result to "dst" -func (g *stupidChacha20poly1305) Open(dst, iv, in, authData []byte) ([]byte, error) { - if g.wiped { - panic("BUG: tried to use wiped key") - } - if len(iv) != g.NonceSize() { - log.Panicf("Only %d-byte IVs are supported", g.NonceSize()) - } - if len(g.key) != chacha20poly1305.KeySize { - log.Panicf("Wrong key length: %d. Key has been wiped?", len(g.key)) - } - if len(in) <= tagLen { - return nil, fmt.Errorf("stupidChacha20poly1305: input data too short (%d bytes)", len(in)) - } - - // If the "dst" slice is large enough we can use it as our output buffer - outLen := len(in) - tagLen - var buf []byte - inplace := false - if cap(dst)-len(dst) >= outLen { - inplace = true - buf = dst[len(dst) : len(dst)+outLen] - } else { - buf = make([]byte, len(in)-tagLen) - } - - ciphertext := in[:len(in)-tagLen] - tag := in[len(in)-tagLen:] - - res := int(C.aead_open(C.aeadTypeChacha, - (*C.uchar)(&ciphertext[0]), - C.int(len(ciphertext)), - (*C.uchar)(&authData[0]), - C.int(len(authData)), - (*C.uchar)(&tag[0]), - C.int(len(tag)), - (*C.uchar)(&g.key[0]), - C.int(len(g.key)), - (*C.uchar)(&iv[0]), - C.int(len(iv)), - (*C.uchar)(&buf[0]), - C.int(len(buf)))) - - if res < 0 { - return nil, ErrAuth - } - if res != outLen { - log.Panicf("unexpected length %d", res) - } - - if inplace { - return dst[:len(dst)+outLen], nil - } - return append(dst, buf...), nil -} - -// Wipe tries to wipe the key from memory by overwriting it with zeros. -// -// This is not bulletproof due to possible GC copies, but -// still raises the bar for extracting the key. -func (g *stupidChacha20poly1305) Wipe() { - g.wiped = true - for i := range g.key { - g.key[i] = 0 - } -} diff --git a/internal/stupidgcm/stupidgcm.go b/internal/stupidgcm/stupidgcm.go deleted file mode 100644 index 46b6b86..0000000 --- a/internal/stupidgcm/stupidgcm.go +++ /dev/null @@ -1,204 +0,0 @@ -// +build !without_openssl - -// Package stupidgcm is a thin wrapper for OpenSSL's GCM encryption and -// decryption functions. It only support 32-byte keys and 16-bit IVs. -package stupidgcm - -// #include -// #include "chacha.h" -// #cgo pkg-config: libcrypto -import "C" - -import ( - "crypto/cipher" - "fmt" - "log" - "unsafe" -) - -const ( - // BuiltWithoutOpenssl indicates if openssl been disabled at compile-time - BuiltWithoutOpenssl = false - - keyLen = 32 - ivLen = 16 - tagLen = 16 -) - -// StupidGCM implements the cipher.AEAD interface -type StupidGCM struct { - key []byte - forceDecode bool -} - -// Verify that we satisfy the cipher.AEAD interface -var _ cipher.AEAD = &StupidGCM{} - -// New returns a new cipher.AEAD implementation.. -func New(keyIn []byte, forceDecode bool) cipher.AEAD { - if len(keyIn) != keyLen { - log.Panicf("Only %d-byte keys are supported", keyLen) - } - // Create a private copy of the key - key := append([]byte{}, keyIn...) - return &StupidGCM{key: key, forceDecode: forceDecode} -} - -// NonceSize returns the required size of the nonce / IV. -func (g *StupidGCM) NonceSize() int { - return ivLen -} - -// Overhead returns the number of bytes that are added for authentication. -func (g *StupidGCM) Overhead() int { - return tagLen -} - -// Seal encrypts "in" using "iv" and "authData" and append the result to "dst" -func (g *StupidGCM) Seal(dst, iv, in, authData []byte) []byte { - if len(iv) != ivLen { - log.Panicf("Only %d-byte IVs are supported", ivLen) - } - if len(in) == 0 { - log.Panic("Zero-length input data is not supported") - } - if len(g.key) != keyLen { - log.Panicf("Wrong key length: %d. Key has been wiped?", len(g.key)) - } - - // If the "dst" slice is large enough we can use it as our output buffer - outLen := len(in) + tagLen - var buf []byte - inplace := false - if cap(dst)-len(dst) >= outLen { - inplace = true - buf = dst[len(dst) : len(dst)+outLen] - } else { - buf = make([]byte, outLen) - } - - C.aead_seal(C.aeadTypeGcm, - (*C.uchar)(&in[0]), - C.int(len(in)), - (*C.uchar)(&authData[0]), - C.int(len(authData)), - (*C.uchar)(&g.key[0]), - C.int(len(g.key)), - (*C.uchar)(&iv[0]), - C.int(len(iv)), - (*C.uchar)(&buf[0]), - C.int(len(buf))) - - if inplace { - return dst[:len(dst)+outLen] - } - return append(dst, buf...) -} - -// Open decrypts "in" using "iv" and "authData" and append the result to "dst" -func (g *StupidGCM) Open(dst, iv, in, authData []byte) ([]byte, error) { - if len(iv) != ivLen { - log.Panicf("Only %d-byte IVs are supported", ivLen) - } - if len(g.key) != keyLen { - log.Panicf("Wrong key length: %d. Key has been wiped?", len(g.key)) - } - if len(in) <= tagLen { - return nil, fmt.Errorf("stupidgcm: input data too short (%d bytes)", len(in)) - } - - // If the "dst" slice is large enough we can use it as our output buffer - outLen := len(in) - tagLen - var buf []byte - inplace := false - if cap(dst)-len(dst) >= outLen { - inplace = true - buf = dst[len(dst) : len(dst)+outLen] - } else { - buf = make([]byte, len(in)-tagLen) - } - - ciphertext := in[:len(in)-tagLen] - tag := in[len(in)-tagLen:] - - // https://wiki.openssl.org/index.php/EVP_Authenticated_Encryption_and_Decryption#Authenticated_Encryption_using_GCM_mode - - // Create scratch space "context" - ctx := C.EVP_CIPHER_CTX_new() - if ctx == nil { - log.Panic("EVP_CIPHER_CTX_new failed") - } - - // Set cipher to AES-256 - if C.EVP_DecryptInit_ex(ctx, C.EVP_aes_256_gcm(), nil, nil, nil) != 1 { - log.Panic("EVP_DecryptInit_ex I failed") - } - - // Use 16-byte IV - if C.EVP_CIPHER_CTX_ctrl(ctx, C.EVP_CTRL_GCM_SET_IVLEN, ivLen, nil) != 1 { - log.Panic("EVP_CIPHER_CTX_ctrl EVP_CTRL_GCM_SET_IVLEN failed") - } - - // Set key and IV - if C.EVP_DecryptInit_ex(ctx, nil, nil, (*C.uchar)(&g.key[0]), (*C.uchar)(&iv[0])) != 1 { - log.Panic("EVP_DecryptInit_ex II failed") - } - - // Set expected GMAC tag - if C.EVP_CIPHER_CTX_ctrl(ctx, C.EVP_CTRL_GCM_SET_TAG, tagLen, (unsafe.Pointer)(&tag[0])) != 1 { - log.Panic("EVP_CIPHER_CTX_ctrl failed") - } - - // Provide authentication data - var resultLen C.int - if C.EVP_DecryptUpdate(ctx, nil, &resultLen, (*C.uchar)(&authData[0]), C.int(len(authData))) != 1 { - log.Panic("EVP_DecryptUpdate authData failed") - } - if int(resultLen) != len(authData) { - log.Panicf("Unexpected length %d", resultLen) - } - - // Decrypt "ciphertext" into "buf" - if C.EVP_DecryptUpdate(ctx, (*C.uchar)(&buf[0]), &resultLen, (*C.uchar)(&ciphertext[0]), C.int(len(ciphertext))) != 1 { - log.Panic("EVP_DecryptUpdate failed") - } - if int(resultLen) != len(ciphertext) { - log.Panicf("Unexpected length %d", resultLen) - } - - // Check GMAC - dummy := make([]byte, 16) - res := C.EVP_DecryptFinal_ex(ctx, (*C.uchar)(&dummy[0]), &resultLen) - if resultLen != 0 { - log.Panicf("Unexpected length %d", resultLen) - } - - // Free scratch space - C.EVP_CIPHER_CTX_free(ctx) - - if res != 1 { - // The error code must always be checked by the calling function, because the decrypted buffer - // may contain corrupted data that we are returning in case the user forced reads - if g.forceDecode { - return append(dst, buf...), ErrAuth - } - return nil, ErrAuth - } - - if inplace { - return dst[:len(dst)+outLen], nil - } - return append(dst, buf...), nil -} - -// Wipe tries to wipe the AES key from memory by overwriting it with zeros -// and setting the reference to nil. -// -// This is not bulletproof due to possible GC copies, but -// still raises the bar for extracting the key. -func (g *StupidGCM) Wipe() { - for i := range g.key { - g.key[i] = 0 - } - g.key = nil -} diff --git a/internal/stupidgcm/stupidxchacha.go b/internal/stupidgcm/xchacha.go similarity index 94% rename from internal/stupidgcm/stupidxchacha.go rename to internal/stupidgcm/xchacha.go index 9f2ac2f..d8668dc 100644 --- a/internal/stupidgcm/stupidxchacha.go +++ b/internal/stupidgcm/xchacha.go @@ -17,6 +17,8 @@ import ( ) type stupidXchacha20poly1305 struct { + // array instead of byte slice like + // `struct xchacha20poly1305` in x/crypto/chacha20poly1305 key [chacha20poly1305.KeySize]byte wiped bool } @@ -41,7 +43,7 @@ func (*stupidXchacha20poly1305) NonceSize() int { } func (*stupidXchacha20poly1305) Overhead() int { - return 16 + return tagLen } func (x *stupidXchacha20poly1305) Seal(dst, nonce, plaintext, additionalData []byte) []byte { @@ -61,9 +63,8 @@ func (x *stupidXchacha20poly1305) Seal(dst, nonce, plaintext, additionalData []b panic("plaintext too large") } - c := new(stupidChacha20poly1305) hKey, _ := chacha20.HChaCha20(x.key[:], nonce[0:16]) - copy(c.key[:], hKey) + c := newChacha20poly1305(hKey) defer c.Wipe() // The first 4 bytes of the final nonce are unused counter space. @@ -87,9 +88,8 @@ func (x *stupidXchacha20poly1305) Open(dst, nonce, ciphertext, additionalData [] panic("ciphertext too large") } - c := new(stupidChacha20poly1305) hKey, _ := chacha20.HChaCha20(x.key[:], nonce[0:16]) - copy(c.key[:], hKey) + c := newChacha20poly1305(hKey) defer c.Wipe() // The first 4 bytes of the final nonce are unused counter space. diff --git a/internal/stupidgcm/stupidxchacha_test.go b/internal/stupidgcm/xchacha_test.go similarity index 100% rename from internal/stupidgcm/stupidxchacha_test.go rename to internal/stupidgcm/xchacha_test.go From c9728247edb4cb29755908629528b22c8419a11d Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Sat, 4 Sep 2021 11:56:43 +0200 Subject: [PATCH 28/59] test.bash: only check go files for naked panic This found a lot of panics in the new file openssl_aead.c. --- test.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.bash b/test.bash index 99884a8..b0faf68 100755 --- a/test.bash +++ b/test.bash @@ -79,7 +79,7 @@ else rm -Rf "$TESTDIR" fi -if grep -R "panic(" ./*.go internal ; then +if find internal -type f -name \*.go -print0 | xargs -0 grep "panic("; then echo "$MYNAME: Please use log.Panic instead of naked panic!" exit 1 fi From 8f820c429d88f601ba6c7a614fef23b7f53eb489 Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Sat, 4 Sep 2021 11:58:43 +0200 Subject: [PATCH 29/59] stupidgcm: fix without_openssl build $ ./build-without-openssl.bash internal/speed/speed.go:152:14: undefined: stupidgcm.NewXchacha20poly1305 --- internal/stupidgcm/openssl.go | 2 ++ internal/stupidgcm/without_openssl.go | 28 +++++---------------------- internal/stupidgcm/xchacha.go | 2 ++ 3 files changed, 9 insertions(+), 23 deletions(-) diff --git a/internal/stupidgcm/openssl.go b/internal/stupidgcm/openssl.go index d57d100..8e1357b 100644 --- a/internal/stupidgcm/openssl.go +++ b/internal/stupidgcm/openssl.go @@ -1,3 +1,5 @@ +// +build !without_openssl + package stupidgcm import ( diff --git a/internal/stupidgcm/without_openssl.go b/internal/stupidgcm/without_openssl.go index 4a14e73..81bae07 100644 --- a/internal/stupidgcm/without_openssl.go +++ b/internal/stupidgcm/without_openssl.go @@ -6,6 +6,8 @@ import ( "fmt" "os" + "crypto/cipher" + "github.com/rfjakob/gocryptfs/v2/internal/exitcodes" ) @@ -21,32 +23,12 @@ func errExit() { os.Exit(exitcodes.OpenSSL) } -func New(_ []byte, _ bool) *StupidGCM { - errExit() - // Never reached - return &StupidGCM{} -} - -func (g *StupidGCM) NonceSize() int { - errExit() - return -1 -} - -func (g *StupidGCM) Overhead() int { - errExit() - return -1 -} - -func (g *StupidGCM) Seal(_, _, _, _ []byte) []byte { +func New(_ []byte, _ bool) cipher.AEAD { errExit() return nil } -func (g *StupidGCM) Open(_, _, _, _ []byte) ([]byte, error) { - errExit() - return nil, nil -} - -func (g *StupidGCM) Wipe() { +func NewXchacha20poly1305(_ []byte) cipher.AEAD { errExit() + return nil } diff --git a/internal/stupidgcm/xchacha.go b/internal/stupidgcm/xchacha.go index d8668dc..eec8852 100644 --- a/internal/stupidgcm/xchacha.go +++ b/internal/stupidgcm/xchacha.go @@ -1,3 +1,5 @@ +// +build !without_openssl + // Copyright 2018 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. From b8c56ccffc7d2512cb33434a4fb86467fea1e36f Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Sat, 4 Sep 2021 12:01:50 +0200 Subject: [PATCH 30/59] stupidgcm: replace naked panics --- internal/stupidgcm/openssl.go | 4 ++-- internal/stupidgcm/xchacha.go | 15 ++++++++------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/internal/stupidgcm/openssl.go b/internal/stupidgcm/openssl.go index 8e1357b..cb9845e 100644 --- a/internal/stupidgcm/openssl.go +++ b/internal/stupidgcm/openssl.go @@ -15,7 +15,7 @@ import "C" func openSSLSeal(a *stupidAEADCommon, dst, iv, in, authData []byte) []byte { if a.Wiped() { - panic("BUG: tried to use wiped key") + log.Panic("BUG: tried to use wiped key") } if len(iv) != a.NonceSize() { log.Panicf("Only %d-byte IVs are supported, you passed %d bytes", a.NonceSize(), len(iv)) @@ -59,7 +59,7 @@ func openSSLSeal(a *stupidAEADCommon, dst, iv, in, authData []byte) []byte { func openSSLOpen(a *stupidAEADCommon, dst, iv, in, authData []byte) ([]byte, error) { if a.Wiped() { - panic("BUG: tried to use wiped key") + log.Panic("BUG: tried to use wiped key") } if len(iv) != a.NonceSize() { log.Panicf("Only %d-byte IVs are supported, you passed %d bytes", a.NonceSize(), len(iv)) diff --git a/internal/stupidgcm/xchacha.go b/internal/stupidgcm/xchacha.go index eec8852..3bf3b5b 100644 --- a/internal/stupidgcm/xchacha.go +++ b/internal/stupidgcm/xchacha.go @@ -13,6 +13,7 @@ package stupidgcm import ( "crypto/cipher" "errors" + "log" "golang.org/x/crypto/chacha20" "golang.org/x/crypto/chacha20poly1305" @@ -33,7 +34,7 @@ type stupidXchacha20poly1305 struct { // nonces are randomly generated. func NewXchacha20poly1305(key []byte) cipher.AEAD { if len(key) != chacha20poly1305.KeySize { - panic("bad key length") + log.Panic("bad key length") } ret := new(stupidXchacha20poly1305) copy(ret.key[:], key) @@ -50,10 +51,10 @@ func (*stupidXchacha20poly1305) Overhead() int { func (x *stupidXchacha20poly1305) Seal(dst, nonce, plaintext, additionalData []byte) []byte { if x.wiped { - panic("BUG: tried to use wiped key") + log.Panic("BUG: tried to use wiped key") } if len(nonce) != chacha20poly1305.NonceSizeX { - panic("bad nonce length passed to Seal") + log.Panic("bad nonce length passed to Seal") } // XChaCha20-Poly1305 technically supports a 64-bit counter, so there is no @@ -62,7 +63,7 @@ func (x *stupidXchacha20poly1305) Seal(dst, nonce, plaintext, additionalData []b // an issue because the cipher.AEAD API requires the entire message to be in // memory, and the counter overflows at 256 GB. if uint64(len(plaintext)) > (1<<38)-64 { - panic("plaintext too large") + log.Panic("plaintext too large") } hKey, _ := chacha20.HChaCha20(x.key[:], nonce[0:16]) @@ -78,16 +79,16 @@ func (x *stupidXchacha20poly1305) Seal(dst, nonce, plaintext, additionalData []b func (x *stupidXchacha20poly1305) Open(dst, nonce, ciphertext, additionalData []byte) ([]byte, error) { if x.wiped { - panic("BUG: tried to use wiped key") + log.Panic("BUG: tried to use wiped key") } if len(nonce) != chacha20poly1305.NonceSizeX { - panic("bad nonce length passed to Open") + log.Panic("bad nonce length passed to Open") } if len(ciphertext) < 16 { return nil, errors.New("message too short") } if uint64(len(ciphertext)) > (1<<38)-48 { - panic("ciphertext too large") + log.Panic("ciphertext too large") } hKey, _ := chacha20.HChaCha20(x.key[:], nonce[0:16]) From a2eaa5e3d1d9dfd8cf840076c215d253f82abf4b Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Sat, 4 Sep 2021 19:18:42 +0200 Subject: [PATCH 31/59] speed: add BenchmarkStupidChacha gocryptfs/internal/speed$ go test -bench . goos: linux goarch: amd64 pkg: github.com/rfjakob/gocryptfs/v2/internal/speed cpu: Intel(R) Core(TM) i5-3470 CPU @ 3.20GHz BenchmarkStupidGCM-4 249396 4722 ns/op 867.50 MB/s BenchmarkStupidGCMDecrypt-4 257872 4616 ns/op 887.35 MB/s BenchmarkGoGCM-4 290952 4097 ns/op 999.83 MB/s BenchmarkGoGCMDecrypt-4 294106 4060 ns/op 1008.84 MB/s BenchmarkAESSIV-4 46520 25532 ns/op 160.42 MB/s BenchmarkAESSIVDecrypt-4 46974 25478 ns/op 160.76 MB/s BenchmarkXchacha-4 244108 4881 ns/op 839.14 MB/s BenchmarkXchachaDecrypt-4 249658 4786 ns/op 855.86 MB/s BenchmarkStupidXchacha-4 205339 5768 ns/op 710.11 MB/s BenchmarkStupidXchachaDecrypt-4 204577 5836 ns/op 701.84 MB/s BenchmarkStupidChacha-4 227510 5224 ns/op 784.06 MB/s BenchmarkStupidChachaDecrypt-4 222787 5359 ns/op 764.34 MB/s PASS ok github.com/rfjakob/gocryptfs/v2/internal/speed 15.328s --- go.sum | 2 -- internal/speed/speed_test.go | 8 ++++++++ internal/stupidgcm/chacha.go | 2 +- internal/stupidgcm/chacha_test.go | 2 +- internal/stupidgcm/xchacha.go | 4 ++-- 5 files changed, 12 insertions(+), 6 deletions(-) diff --git a/go.sum b/go.sum index 9131acc..be0be5f 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,5 @@ github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/hanwen/go-fuse/v2 v2.1.1-0.20210825070001-74a933d6e856 h1:rQb7H5igQ2oIeT+Ul1UtIsGhUiSGeCoyLg84otnHdXU= -github.com/hanwen/go-fuse/v2 v2.1.1-0.20210825070001-74a933d6e856/go.mod h1:B1nGE/6RBFyBRC1RRnf23UpwCdyJ31eukw34oAKukAc= github.com/hanwen/go-fuse/v2 v2.1.1-0.20210825171523-3ab5d95a30ae h1:4CB6T4YTUVvnro5ba8ju1QCbOuyGAeF3vvKlo50EJ4k= github.com/hanwen/go-fuse/v2 v2.1.1-0.20210825171523-3ab5d95a30ae/go.mod h1:B1nGE/6RBFyBRC1RRnf23UpwCdyJ31eukw34oAKukAc= github.com/jacobsa/crypto v0.0.0-20190317225127-9f44e2d11115 h1:YuDUUFNM21CAbyPOpOP8BicaTD/0klJEKt5p8yuw+uY= diff --git a/internal/speed/speed_test.go b/internal/speed/speed_test.go index 4d09148..e9bbc0d 100644 --- a/internal/speed/speed_test.go +++ b/internal/speed/speed_test.go @@ -74,3 +74,11 @@ func BenchmarkStupidXchacha(b *testing.B) { func BenchmarkStupidXchachaDecrypt(b *testing.B) { bDecrypt(b, stupidgcm.NewXchacha20poly1305(randBytes(32))) } + +func BenchmarkStupidChacha(b *testing.B) { + bEncrypt(b, stupidgcm.NewChacha20poly1305(randBytes(32))) +} + +func BenchmarkStupidChachaDecrypt(b *testing.B) { + bDecrypt(b, stupidgcm.NewChacha20poly1305(randBytes(32))) +} diff --git a/internal/stupidgcm/chacha.go b/internal/stupidgcm/chacha.go index 37f7e1f..2e6e6e6 100644 --- a/internal/stupidgcm/chacha.go +++ b/internal/stupidgcm/chacha.go @@ -21,7 +21,7 @@ type stupidChacha20poly1305 struct { // Verify that we satisfy the cipher.AEAD interface var _ cipher.AEAD = &stupidChacha20poly1305{} -func newChacha20poly1305(key []byte) *stupidChacha20poly1305 { +func NewChacha20poly1305(key []byte) *stupidChacha20poly1305 { if len(key) != chacha20poly1305.KeySize { log.Panicf("Only %d-byte keys are supported, you passed %d bytes", chacha20poly1305.KeySize, len(key)) } diff --git a/internal/stupidgcm/chacha_test.go b/internal/stupidgcm/chacha_test.go index 513b68f..5f803aa 100644 --- a/internal/stupidgcm/chacha_test.go +++ b/internal/stupidgcm/chacha_test.go @@ -10,7 +10,7 @@ import ( func TestStupidChacha20poly1305(t *testing.T) { key := randBytes(32) - c := newChacha20poly1305(key) + c := NewChacha20poly1305(key) ref, err := chacha20poly1305.New(key) if err != nil { t.Fatal(err) diff --git a/internal/stupidgcm/xchacha.go b/internal/stupidgcm/xchacha.go index 3bf3b5b..055b7f7 100644 --- a/internal/stupidgcm/xchacha.go +++ b/internal/stupidgcm/xchacha.go @@ -67,7 +67,7 @@ func (x *stupidXchacha20poly1305) Seal(dst, nonce, plaintext, additionalData []b } hKey, _ := chacha20.HChaCha20(x.key[:], nonce[0:16]) - c := newChacha20poly1305(hKey) + c := NewChacha20poly1305(hKey) defer c.Wipe() // The first 4 bytes of the final nonce are unused counter space. @@ -92,7 +92,7 @@ func (x *stupidXchacha20poly1305) Open(dst, nonce, ciphertext, additionalData [] } hKey, _ := chacha20.HChaCha20(x.key[:], nonce[0:16]) - c := newChacha20poly1305(hKey) + c := NewChacha20poly1305(hKey) defer c.Wipe() // The first 4 bytes of the final nonce are unused counter space. From 6a0206897c83e1f3e4539d6a6c77149167f49626 Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Sun, 5 Sep 2021 12:17:38 +0200 Subject: [PATCH 32/59] stupidgcm: add BenchmarkCCall gocryptfs/internal/stupidgcm$ go test -bench . goos: linux goarch: amd64 pkg: github.com/rfjakob/gocryptfs/v2/internal/stupidgcm cpu: Intel(R) Core(TM) i5-3470 CPU @ 3.20GHz BenchmarkCCall-4 15864030 78.60 ns/op PASS ok github.com/rfjakob/gocryptfs/v2/internal/stupidgcm 1.898s --- internal/stupidgcm/common_test.go | 21 +++++++++++++++++++++ internal/stupidgcm/openssl.go | 6 ++++++ internal/stupidgcm/openssl_aead.c | 5 +++++ internal/stupidgcm/openssl_aead.h | 2 ++ 4 files changed, 34 insertions(+) diff --git a/internal/stupidgcm/common_test.go b/internal/stupidgcm/common_test.go index a8080ca..589d974 100644 --- a/internal/stupidgcm/common_test.go +++ b/internal/stupidgcm/common_test.go @@ -208,3 +208,24 @@ func randBytes(n int) []byte { } return b } + +/* +BenchmarkCCall benchmarks the overhead of calling from Go into C. +Looks like things improved a bit compared to +https://www.cockroachlabs.com/blog/the-cost-and-complexity-of-cgo/ +where they measured 171ns/op: + +$ go test -bench . +goos: linux +goarch: amd64 +pkg: github.com/rfjakob/gocryptfs/v2/internal/stupidgcm +cpu: Intel(R) Core(TM) i5-3470 CPU @ 3.20GHz +BenchmarkCCall-4 13989364 76.72 ns/op +PASS +ok github.com/rfjakob/gocryptfs/v2/internal/stupidgcm 1.735s +*/ +func BenchmarkCCall(b *testing.B) { + for i := 0; i < b.N; i++ { + noopCFunction() + } +} diff --git a/internal/stupidgcm/openssl.go b/internal/stupidgcm/openssl.go index cb9845e..82ec0a1 100644 --- a/internal/stupidgcm/openssl.go +++ b/internal/stupidgcm/openssl.go @@ -108,3 +108,9 @@ func openSSLOpen(a *stupidAEADCommon, dst, iv, in, authData []byte) ([]byte, err } return append(dst, buf...), nil } + +// This functions exists to benchmark the C call overhead from Go. +// See BenchmarkCCall for resuts. +func noopCFunction() { + C.noop_c_function() +} diff --git a/internal/stupidgcm/openssl_aead.c b/internal/stupidgcm/openssl_aead.c index 9dc6866..77898d5 100644 --- a/internal/stupidgcm/openssl_aead.c +++ b/internal/stupidgcm/openssl_aead.c @@ -176,3 +176,8 @@ int openssl_aead_open( return plaintextLen; } + +// This functions exists to benchmark the C call overhead from Go. +void noop_c_function(void) { + return; +} diff --git a/internal/stupidgcm/openssl_aead.h b/internal/stupidgcm/openssl_aead.h index 6a818b6..820beeb 100644 --- a/internal/stupidgcm/openssl_aead.h +++ b/internal/stupidgcm/openssl_aead.h @@ -27,3 +27,5 @@ int openssl_aead_open( const int ivLen, unsigned char* const plaintext, const int plaintextBufLen); + +void noop_c_function(void); From f89b14ee3dfa944e275961bf588d0e49500579c4 Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Tue, 7 Sep 2021 12:34:47 +0200 Subject: [PATCH 33/59] stupidgcm: cache C.EVP_chacha20_poly1305() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 2% performance improvement, almost for free. gocryptfs/internal/speed$ benchstat old new name old time/op new time/op delta StupidXchacha-4 5.82µs ± 0% 5.68µs ± 0% -2.37% (p=0.008 n=5+5) name old speed new speed delta StupidXchacha-4 704MB/s ± 0% 721MB/s ± 0% +2.43% (p=0.008 n=5+5) --- internal/stupidgcm/chacha.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/internal/stupidgcm/chacha.go b/internal/stupidgcm/chacha.go index 2e6e6e6..c90d721 100644 --- a/internal/stupidgcm/chacha.go +++ b/internal/stupidgcm/chacha.go @@ -21,6 +21,20 @@ type stupidChacha20poly1305 struct { // Verify that we satisfy the cipher.AEAD interface var _ cipher.AEAD = &stupidChacha20poly1305{} +// _EVP_chacha20_poly1305 caches C.EVP_chacha20_poly1305() to avoid the Cgo call +// overhead for each instantiation of NewChacha20poly1305. +var _EVP_chacha20_poly1305 *C.EVP_CIPHER + +func init() { + _EVP_chacha20_poly1305 = C.EVP_chacha20_poly1305() +} + +// NewChacha20poly1305 returns a new instance of the OpenSSL ChaCha20-Poly1305 AEAD +// cipher ( https://www.openssl.org/docs/man1.1.1/man3/EVP_chacha20_poly1305.html ). +// +// gocryptfs only uses ChaCha20-Poly1305 as a building block for OpenSSL +// XChaCha20-Poly1305. This function is hot because it gets called once for each +// block by XChaCha20-Poly1305. func NewChacha20poly1305(key []byte) *stupidChacha20poly1305 { if len(key) != chacha20poly1305.KeySize { log.Panicf("Only %d-byte keys are supported, you passed %d bytes", chacha20poly1305.KeySize, len(key)) @@ -28,7 +42,7 @@ func NewChacha20poly1305(key []byte) *stupidChacha20poly1305 { return &stupidChacha20poly1305{ stupidAEADCommon{ key: append([]byte{}, key...), // private copy - openSSLEVPCipher: C.EVP_chacha20_poly1305(), + openSSLEVPCipher: _EVP_chacha20_poly1305, nonceSize: chacha20poly1305.NonceSize, }, } From 39b1070506436dcf35ac83c7f8cee65067856062 Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Tue, 7 Sep 2021 12:41:58 +0200 Subject: [PATCH 34/59] stupidgcm: add testConcurrency Verifies that we don't corrupt data when called concurrently. --- internal/stupidgcm/common_test.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/internal/stupidgcm/common_test.go b/internal/stupidgcm/common_test.go index 589d974..519efb0 100644 --- a/internal/stupidgcm/common_test.go +++ b/internal/stupidgcm/common_test.go @@ -1,3 +1,5 @@ +// +build cgo,!without_openssl + package stupidgcm import ( @@ -6,6 +8,7 @@ import ( "crypto/rand" "encoding/hex" "log" + "sync" "testing" ) @@ -15,6 +18,7 @@ func testCiphers(t *testing.T, our cipher.AEAD, ref cipher.AEAD) { t.Run("testInplaceOpen", func(t *testing.T) { testInplaceOpen(t, our, ref) }) t.Run("testCorruption_c1", func(t *testing.T) { testCorruption(t, our) }) t.Run("testCorruption_c2", func(t *testing.T) { testCorruption(t, ref) }) + t.Run("testConcurrency", func(t *testing.T) { testConcurrency(t, our, ref) }) t.Run("testWipe", func(t *testing.T) { testWipe(t, our) }) } @@ -65,6 +69,25 @@ func testEncryptDecrypt(t *testing.T, c1 cipher.AEAD, c2 cipher.AEAD) { } } +// testConcurrency verifies that we don't corrupt data when called concurrently +func testConcurrency(t *testing.T, c1 cipher.AEAD, c2 cipher.AEAD) { + const loopCount = 2 + const goroutineCount = 4 + + for h := 0; h < loopCount; h++ { + var wg sync.WaitGroup + for i := 0; i < goroutineCount; i++ { + wg.Add(1) + go func() { + testEncryptDecrypt(t, c1, c2) + wg.Done() + }() + wg.Wait() + } + } +} + +// testInplaceSeal: // Seal re-uses the "dst" buffer it is large enough. // Check that this works correctly by testing different "dst" capacities from // 5000 to 16 and "in" lengths from 1 to 5000. From d9510d0c0ba2ef31145499f4d49ab1d10c03de94 Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Tue, 7 Sep 2021 12:44:42 +0200 Subject: [PATCH 35/59] stupidgcm: NewChacha20poly1305: avoid slice append MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I noticed that growslice() shows up in the cpuprofile. Avoiding slice append for the private jey copy gives a 0.6% speedup: gocryptfs/internal/speed$ benchstat old new name old time/op new time/op delta StupidXchacha-4 5.68µs ± 0% 5.65µs ± 0% -0.63% (p=0.008 n=5+5) name old speed new speed delta StupidXchacha-4 721MB/s ± 0% 725MB/s ± 0% +0.63% (p=0.008 n=5+5) --- internal/stupidgcm/chacha.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/stupidgcm/chacha.go b/internal/stupidgcm/chacha.go index c90d721..30d57e3 100644 --- a/internal/stupidgcm/chacha.go +++ b/internal/stupidgcm/chacha.go @@ -39,9 +39,12 @@ func NewChacha20poly1305(key []byte) *stupidChacha20poly1305 { if len(key) != chacha20poly1305.KeySize { log.Panicf("Only %d-byte keys are supported, you passed %d bytes", chacha20poly1305.KeySize, len(key)) } + // private copy + key2 := make([]byte, chacha20poly1305.KeySize) + copy(key2, key) return &stupidChacha20poly1305{ stupidAEADCommon{ - key: append([]byte{}, key...), // private copy + key: key2, openSSLEVPCipher: _EVP_chacha20_poly1305, nonceSize: chacha20poly1305.NonceSize, }, From 738d5a2b3a001318064be649d683684630eff4c2 Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Tue, 7 Sep 2021 12:57:52 +0200 Subject: [PATCH 36/59] stupidgcm: fix build with CGO_ENABLED=1 without_openssl We missed some "// +build" lines --- internal/stupidgcm/Makefile | 11 +++++++++++ internal/stupidgcm/common.go | 2 ++ internal/stupidgcm/openssl_aead.c | 2 ++ 3 files changed, 15 insertions(+) diff --git a/internal/stupidgcm/Makefile b/internal/stupidgcm/Makefile index 22e3b2e..143819d 100644 --- a/internal/stupidgcm/Makefile +++ b/internal/stupidgcm/Makefile @@ -1,3 +1,14 @@ +.PHONY: test +test: gcc + # All three ways of building this must work + go build + go build -tags without_openssl + CGO_ENABLED=0 go build -tags without_openssl + # Likewise, all three ways of testing this must work + go test -v + go test -v -tags without_openssl + CGO_ENABLED=0 go test -v -tags without_openssl + .PHONY: gcc gcc: gcc -Wall -Wextra -Wformat-security -Wconversion -lcrypto -c *.c diff --git a/internal/stupidgcm/common.go b/internal/stupidgcm/common.go index 3788315..bb100eb 100644 --- a/internal/stupidgcm/common.go +++ b/internal/stupidgcm/common.go @@ -1,3 +1,5 @@ +// +build !without_openssl + package stupidgcm import ( diff --git a/internal/stupidgcm/openssl_aead.c b/internal/stupidgcm/openssl_aead.c index 77898d5..e02466f 100644 --- a/internal/stupidgcm/openssl_aead.c +++ b/internal/stupidgcm/openssl_aead.c @@ -1,3 +1,5 @@ +// +build !without_openssl + #include "openssl_aead.h" #include #include From 3a80db953da93c741ad391ae124121459c1046b0 Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Tue, 7 Sep 2021 17:47:48 +0200 Subject: [PATCH 37/59] stupidgcm: allow zero-length input data We used to panic in this case because it is useless. But Go stdlib supports it, so we should as well. --- internal/stupidgcm/common_test.go | 40 ++++++++++++++++++++++++++----- internal/stupidgcm/openssl.go | 21 ++++++++++------ 2 files changed, 48 insertions(+), 13 deletions(-) diff --git a/internal/stupidgcm/common_test.go b/internal/stupidgcm/common_test.go index 519efb0..10a7ce1 100644 --- a/internal/stupidgcm/common_test.go +++ b/internal/stupidgcm/common_test.go @@ -19,6 +19,8 @@ func testCiphers(t *testing.T, our cipher.AEAD, ref cipher.AEAD) { t.Run("testCorruption_c1", func(t *testing.T) { testCorruption(t, our) }) t.Run("testCorruption_c2", func(t *testing.T) { testCorruption(t, ref) }) t.Run("testConcurrency", func(t *testing.T) { testConcurrency(t, our, ref) }) + t.Run("testOpenAllZero_our", func(t *testing.T) { testOpenAllZero(t, our) }) + t.Run("testOpenAllZero_ref", func(t *testing.T) { testOpenAllZero(t, ref) }) t.Run("testWipe", func(t *testing.T) { testWipe(t, our) }) } @@ -37,9 +39,13 @@ func testEncryptDecrypt(t *testing.T, c1 cipher.AEAD, c2 cipher.AEAD) { dst := make([]byte, 71) // 71 = arbitrary length - // Check all block sizes from 1 to 5000 - for i := 1; i < 5000; i++ { - in := make([]byte, i) + // Check all block sizes from nil to 0 to 5000 + for i := -1; i < 5000; i++ { + // stays nil at i == -1 + var in []byte + if i >= 0 { + in = make([]byte, i) + } c1out := c1.Seal(dst, iv, in, authData) c2out := c2.Seal(dst, iv, in, authData) @@ -79,8 +85,8 @@ func testConcurrency(t *testing.T, c1 cipher.AEAD, c2 cipher.AEAD) { for i := 0; i < goroutineCount; i++ { wg.Add(1) go func() { + defer wg.Done() testEncryptDecrypt(t, c1, c2) - wg.Done() }() wg.Wait() } @@ -96,8 +102,9 @@ func testInplaceSeal(t *testing.T, c1 cipher.AEAD, c2 cipher.AEAD) { iv := randBytes(c1.NonceSize()) max := 5016 - // Check all block sizes from 1 to 5000 - for i := 1; i < max-16; i++ { + + // Check all block sizes from 0 to 5000 + for i := 0; i < max-16; i++ { in := make([]byte, i) dst := make([]byte, max-i) dst = dst[:16] @@ -185,6 +192,27 @@ func testCorruption(t *testing.T, c cipher.AEAD) { } } +// testOpenAllZero tests that we do not crash for nil or any block size of zeros +func testOpenAllZero(t *testing.T, c cipher.AEAD) { + authData := make([]byte, 24) + iv := make([]byte, c.NonceSize()) + + for i := -1; i < 5000; i++ { + // stays nil at i == -1 + var cipher []byte + if i >= 0 { + cipher = make([]byte, i) + } + plain, err := c.Open(nil, iv, cipher, authData) + if err == nil { + t.Error("should have gotten error, but did not") + } + if len(plain) > 0 { + t.Errorf("should not have received data, but got %d bytes", len(plain)) + } + } +} + func testWipe(t *testing.T, c cipher.AEAD) { switch c2 := c.(type) { case *StupidGCM: diff --git a/internal/stupidgcm/openssl.go b/internal/stupidgcm/openssl.go index 82ec0a1..ae0ee5c 100644 --- a/internal/stupidgcm/openssl.go +++ b/internal/stupidgcm/openssl.go @@ -20,9 +20,6 @@ func openSSLSeal(a *stupidAEADCommon, dst, iv, in, authData []byte) []byte { if len(iv) != a.NonceSize() { log.Panicf("Only %d-byte IVs are supported, you passed %d bytes", a.NonceSize(), len(iv)) } - if len(in) == 0 { - log.Panic("Zero-length input data is not supported") - } // If the "dst" slice is large enough we can use it as our output buffer outLen := len(in) + tagLen @@ -36,7 +33,7 @@ func openSSLSeal(a *stupidAEADCommon, dst, iv, in, authData []byte) []byte { } res := int(C.openssl_aead_seal(a.openSSLEVPCipher, - (*C.uchar)(&in[0]), + slicePointerOrNull(in), C.int(len(in)), (*C.uchar)(&authData[0]), C.int(len(authData)), @@ -64,7 +61,7 @@ func openSSLOpen(a *stupidAEADCommon, dst, iv, in, authData []byte) ([]byte, err if len(iv) != a.NonceSize() { log.Panicf("Only %d-byte IVs are supported, you passed %d bytes", a.NonceSize(), len(iv)) } - if len(in) <= tagLen { + if len(in) < tagLen { return nil, fmt.Errorf("stupidChacha20poly1305: input data too short (%d bytes)", len(in)) } @@ -83,7 +80,7 @@ func openSSLOpen(a *stupidAEADCommon, dst, iv, in, authData []byte) ([]byte, err tag := in[len(in)-tagLen:] res := int(C.openssl_aead_open(a.openSSLEVPCipher, - (*C.uchar)(&ciphertext[0]), + slicePointerOrNull(ciphertext), C.int(len(ciphertext)), (*C.uchar)(&authData[0]), C.int(len(authData)), @@ -93,7 +90,7 @@ func openSSLOpen(a *stupidAEADCommon, dst, iv, in, authData []byte) ([]byte, err C.int(len(a.key)), (*C.uchar)(&iv[0]), C.int(len(iv)), - (*C.uchar)(&buf[0]), + slicePointerOrNull(buf), C.int(len(buf)))) if res < 0 { @@ -109,6 +106,16 @@ func openSSLOpen(a *stupidAEADCommon, dst, iv, in, authData []byte) ([]byte, err return append(dst, buf...), nil } +// slicePointerOrNull returns a C pointer to the beginning of the byte slice, +// or NULL if the byte slice is empty. This is useful for slices that can be +// empty, otherwise you can directly use "(*C.uchar)(&s[0])". +func slicePointerOrNull(s []byte) (ptr *C.uchar) { + if len(s) == 0 { + return + } + return (*C.uchar)(&s[0]) +} + // This functions exists to benchmark the C call overhead from Go. // See BenchmarkCCall for resuts. func noopCFunction() { From d598536709db355366e90870c6df3508c71c5884 Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Tue, 7 Sep 2021 17:48:55 +0200 Subject: [PATCH 38/59] stupidgcm: unexport stupidGCM struct No need to have it exported. --- internal/stupidgcm/common_test.go | 2 +- internal/stupidgcm/gcm.go | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/internal/stupidgcm/common_test.go b/internal/stupidgcm/common_test.go index 10a7ce1..1b32cfa 100644 --- a/internal/stupidgcm/common_test.go +++ b/internal/stupidgcm/common_test.go @@ -215,7 +215,7 @@ func testOpenAllZero(t *testing.T, c cipher.AEAD) { func testWipe(t *testing.T, c cipher.AEAD) { switch c2 := c.(type) { - case *StupidGCM: + case *stupidGCM: c2.Wipe() if !c2.Wiped() { t.Error("c2.wiped is not set") diff --git a/internal/stupidgcm/gcm.go b/internal/stupidgcm/gcm.go index 439e7a7..0cadd3c 100644 --- a/internal/stupidgcm/gcm.go +++ b/internal/stupidgcm/gcm.go @@ -21,20 +21,19 @@ const ( tagLen = 16 ) -// StupidGCM implements the cipher.AEAD interface -type StupidGCM struct { +type stupidGCM struct { stupidAEADCommon } // Verify that we satisfy the interface -var _ cipher.AEAD = &StupidGCM{} +var _ cipher.AEAD = &stupidGCM{} // New returns a new cipher.AEAD implementation.. func New(keyIn []byte, forceDecode bool) cipher.AEAD { if len(keyIn) != keyLen { log.Panicf("Only %d-byte keys are supported", keyLen) } - return &StupidGCM{ + return &stupidGCM{ stupidAEADCommon{ // Create a private copy of the key key: append([]byte{}, keyIn...), From f47e287c202ba92e9b48e65bf95e0771c6c4997c Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Tue, 7 Sep 2021 17:58:42 +0200 Subject: [PATCH 39/59] stupidgcm: revamp package documentation Maybe interesting for people following https://github.com/rfjakob/gocryptfs/issues/452 --- internal/stupidgcm/chacha.go | 2 ++ internal/stupidgcm/doc.go | 59 +++++++++++++++++++++++++++++++++++ internal/stupidgcm/gcm.go | 9 ++---- internal/stupidgcm/xchacha.go | 5 ++- 4 files changed, 68 insertions(+), 7 deletions(-) create mode 100644 internal/stupidgcm/doc.go diff --git a/internal/stupidgcm/chacha.go b/internal/stupidgcm/chacha.go index 30d57e3..1117d97 100644 --- a/internal/stupidgcm/chacha.go +++ b/internal/stupidgcm/chacha.go @@ -35,6 +35,8 @@ func init() { // gocryptfs only uses ChaCha20-Poly1305 as a building block for OpenSSL // XChaCha20-Poly1305. This function is hot because it gets called once for each // block by XChaCha20-Poly1305. +// +// Only 32-bytes keys and 12-byte IVs are supported. func NewChacha20poly1305(key []byte) *stupidChacha20poly1305 { if len(key) != chacha20poly1305.KeySize { log.Panicf("Only %d-byte keys are supported, you passed %d bytes", chacha20poly1305.KeySize, len(key)) diff --git a/internal/stupidgcm/doc.go b/internal/stupidgcm/doc.go new file mode 100644 index 0000000..36c189b --- /dev/null +++ b/internal/stupidgcm/doc.go @@ -0,0 +1,59 @@ +// Package stupidgcm wraps OpenSSL to provide a cipher.AEAD interface for +// authenticated encryption algorithms. +// +// The supported algorithms are: +// +// (1) AES-GCM-256 (OpenSSL EVP_aes_256_gcm) +// +// (2) ChaCha20-Poly1305 (OpenSSL EVP_chacha20_poly1305) +// +// (3) XChaCha20-Poly1305 (OpenSSL EVP_chacha20_poly1305 + Go HChaCha20) +// +// The golang.org/x/crypto libraries provides implementations for all algorithms, +// and the test suite verifies that the implementation in this package gives +// the exact same results. +// +// However, OpenSSL has optimized assembly for almost all platforms, which Go +// does not. Example for a 32-bit ARM device (Odroid XU4): +// +// $ gocrypts -speed +// gocryptfs v2.1-68-gedf9d4c.stupidchacha; go-fuse v2.1.1-0.20210825171523-3ab5d95a30ae; 2021-09-04 go1.16.7 linux/arm +// AES-GCM-256-OpenSSL 56.84 MB/s (selected in auto mode) +// AES-GCM-256-Go 16.61 MB/s +// AES-SIV-512-Go 16.49 MB/s +// XChaCha20-Poly1305-Go 39.08 MB/s (use via -xchacha flag) +// XChaCha20-Poly1305-OpenSSL 141.82 MB/s +// +// This package is "stupid" in the sense that it only supports a narrow set of +// key- and iv-lengths, and panics if it does not like what you pass it. +// See the constructor functions for which restrictions apply for each algorithm. +// Also, it is only tested for block lengths up to 5000 bytes, because this is +// what gocryptfs uses. +// +// Corrupt ciphertexts never cause a panic. Instead, ErrAuth is returned on +// decryption. +// +// XChaCha20-Poly1305 +// +// The XChaCha20-Poly1305 implementation is more complicated than the others, +// because OpenSSL does not support XChaCha20-Poly1305 directly. Follow +// https://github.com/openssl/openssl/issues/5523 to get notified when it is +// accepted into OpenSSL. +// +// Fortunately, XChaCha20-Poly1305 is just ChaCha20-Poly1305 with some key+iv +// mixing using HChaCha20 in front: +// +// key (32 bytes), iv (24 bytes) +// | +// v +// HChaCha20 (provided by golang.org/x/crypto/chacha20) +// | +// v +// key2 (32 bytes), iv2 (16 bytes) +// | +// v +// ChaCha20-Poly1305 (OpenSSL EVP_chacha20_poly1305) +// +// As HChaCha20 is very fast, XChaCha20-Poly1305 gets almost the same throughput +// as ChaCha20-Poly1305 (for 4kiB blocks). +package stupidgcm diff --git a/internal/stupidgcm/gcm.go b/internal/stupidgcm/gcm.go index 0cadd3c..a9377b1 100644 --- a/internal/stupidgcm/gcm.go +++ b/internal/stupidgcm/gcm.go @@ -1,7 +1,5 @@ // +build !without_openssl -// Package stupidgcm is a thin wrapper for OpenSSL's GCM encryption and -// decryption functions. It only support 32-byte keys and 16-bit IVs. package stupidgcm // #include @@ -25,10 +23,9 @@ type stupidGCM struct { stupidAEADCommon } -// Verify that we satisfy the interface -var _ cipher.AEAD = &stupidGCM{} - -// New returns a new cipher.AEAD implementation.. +// New returns a new AES-GCM-256 cipher that satisfies the cipher.AEAD interface. +// +// Only 32-bytes keys and 16-byte IVs are supported. func New(keyIn []byte, forceDecode bool) cipher.AEAD { if len(keyIn) != keyLen { log.Panicf("Only %d-byte keys are supported", keyLen) diff --git a/internal/stupidgcm/xchacha.go b/internal/stupidgcm/xchacha.go index 055b7f7..deb6e2f 100644 --- a/internal/stupidgcm/xchacha.go +++ b/internal/stupidgcm/xchacha.go @@ -26,12 +26,15 @@ type stupidXchacha20poly1305 struct { wiped bool } -// NewXchacha20poly1305 returns a XChaCha20-Poly1305 AEAD that uses the given 256-bit key. +// NewXchacha20poly1305 returns a XChaCha20-Poly1305 cipher that satisfied the +// cipher.AEAD interface. // // XChaCha20-Poly1305 is a ChaCha20-Poly1305 variant that takes a longer nonce, // suitable to be generated randomly without risk of collisions. It should be // preferred when nonce uniqueness cannot be trivially ensured, or whenever // nonces are randomly generated. +// +// Only 32-bytes keys and 24-byte IVs are supported. func NewXchacha20poly1305(key []byte) cipher.AEAD { if len(key) != chacha20poly1305.KeySize { log.Panic("bad key length") From 85c2beccaf674c69b3e30b0d646f5c11d91ecb9b Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Tue, 7 Sep 2021 18:11:11 +0200 Subject: [PATCH 40/59] stupidgcm: normalize constructor naming New() -> NewAES256GCM() Also add missing NewChacha20poly1305 constructor in without_openssl.go. --- internal/cryptocore/cryptocore.go | 2 +- internal/speed/speed.go | 2 +- internal/speed/speed_test.go | 2 +- internal/stupidgcm/chacha.go | 2 +- internal/stupidgcm/gcm.go | 4 ++-- internal/stupidgcm/gcm_test.go | 2 +- internal/stupidgcm/without_openssl.go | 11 +++++++---- internal/stupidgcm/xchacha.go | 4 ++-- 8 files changed, 16 insertions(+), 13 deletions(-) diff --git a/internal/cryptocore/cryptocore.go b/internal/cryptocore/cryptocore.go index 1b692ff..d7b7527 100644 --- a/internal/cryptocore/cryptocore.go +++ b/internal/cryptocore/cryptocore.go @@ -120,7 +120,7 @@ func New(key []byte, aeadType AEADTypeEnum, IVBitLen int, useHKDF bool, forceDec if IVBitLen != 128 { log.Panicf("stupidgcm only supports 128-bit IVs, you wanted %d", IVBitLen) } - aeadCipher = stupidgcm.New(gcmKey, forceDecode) + aeadCipher = stupidgcm.NewAES256GCM(gcmKey, forceDecode) case BackendGoGCM: goGcmBlockCipher, err := aes.NewCipher(gcmKey) if err != nil { diff --git a/internal/speed/speed.go b/internal/speed/speed.go index a696703..2f818d0 100644 --- a/internal/speed/speed.go +++ b/internal/speed/speed.go @@ -116,7 +116,7 @@ func bStupidGCM(b *testing.B) { if stupidgcm.BuiltWithoutOpenssl { b.Skip("openssl has been disabled at compile-time") } - bEncrypt(b, stupidgcm.New(randBytes(32), false)) + bEncrypt(b, stupidgcm.NewAES256GCM(randBytes(32), false)) } // bGoGCM benchmarks Go stdlib GCM diff --git a/internal/speed/speed_test.go b/internal/speed/speed_test.go index e9bbc0d..11c68d0 100644 --- a/internal/speed/speed_test.go +++ b/internal/speed/speed_test.go @@ -31,7 +31,7 @@ func BenchmarkStupidGCMDecrypt(b *testing.B) { if stupidgcm.BuiltWithoutOpenssl { b.Skip("openssl has been disabled at compile-time") } - bDecrypt(b, stupidgcm.New(randBytes(32), false)) + bDecrypt(b, stupidgcm.NewAES256GCM(randBytes(32), false)) } func BenchmarkGoGCM(b *testing.B) { diff --git a/internal/stupidgcm/chacha.go b/internal/stupidgcm/chacha.go index 1117d97..e09ed0b 100644 --- a/internal/stupidgcm/chacha.go +++ b/internal/stupidgcm/chacha.go @@ -37,7 +37,7 @@ func init() { // block by XChaCha20-Poly1305. // // Only 32-bytes keys and 12-byte IVs are supported. -func NewChacha20poly1305(key []byte) *stupidChacha20poly1305 { +func NewChacha20poly1305(key []byte) cipher.AEAD { if len(key) != chacha20poly1305.KeySize { log.Panicf("Only %d-byte keys are supported, you passed %d bytes", chacha20poly1305.KeySize, len(key)) } diff --git a/internal/stupidgcm/gcm.go b/internal/stupidgcm/gcm.go index a9377b1..c38dd5f 100644 --- a/internal/stupidgcm/gcm.go +++ b/internal/stupidgcm/gcm.go @@ -23,10 +23,10 @@ type stupidGCM struct { stupidAEADCommon } -// New returns a new AES-GCM-256 cipher that satisfies the cipher.AEAD interface. +// NewAES256GCM returns a new AES-256-GCM cipher that satisfies the cipher.AEAD interface. // // Only 32-bytes keys and 16-byte IVs are supported. -func New(keyIn []byte, forceDecode bool) cipher.AEAD { +func NewAES256GCM(keyIn []byte, forceDecode bool) cipher.AEAD { if len(keyIn) != keyLen { log.Panicf("Only %d-byte keys are supported", keyLen) } diff --git a/internal/stupidgcm/gcm_test.go b/internal/stupidgcm/gcm_test.go index 5323afa..b587e58 100644 --- a/internal/stupidgcm/gcm_test.go +++ b/internal/stupidgcm/gcm_test.go @@ -13,7 +13,7 @@ import ( func TestStupidGCM(t *testing.T) { key := randBytes(32) - sGCM := New(key, false) + sGCM := NewAES256GCM(key, false) gAES, err := aes.NewCipher(key) if err != nil { diff --git a/internal/stupidgcm/without_openssl.go b/internal/stupidgcm/without_openssl.go index 81bae07..93efcb4 100644 --- a/internal/stupidgcm/without_openssl.go +++ b/internal/stupidgcm/without_openssl.go @@ -11,19 +11,22 @@ import ( "github.com/rfjakob/gocryptfs/v2/internal/exitcodes" ) -type StupidGCM struct{} - const ( // BuiltWithoutOpenssl indicates if openssl been disabled at compile-time BuiltWithoutOpenssl = true ) func errExit() { - fmt.Fprintln(os.Stderr, "gocryptfs has been compiled without openssl support but you are still trying to use openssl") + fmt.Fprintln(os.Stderr, "I have been compiled without openssl support but you are still trying to use openssl") os.Exit(exitcodes.OpenSSL) } -func New(_ []byte, _ bool) cipher.AEAD { +func NewAES256GCM(_ []byte, _ bool) cipher.AEAD { + errExit() + return nil +} + +func NewChacha20poly1305(_ []byte) cipher.AEAD { errExit() return nil } diff --git a/internal/stupidgcm/xchacha.go b/internal/stupidgcm/xchacha.go index deb6e2f..ca740e4 100644 --- a/internal/stupidgcm/xchacha.go +++ b/internal/stupidgcm/xchacha.go @@ -70,7 +70,7 @@ func (x *stupidXchacha20poly1305) Seal(dst, nonce, plaintext, additionalData []b } hKey, _ := chacha20.HChaCha20(x.key[:], nonce[0:16]) - c := NewChacha20poly1305(hKey) + c := NewChacha20poly1305(hKey).(*stupidChacha20poly1305) defer c.Wipe() // The first 4 bytes of the final nonce are unused counter space. @@ -95,7 +95,7 @@ func (x *stupidXchacha20poly1305) Open(dst, nonce, ciphertext, additionalData [] } hKey, _ := chacha20.HChaCha20(x.key[:], nonce[0:16]) - c := NewChacha20poly1305(hKey) + c := NewChacha20poly1305(hKey).(*stupidChacha20poly1305) defer c.Wipe() // The first 4 bytes of the final nonce are unused counter space. From 1a5866729387c09eca1cdc9737d1b02c74c25901 Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Wed, 8 Sep 2021 19:48:13 +0200 Subject: [PATCH 41/59] stupidgcm: add PreferOpenSSL{AES256GCM,Xchacha20poly1305} Add PreferOpenSSLXchacha20poly1305, rename PreferOpenSSL -> PreferOpenSSLAES256GCM. --- cli_args.go | 2 +- cli_args_test.go | 2 +- internal/speed/speed.go | 4 ++-- internal/stupidgcm/prefer.go | 20 ++++++++++++++++++-- 4 files changed, 22 insertions(+), 6 deletions(-) diff --git a/cli_args.go b/cli_args.go index 0287741..7175006 100644 --- a/cli_args.go +++ b/cli_args.go @@ -253,7 +253,7 @@ func parseCliOpts(osArgs []string) (args argContainer) { } // "-openssl" needs some post-processing if opensslAuto == "auto" { - args.openssl = stupidgcm.PreferOpenSSL() + args.openssl = stupidgcm.PreferOpenSSLAES256GCM() } else { args.openssl, err = strconv.ParseBool(opensslAuto) if err != nil { diff --git a/cli_args_test.go b/cli_args_test.go index 9bb9654..6c923f4 100644 --- a/cli_args_test.go +++ b/cli_args_test.go @@ -119,7 +119,7 @@ func TestParseCliOpts(t *testing.T) { longnames: true, raw64: true, hkdf: true, - openssl: stupidgcm.PreferOpenSSL(), // depends on CPU and build flags + openssl: stupidgcm.PreferOpenSSLAES256GCM(), // depends on CPU and build flags scryptn: 16, } diff --git a/internal/speed/speed.go b/internal/speed/speed.go index 2f818d0..d5aa696 100644 --- a/internal/speed/speed.go +++ b/internal/speed/speed.go @@ -32,8 +32,8 @@ func Run() { f func(*testing.B) preferred bool }{ - {name: cryptocore.BackendOpenSSL.Name, f: bStupidGCM, preferred: stupidgcm.PreferOpenSSL()}, - {name: cryptocore.BackendGoGCM.Name, f: bGoGCM, preferred: !stupidgcm.PreferOpenSSL()}, + {name: cryptocore.BackendOpenSSL.Name, f: bStupidGCM, preferred: stupidgcm.PreferOpenSSLAES256GCM()}, + {name: cryptocore.BackendGoGCM.Name, f: bGoGCM, preferred: !stupidgcm.PreferOpenSSLAES256GCM()}, {name: cryptocore.BackendAESSIV.Name, f: bAESSIV, preferred: false}, {name: cryptocore.BackendXChaCha20Poly1305.Name, f: bXchacha20poly1305, preferred: false}, {name: cryptocore.BackendXChaCha20Poly1305OpenSSL.Name, f: bStupidXchacha, preferred: false}, diff --git a/internal/stupidgcm/prefer.go b/internal/stupidgcm/prefer.go index bacd56a..ffbaf58 100644 --- a/internal/stupidgcm/prefer.go +++ b/internal/stupidgcm/prefer.go @@ -6,7 +6,8 @@ import ( "golang.org/x/sys/cpu" ) -// PreferOpenSSL tells us if OpenSSL is faster than Go GCM on this machine. +// PreferOpenSSLAES256GCM tells us if OpenSSL AES-256-GCM is faster than Go stdlib +// on this machine. // // Go GCM is only faster if the CPU either: // @@ -16,7 +17,7 @@ import ( // // See https://github.com/rfjakob/gocryptfs/wiki/CPU-Benchmarks // for benchmarks. -func PreferOpenSSL() bool { +func PreferOpenSSLAES256GCM() bool { if BuiltWithoutOpenssl { return false } @@ -33,3 +34,18 @@ func PreferOpenSSL() bool { // OpenSSL is probably faster return true } + +// PreferOpenSSLXchacha20poly1305 returns true if OpenSSL Xchacha20poly1305 is +// faster than Go stdlib on this machine. +func PreferOpenSSLXchacha20poly1305() bool { + if BuiltWithoutOpenssl { + return false + } + // Go x/crypto has optimized assembly for amd64: + // https://github.com/golang/crypto/blob/master/chacha20poly1305/chacha20poly1305_amd64.s + if runtime.GOARCH == "amd64" { + return false + } + // On arm64 and arm, OpenSSL is faster. Probably everwhere else too. + return true +} From 94e8004b6ce497dafd13e8c3f6f6596b49169970 Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Wed, 8 Sep 2021 20:32:16 +0200 Subject: [PATCH 42/59] Make -openssl also apply to xchacha Now that stupidgcm supports xchacha, make it available on mount. --- cli_args.go | 6 +++++- internal/cryptocore/cryptocore.go | 16 ++++++++++++---- mount.go | 17 ++++++++++++++--- 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/cli_args.go b/cli_args.go index 7175006..d666b47 100644 --- a/cli_args.go +++ b/cli_args.go @@ -253,7 +253,11 @@ func parseCliOpts(osArgs []string) (args argContainer) { } // "-openssl" needs some post-processing if opensslAuto == "auto" { - args.openssl = stupidgcm.PreferOpenSSLAES256GCM() + if args.xchacha { + args.openssl = stupidgcm.PreferOpenSSLXchacha20poly1305() + } else { + args.openssl = stupidgcm.PreferOpenSSLAES256GCM() + } } else { args.openssl, err = strconv.ParseBool(opensslAuto) if err != nil { diff --git a/internal/cryptocore/cryptocore.go b/internal/cryptocore/cryptocore.go index d7b7527..dd7c98b 100644 --- a/internal/cryptocore/cryptocore.go +++ b/internal/cryptocore/cryptocore.go @@ -32,11 +32,11 @@ type AEADTypeEnum struct { NonceSize int } -// BackendOpenSSL specifies the OpenSSL backend. +// BackendOpenSSL specifies the OpenSSL AES-256-GCM backend. // "AES-GCM-256-OpenSSL" in gocryptfs -speed. var BackendOpenSSL AEADTypeEnum = AEADTypeEnum{"AES-GCM-256-OpenSSL", 16} -// BackendGoGCM specifies the Go based GCM backend. +// BackendGoGCM specifies the Go based AES-256-GCM backend. // "AES-GCM-256-Go" in gocryptfs -speed. var BackendGoGCM AEADTypeEnum = AEADTypeEnum{"AES-GCM-256-Go", 16} @@ -130,6 +130,8 @@ func New(key []byte, aeadType AEADTypeEnum, IVBitLen int, useHKDF bool, forceDec if err != nil { log.Panic(err) } + default: + log.Panicf("BUG: unhandled case: %v", aeadType) } for i := range gcmKey { gcmKey[i] = 0 @@ -154,7 +156,7 @@ func New(key []byte, aeadType AEADTypeEnum, IVBitLen int, useHKDF bool, forceDec for i := range key64 { key64[i] = 0 } - } else if aeadType == BackendXChaCha20Poly1305 { + } else if aeadType == BackendXChaCha20Poly1305 || aeadType == BackendXChaCha20Poly1305OpenSSL { // We don't support legacy modes with XChaCha20-Poly1305 if IVBitLen != chacha20poly1305.NonceSizeX*8 { log.Panicf("XChaCha20-Poly1305 must use 192-bit IVs, you wanted %d", IVBitLen) @@ -163,7 +165,13 @@ func New(key []byte, aeadType AEADTypeEnum, IVBitLen int, useHKDF bool, forceDec log.Panic("XChaCha20-Poly1305 must use HKDF, but it is disabled") } derivedKey := hkdfDerive(key, hkdfInfoXChaChaPoly1305Content, chacha20poly1305.KeySize) - aeadCipher, err = chacha20poly1305.NewX(derivedKey) + if aeadType == BackendXChaCha20Poly1305 { + aeadCipher, err = chacha20poly1305.NewX(derivedKey) + } else if aeadType == BackendXChaCha20Poly1305OpenSSL { + aeadCipher = stupidgcm.NewXchacha20poly1305(derivedKey) + } else { + log.Panicf("BUG: unhandled case: %v", aeadType) + } if err != nil { log.Panic(err) } diff --git a/mount.go b/mount.go index d7cd7db..b1c76dd 100644 --- a/mount.go +++ b/mount.go @@ -259,7 +259,11 @@ func initFuseFrontend(args *argContainer) (rootNode fs.InodeEmbedder, wipeKeys f cryptoBackend = cryptocore.BackendAESSIV } if args.xchacha { - cryptoBackend = cryptocore.BackendXChaCha20Poly1305 + if args.openssl { + cryptoBackend = cryptocore.BackendXChaCha20Poly1305OpenSSL + } else { + cryptoBackend = cryptocore.BackendXChaCha20Poly1305 + } IVBits = chacha20poly1305.NonceSizeX * 8 } // forceOwner implies allow_other, as documented. @@ -291,6 +295,7 @@ func initFuseFrontend(args *argContainer) (rootNode fs.InodeEmbedder, wipeKeys f frontendArgs.DeterministicNames = !confFile.IsFeatureFlagSet(configfile.FlagDirIV) args.raw64 = confFile.IsFeatureFlagSet(configfile.FlagRaw64) args.hkdf = confFile.IsFeatureFlagSet(configfile.FlagHKDF) + // Note: this will always return the non-openssl variant cryptoBackend, err = confFile.ContentEncryption() if err != nil { tlog.Fatal.Printf("%v", err) @@ -301,8 +306,14 @@ func initFuseFrontend(args *argContainer) (rootNode fs.InodeEmbedder, wipeKeys f tlog.Fatal.Printf("AES-SIV is required by reverse mode, but not enabled in the config file") os.Exit(exitcodes.Usage) } - if cryptoBackend == cryptocore.BackendGoGCM && args.openssl { - cryptoBackend = cryptocore.BackendOpenSSL + // Upgrade to OpenSSL variant if requested + if args.openssl { + switch cryptoBackend { + case cryptocore.BackendGoGCM: + cryptoBackend = cryptocore.BackendOpenSSL + case cryptocore.BackendXChaCha20Poly1305: + cryptoBackend = cryptocore.BackendXChaCha20Poly1305OpenSSL + } } } // If allow_other is set and we run as root, try to give newly created files to From 2620cad0dc003c4d06b3c1c082a3337b64bc9410 Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Wed, 8 Sep 2021 20:34:01 +0200 Subject: [PATCH 43/59] tests/matrix: test xchacha with and without openssl --- tests/matrix/matrix_test.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/matrix/matrix_test.go b/tests/matrix/matrix_test.go index abcb7e0..34c111f 100644 --- a/tests/matrix/matrix_test.go +++ b/tests/matrix/matrix_test.go @@ -20,6 +20,7 @@ import ( "sync" "syscall" "testing" + "time" "golang.org/x/sys/unix" @@ -66,7 +67,9 @@ var matrix = []testcaseMatrix{ {false, "auto", false, false, []string{"-serialize_reads"}}, {false, "auto", false, false, []string{"-sharedstorage"}}, {false, "auto", false, false, []string{"-deterministic-names"}}, - {false, "auto", false, true, []string{"-xchacha"}}, + // Test xchacha with and without openssl + {false, "true", false, true, []string{"-xchacha"}}, + {false, "false", false, true, []string{"-xchacha"}}, } // This is the entry point for the tests @@ -97,7 +100,11 @@ func TestMain(m *testing.M) { opts = append(opts, testcase.extraArgs...) test_helpers.MountOrExit(test_helpers.DefaultCipherDir, test_helpers.DefaultPlainDir, opts...) before := test_helpers.ListFds(0, test_helpers.TmpDir) + t0 := time.Now() r := m.Run() + if testing.Verbose() { + fmt.Printf("matrix: run took %v\n", time.Since(t0)) + } // Catch fd leaks in the tests. NOTE: this does NOT catch leaks in // the gocryptfs FUSE process, but only in the tests that access it! // All fds that point outside TmpDir are not interesting (the Go test From ad21647f250697d88eade461c08073732d2cf7e0 Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Wed, 8 Sep 2021 20:46:52 +0200 Subject: [PATCH 44/59] -speed: show which xchacha implementation is preferred --- internal/speed/speed.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/internal/speed/speed.go b/internal/speed/speed.go index d5aa696..f7cb5d3 100644 --- a/internal/speed/speed.go +++ b/internal/speed/speed.go @@ -35,8 +35,8 @@ func Run() { {name: cryptocore.BackendOpenSSL.Name, f: bStupidGCM, preferred: stupidgcm.PreferOpenSSLAES256GCM()}, {name: cryptocore.BackendGoGCM.Name, f: bGoGCM, preferred: !stupidgcm.PreferOpenSSLAES256GCM()}, {name: cryptocore.BackendAESSIV.Name, f: bAESSIV, preferred: false}, - {name: cryptocore.BackendXChaCha20Poly1305.Name, f: bXchacha20poly1305, preferred: false}, - {name: cryptocore.BackendXChaCha20Poly1305OpenSSL.Name, f: bStupidXchacha, preferred: false}, + {name: cryptocore.BackendXChaCha20Poly1305OpenSSL.Name, f: bStupidXchacha, preferred: stupidgcm.PreferOpenSSLXchacha20poly1305()}, + {name: cryptocore.BackendXChaCha20Poly1305.Name, f: bXchacha20poly1305, preferred: !stupidgcm.PreferOpenSSLXchacha20poly1305()}, } for _, b := range bTable { fmt.Printf("%-26s\t", b.name) @@ -48,8 +48,6 @@ func Run() { } if b.preferred { fmt.Printf("\t(selected in auto mode)\n") - } else if b.name == cryptocore.BackendXChaCha20Poly1305.Name { - fmt.Printf("\t(use via -xchacha flag)\n") } else { fmt.Printf("\t\n") } From c50d67f1039b8db90de441a3907994adb21668e8 Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Fri, 10 Sep 2021 11:51:41 +0200 Subject: [PATCH 45/59] profiling: accept parameters & show actual command lines --- profiling/ls.bash | 10 +++++++--- profiling/streaming-read.bash | 10 +++++++--- profiling/streaming-write.bash | 6 ++++-- profiling/tar-extract.bash | 6 ++++-- profiling/write-trace.bash | 6 ++++-- 5 files changed, 26 insertions(+), 12 deletions(-) diff --git a/profiling/ls.bash b/profiling/ls.bash index d8d55f6..35f5a39 100755 --- a/profiling/ls.bash +++ b/profiling/ls.bash @@ -8,8 +8,10 @@ cd "$(dirname "$0")" T=$(mktemp -d) mkdir "$T/a" "$T/b" -../gocryptfs -init -quiet -scryptn 10 -extpass "echo test" "$T/a" -../gocryptfs -quiet -nosyslog -extpass "echo test" "$T/a" "$T/b" +set -x +../gocryptfs -init -quiet -scryptn 10 -extpass "echo test" "$@" "$T/a" +{ set +x ; } 2> /dev/null +../gocryptfs -quiet -nosyslog -extpass "echo test" "$@" "$T/a" "$T/b" # Cleanup trap trap "cd /; fusermount -u -z $T/b; rm -Rf $T/a" EXIT @@ -24,8 +26,10 @@ echo "done, $SECONDS seconds" echo "Remount..." fusermount -u "$T/b" +set -x ../gocryptfs -quiet -nosyslog -extpass "echo test" -cpuprofile "$T/cprof" -memprofile "$T/mprof" \ - "$T/a" "$T/b" + "$@" "$T/a" "$T/b" +{ set +x ; } 2> /dev/null echo "Running ls under profiler (3x)..." for i in 1 2 3; do diff --git a/profiling/streaming-read.bash b/profiling/streaming-read.bash index ef55138..86ef942 100755 --- a/profiling/streaming-read.bash +++ b/profiling/streaming-read.bash @@ -5,8 +5,10 @@ cd "$(dirname "$0")" T=$(mktemp -d) mkdir "$T/a" "$T/b" -../gocryptfs -init -quiet -scryptn 10 -extpass "echo test" "$T/a" -../gocryptfs -quiet -extpass "echo test" "$T/a" "$T/b" +set -x +../gocryptfs -init -quiet -scryptn 10 -extpass "echo test" "$@" "$T/a" +{ set +x ; } 2> /dev/null +../gocryptfs -quiet -extpass "echo test" "$@" "$T/a" "$T/b" # Cleanup trap trap "cd /; fusermount -u -z $T/b; rm -Rf $T/a" EXIT @@ -16,8 +18,10 @@ dd if=/dev/zero of="$T/b/zero" bs=1M count=100 status=none # Remount with profiling fusermount -u "$T/b" +set -x ../gocryptfs -quiet -extpass "echo test" -cpuprofile "$T/cprof" -memprofile "$T/mprof" \ - "$T/a" "$T/b" + "$@" "$T/a" "$T/b" +{ set +x ; } 2> /dev/null # Read 10 x 100MB instead of 1 x 1GB to keep the used disk space low for i in $(seq 1 10); do diff --git a/profiling/streaming-write.bash b/profiling/streaming-write.bash index 2cd74ad..6f3af56 100755 --- a/profiling/streaming-write.bash +++ b/profiling/streaming-write.bash @@ -5,9 +5,11 @@ cd "$(dirname "$0")" T=$(mktemp -d) mkdir "$T/a" "$T/b" -../gocryptfs -init -quiet -scryptn 10 -extpass "echo test" "$T/a" +set -x +../gocryptfs -init -quiet -scryptn 10 -extpass "echo test" "$@" "$T/a" ../gocryptfs -quiet -extpass "echo test" -cpuprofile "$T/cprof" -memprofile "$T/mprof" \ - "$T/a" "$T/b" + "$@" "$T/a" "$T/b" +{ set +x ; } 2> /dev/null # Cleanup trap trap "cd /; fusermount -u -z $T/b; rm -Rf $T/a" EXIT diff --git a/profiling/tar-extract.bash b/profiling/tar-extract.bash index 25f99a6..f176368 100755 --- a/profiling/tar-extract.bash +++ b/profiling/tar-extract.bash @@ -8,9 +8,11 @@ cd "$(dirname "$0")" T=$(mktemp -d) mkdir "$T/a" "$T/b" -../gocryptfs -init -quiet -scryptn 10 -extpass "echo test" "$T/a" +set -x +../gocryptfs -init -quiet -scryptn 10 -extpass "echo test" "$@" "$T/a" ../gocryptfs -quiet -extpass "echo test" -cpuprofile "$T/cprof" -memprofile "$T/mprof" \ - "$T/a" "$T/b" + "$@" "$T/a" "$T/b" +{ set +x ; } 2> /dev/null # Cleanup trap trap "cd /; fusermount -u -z $T/b; rm -Rf $T/a" EXIT diff --git a/profiling/write-trace.bash b/profiling/write-trace.bash index 707e1d3..31af492 100755 --- a/profiling/write-trace.bash +++ b/profiling/write-trace.bash @@ -8,9 +8,11 @@ cd "$(dirname "$0")" T=$(mktemp -d) mkdir "$T/a" "$T/b" -../gocryptfs -init -quiet -scryptn 10 -extpass "echo test" "$T/a" +set -x +../gocryptfs -init -quiet -scryptn 10 -extpass "echo test" "$@" "$T/a" ../gocryptfs -quiet -extpass "echo test" -trace "$T/trace" \ - "$T/a" "$T/b" + "$@" "$T/a" "$T/b" +{ set +x ; } 2> /dev/null # Cleanup trap trap "cd /; fusermount -u -z $T/b; rm -Rf $T/a" EXIT From c974116322f057a36ffb0b2ec0338b7f60872773 Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Fri, 10 Sep 2021 12:09:30 +0200 Subject: [PATCH 46/59] test.bash: call out if build-without-openssl.bash failed This can print out compile errors that are hard to understand if you are not aware that it builds without_openssl. --- test.bash | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test.bash b/test.bash index b0faf68..efc6456 100755 --- a/test.bash +++ b/test.bash @@ -39,7 +39,10 @@ fi # Clean up dangling filesystems and don't exit if we found some unmount_leftovers || true -./build-without-openssl.bash +./build-without-openssl.bash || { + echo "$MYNAME: build-without-openssl.bash failed" + exit 1 +} # Don't build with openssl if we were passed "-tags without_openssl" if [[ "$*" != *without_openssl* ]] ; then ./build.bash From d023cd6c95fcbc6b5056ba1f425d2ac3df4abc5a Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Fri, 10 Sep 2021 12:14:19 +0200 Subject: [PATCH 47/59] cli: drop -forcedecode flag The rewritten openssl backend does not support this flag anymore, and it was inherently dangerour. Drop it (ignored for compatibility) --- Documentation/MANPAGE.md | 15 +---------- cli_args.go | 33 +++--------------------- internal/configfile/config_file.go | 4 +-- internal/contentenc/content.go | 18 +++---------- internal/contentenc/content_test.go | 12 ++++----- internal/contentenc/offsets_test.go | 4 +-- internal/cryptocore/cryptocore.go | 8 +++--- internal/cryptocore/cryptocore_test.go | 8 +++--- internal/fusefrontend/args.go | 2 -- internal/fusefrontend/file.go | 14 +++------- internal/fusefrontend/xattr_unit_test.go | 4 +-- internal/speed/speed.go | 2 +- internal/speed/speed_test.go | 2 +- internal/stupidgcm/gcm.go | 2 +- internal/stupidgcm/gcm_test.go | 2 +- internal/stupidgcm/without_openssl.go | 2 +- mount.go | 9 ++----- 17 files changed, 37 insertions(+), 104 deletions(-) diff --git a/Documentation/MANPAGE.md b/Documentation/MANPAGE.md index 7371fc2..e4e8f2e 100644 --- a/Documentation/MANPAGE.md +++ b/Documentation/MANPAGE.md @@ -256,21 +256,8 @@ of a case where this may be useful is a situation where content is stored on a filesystem that doesn't properly support UNIX ownership and permissions. #### -forcedecode -Force decode of encrypted files even if the integrity check fails, instead of -failing with an IO error. Warning messages are still printed to syslog if corrupted -files are encountered. -It can be useful to recover files from disks with bad sectors or other corrupted -media. It shall not be used if the origin of corruption is unknown, specially -if you want to run executable files. -For corrupted media, note that you probably want to use dd_rescue(1) -instead, which will recover all but the corrupted 4kB block. - -This option makes no sense in reverse mode. It requires gocryptfs to be compiled with openssl -support and implies -openssl true. Because of this, it is not compatible with -aessiv, -that uses built-in Go crypto. - -Setting this option forces the filesystem to read-only and noexec. +Obsolete and ignored on gocryptfs v2.2 and later. #### -fsname string Override the filesystem name (first column in df -T). Can also be diff --git a/cli_args.go b/cli_args.go index d666b47..b415b21 100644 --- a/cli_args.go +++ b/cli_args.go @@ -29,7 +29,7 @@ type argContainer struct { debug, init, zerokey, fusedebug, openssl, passwd, fg, version, plaintextnames, quiet, nosyslog, wpanic, longnames, allow_other, reverse, aessiv, nonempty, raw64, - noprealloc, speed, hkdf, serialize_reads, forcedecode, hh, info, + noprealloc, speed, hkdf, serialize_reads, hh, info, sharedstorage, fsck, one_file_system, deterministic_names, xchacha bool // Mount options with opposites @@ -172,8 +172,6 @@ func parseCliOpts(osArgs []string) (args argContainer) { flagSet.BoolVar(&args.speed, "speed", false, "Run crypto speed test") flagSet.BoolVar(&args.hkdf, "hkdf", true, "Use HKDF as an additional key derivation step") flagSet.BoolVar(&args.serialize_reads, "serialize_reads", false, "Try to serialize read operations") - flagSet.BoolVar(&args.forcedecode, "forcedecode", false, "Force decode of files even if integrity check fails."+ - " Requires gocryptfs to be compiled with openssl support and implies -openssl true") flagSet.BoolVar(&args.hh, "hh", false, "Show this long help text") flagSet.BoolVar(&args.info, "info", false, "Display information about CIPHERDIR") flagSet.BoolVar(&args.sharedstorage, "sharedstorage", false, "Make concurrent access to a shared CIPHERDIR safer") @@ -234,7 +232,8 @@ func parseCliOpts(osArgs []string) (args argContainer) { { var tmp bool flagSet.BoolVar(&tmp, "nofail", false, "Ignored for /etc/fstab compatibility") - flagSet.BoolVar(&tmp, "devrandom", false, "Deprecated (ignored for compatibility)") + flagSet.BoolVar(&tmp, "devrandom", false, "Obsolete, ignored for compatibility") + flagSet.BoolVar(&tmp, "forcedecode", false, "Obsolete, ignored for compatibility") } // Actual parsing @@ -265,32 +264,6 @@ func parseCliOpts(osArgs []string) (args argContainer) { os.Exit(exitcodes.Usage) } } - // "-forcedecode" only works with openssl. Check compilation and command line parameters - if args.forcedecode { - if stupidgcm.BuiltWithoutOpenssl { - tlog.Fatal.Printf("The -forcedecode flag requires openssl support, but gocryptfs was compiled without it!") - os.Exit(exitcodes.Usage) - } - if args.aessiv { - tlog.Fatal.Printf("The -forcedecode and -aessiv flags are incompatible because they use different crypto libs (openssl vs native Go)") - os.Exit(exitcodes.Usage) - } - if args.reverse { - tlog.Fatal.Printf("The reverse mode and the -forcedecode option are not compatible") - os.Exit(exitcodes.Usage) - } - // Has the user explicitly disabled openssl using "-openssl=false/0"? - if !args.openssl && opensslAuto != "auto" { - tlog.Fatal.Printf("-forcedecode requires openssl, but is disabled via command-line option") - os.Exit(exitcodes.Usage) - } - args.openssl = true - - // Try to make it harder for the user to shoot himself in the foot. - args.ro = true - args.allow_other = false - args.ko = "noexec" - } if len(args.extpass) > 0 && len(args.passfile) != 0 { tlog.Fatal.Printf("The options -extpass and -passfile cannot be used at the same time") os.Exit(exitcodes.Usage) diff --git a/internal/configfile/config_file.go b/internal/configfile/config_file.go index 06b665b..828f034 100644 --- a/internal/configfile/config_file.go +++ b/internal/configfile/config_file.go @@ -298,8 +298,8 @@ func getKeyEncrypter(scryptHash []byte, useHKDF bool) *contentenc.ContentEnc { if useHKDF { IVLen = contentenc.DefaultIVBits } - cc := cryptocore.New(scryptHash, cryptocore.BackendGoGCM, IVLen, useHKDF, false) - ce := contentenc.New(cc, 4096, false) + cc := cryptocore.New(scryptHash, cryptocore.BackendGoGCM, IVLen, useHKDF) + ce := contentenc.New(cc, 4096) return ce } diff --git a/internal/contentenc/content.go b/internal/contentenc/content.go index 13e0ce0..3005bf5 100644 --- a/internal/contentenc/content.go +++ b/internal/contentenc/content.go @@ -13,7 +13,6 @@ import ( "github.com/hanwen/go-fuse/v2/fuse" "github.com/rfjakob/gocryptfs/v2/internal/cryptocore" - "github.com/rfjakob/gocryptfs/v2/internal/stupidgcm" "github.com/rfjakob/gocryptfs/v2/internal/tlog" ) @@ -41,8 +40,6 @@ type ContentEnc struct { allZeroBlock []byte // All-zero block of size IVBitLen/8, for fast compares allZeroNonce []byte - // Force decode even if integrity check fails (openSSL only) - forceDecode bool // Ciphertext block "sync.Pool" pool. Always returns cipherBS-sized byte // slices (usually 4128 bytes). @@ -60,9 +57,8 @@ type ContentEnc struct { } // New returns an initialized ContentEnc instance. -func New(cc *cryptocore.CryptoCore, plainBS uint64, forceDecode bool) *ContentEnc { - tlog.Debug.Printf("contentenc.New: plainBS=%d, forceDecode=%v", - plainBS, forceDecode) +func New(cc *cryptocore.CryptoCore, plainBS uint64) *ContentEnc { + tlog.Debug.Printf("contentenc.New: plainBS=%d", plainBS) if fuse.MAX_KERNEL_WRITE%plainBS != 0 { log.Panicf("unaligned MAX_KERNEL_WRITE=%d", fuse.MAX_KERNEL_WRITE) @@ -81,7 +77,6 @@ func New(cc *cryptocore.CryptoCore, plainBS uint64, forceDecode bool) *ContentEn cipherBS: cipherBS, allZeroBlock: make([]byte, cipherBS), allZeroNonce: make([]byte, cc.IVLen), - forceDecode: forceDecode, cBlockPool: newBPool(int(cipherBS)), CReqPool: newBPool(cReqSize), pBlockPool: newBPool(int(plainBS)), @@ -111,11 +106,7 @@ func (be *ContentEnc) DecryptBlocks(ciphertext []byte, firstBlockNo uint64, file var pBlock []byte pBlock, err = be.DecryptBlock(cBlock, blockNo, fileID) if err != nil { - if be.forceDecode && err == stupidgcm.ErrAuth { - tlog.Warn.Printf("DecryptBlocks: authentication failure in block #%d, overridden by forcedecode", firstBlockNo) - } else { - break - } + break } pBuf.Write(pBlock) be.pBlockPool.Put(pBlock) @@ -183,9 +174,6 @@ func (be *ContentEnc) DecryptBlock(ciphertext []byte, blockNo uint64, fileID []b if err != nil { tlog.Debug.Printf("DecryptBlock: %s, len=%d", err.Error(), len(ciphertextOrig)) tlog.Debug.Println(hex.Dump(ciphertextOrig)) - if be.forceDecode && err == stupidgcm.ErrAuth { - return plaintext, err - } return nil, err } diff --git a/internal/contentenc/content_test.go b/internal/contentenc/content_test.go index 9cc8753..4a4b2de 100644 --- a/internal/contentenc/content_test.go +++ b/internal/contentenc/content_test.go @@ -23,8 +23,8 @@ func TestSplitRange(t *testing.T) { testRange{6654, 8945}) key := make([]byte, cryptocore.KeyLen) - cc := cryptocore.New(key, cryptocore.BackendGoGCM, DefaultIVBits, true, false) - f := New(cc, DefaultBS, false) + cc := cryptocore.New(key, cryptocore.BackendGoGCM, DefaultIVBits, true) + f := New(cc, DefaultBS) for _, r := range ranges { parts := f.ExplodePlainRange(r.offset, r.length) @@ -51,8 +51,8 @@ func TestCiphertextRange(t *testing.T) { testRange{6654, 8945}) key := make([]byte, cryptocore.KeyLen) - cc := cryptocore.New(key, cryptocore.BackendGoGCM, DefaultIVBits, true, false) - f := New(cc, DefaultBS, false) + cc := cryptocore.New(key, cryptocore.BackendGoGCM, DefaultIVBits, true) + f := New(cc, DefaultBS) for _, r := range ranges { @@ -74,8 +74,8 @@ func TestCiphertextRange(t *testing.T) { func TestBlockNo(t *testing.T) { key := make([]byte, cryptocore.KeyLen) - cc := cryptocore.New(key, cryptocore.BackendGoGCM, DefaultIVBits, true, false) - f := New(cc, DefaultBS, false) + cc := cryptocore.New(key, cryptocore.BackendGoGCM, DefaultIVBits, true) + f := New(cc, DefaultBS) b := f.CipherOffToBlockNo(788) if b != 0 { diff --git a/internal/contentenc/offsets_test.go b/internal/contentenc/offsets_test.go index 768393c..b35964a 100644 --- a/internal/contentenc/offsets_test.go +++ b/internal/contentenc/offsets_test.go @@ -10,8 +10,8 @@ import ( // TestSizeToSize tests CipherSizeToPlainSize and PlainSizeToCipherSize func TestSizeToSize(t *testing.T) { key := make([]byte, cryptocore.KeyLen) - cc := cryptocore.New(key, cryptocore.BackendGoGCM, DefaultIVBits, true, false) - ce := New(cc, DefaultBS, false) + cc := cryptocore.New(key, cryptocore.BackendGoGCM, DefaultIVBits, true) + ce := New(cc, DefaultBS) const rangeMax = 10000 diff --git a/internal/cryptocore/cryptocore.go b/internal/cryptocore/cryptocore.go index dd7c98b..48386f8 100644 --- a/internal/cryptocore/cryptocore.go +++ b/internal/cryptocore/cryptocore.go @@ -73,9 +73,9 @@ type CryptoCore struct { // // Note: "key" is either the scrypt hash of the password (when decrypting // a config file) or the masterkey (when finally mounting the filesystem). -func New(key []byte, aeadType AEADTypeEnum, IVBitLen int, useHKDF bool, forceDecode bool) *CryptoCore { - tlog.Debug.Printf("cryptocore.New: key=%d bytes, aeadType=%v, IVBitLen=%d, useHKDF=%v, forceDecode=%v", - len(key), aeadType, IVBitLen, useHKDF, forceDecode) +func New(key []byte, aeadType AEADTypeEnum, IVBitLen int, useHKDF bool) *CryptoCore { + tlog.Debug.Printf("cryptocore.New: key=%d bytes, aeadType=%v, IVBitLen=%d, useHKDF=%v", + len(key), aeadType, IVBitLen, useHKDF) if len(key) != KeyLen { log.Panicf("Unsupported key length of %d bytes", len(key)) @@ -120,7 +120,7 @@ func New(key []byte, aeadType AEADTypeEnum, IVBitLen int, useHKDF bool, forceDec if IVBitLen != 128 { log.Panicf("stupidgcm only supports 128-bit IVs, you wanted %d", IVBitLen) } - aeadCipher = stupidgcm.NewAES256GCM(gcmKey, forceDecode) + aeadCipher = stupidgcm.NewAES256GCM(gcmKey) case BackendGoGCM: goGcmBlockCipher, err := aes.NewCipher(gcmKey) if err != nil { diff --git a/internal/cryptocore/cryptocore_test.go b/internal/cryptocore/cryptocore_test.go index 319a900..d37e941 100644 --- a/internal/cryptocore/cryptocore_test.go +++ b/internal/cryptocore/cryptocore_test.go @@ -10,18 +10,18 @@ import ( func TestCryptoCoreNew(t *testing.T) { key := make([]byte, 32) for _, useHKDF := range []bool{true, false} { - c := New(key, BackendGoGCM, 96, useHKDF, false) + c := New(key, BackendGoGCM, 96, useHKDF) if c.IVLen != 12 { t.Fail() } - c = New(key, BackendGoGCM, 128, useHKDF, false) + c = New(key, BackendGoGCM, 128, useHKDF) if c.IVLen != 16 { t.Fail() } if stupidgcm.BuiltWithoutOpenssl { continue } - c = New(key, BackendOpenSSL, 128, useHKDF, false) + c = New(key, BackendOpenSSL, 128, useHKDF) if c.IVLen != 16 { t.Fail() } @@ -37,5 +37,5 @@ func TestNewPanic(t *testing.T) { }() key := make([]byte, 16) - New(key, BackendOpenSSL, 128, true, false) + New(key, BackendOpenSSL, 128, true) } diff --git a/internal/fusefrontend/args.go b/internal/fusefrontend/args.go index 4aedf2e..64a5923 100644 --- a/internal/fusefrontend/args.go +++ b/internal/fusefrontend/args.go @@ -26,8 +26,6 @@ type Args struct { ConfigCustom bool // NoPrealloc disables automatic preallocation before writing NoPrealloc bool - // Force decode even if integrity check fails (openSSL only) - ForceDecode bool // Exclude is a list of paths to make inaccessible, starting match at // the filesystem root Exclude []string diff --git a/internal/fusefrontend/file.go b/internal/fusefrontend/file.go index 661c2b8..3ce1b1e 100644 --- a/internal/fusefrontend/file.go +++ b/internal/fusefrontend/file.go @@ -20,7 +20,6 @@ import ( "github.com/rfjakob/gocryptfs/v2/internal/contentenc" "github.com/rfjakob/gocryptfs/v2/internal/inomap" "github.com/rfjakob/gocryptfs/v2/internal/openfiletable" - "github.com/rfjakob/gocryptfs/v2/internal/stupidgcm" "github.com/rfjakob/gocryptfs/v2/internal/syscallcompat" "github.com/rfjakob/gocryptfs/v2/internal/tlog" ) @@ -208,16 +207,9 @@ func (f *File) doRead(dst []byte, off uint64, length uint64) ([]byte, syscall.Er plaintext, err := f.contentEnc.DecryptBlocks(ciphertext, firstBlockNo, fileID) f.rootNode.contentEnc.CReqPool.Put(ciphertext) if err != nil { - if f.rootNode.args.ForceDecode && err == stupidgcm.ErrAuth { - // We do not have the information which block was corrupt here anymore, - // but DecryptBlocks() has already logged it anyway. - tlog.Warn.Printf("doRead %d: off=%d len=%d: returning corrupt data due to forcedecode", - f.qIno.Ino, off, length) - } else { - curruptBlockNo := firstBlockNo + f.contentEnc.PlainOffToBlockNo(uint64(len(plaintext))) - tlog.Warn.Printf("doRead %d: corrupt block #%d: %v", f.qIno.Ino, curruptBlockNo, err) - return nil, syscall.EIO - } + curruptBlockNo := firstBlockNo + f.contentEnc.PlainOffToBlockNo(uint64(len(plaintext))) + tlog.Warn.Printf("doRead %d: corrupt block #%d: %v", f.qIno.Ino, curruptBlockNo, err) + return nil, syscall.EIO } // Crop down to the relevant part diff --git a/internal/fusefrontend/xattr_unit_test.go b/internal/fusefrontend/xattr_unit_test.go index 7449d24..5bffd5e 100644 --- a/internal/fusefrontend/xattr_unit_test.go +++ b/internal/fusefrontend/xattr_unit_test.go @@ -17,8 +17,8 @@ import ( func newTestFS(args Args) *RootNode { // Init crypto backend key := make([]byte, cryptocore.KeyLen) - cCore := cryptocore.New(key, cryptocore.BackendGoGCM, contentenc.DefaultIVBits, true, false) - cEnc := contentenc.New(cCore, contentenc.DefaultBS, false) + cCore := cryptocore.New(key, cryptocore.BackendGoGCM, contentenc.DefaultIVBits, true) + cEnc := contentenc.New(cCore, contentenc.DefaultBS) n := nametransform.New(cCore.EMECipher, true, true, nil, false) rn := NewRootNode(args, cEnc, n) oneSec := time.Second diff --git a/internal/speed/speed.go b/internal/speed/speed.go index f7cb5d3..9958697 100644 --- a/internal/speed/speed.go +++ b/internal/speed/speed.go @@ -114,7 +114,7 @@ func bStupidGCM(b *testing.B) { if stupidgcm.BuiltWithoutOpenssl { b.Skip("openssl has been disabled at compile-time") } - bEncrypt(b, stupidgcm.NewAES256GCM(randBytes(32), false)) + bEncrypt(b, stupidgcm.NewAES256GCM(randBytes(32))) } // bGoGCM benchmarks Go stdlib GCM diff --git a/internal/speed/speed_test.go b/internal/speed/speed_test.go index 11c68d0..5f3001b 100644 --- a/internal/speed/speed_test.go +++ b/internal/speed/speed_test.go @@ -31,7 +31,7 @@ func BenchmarkStupidGCMDecrypt(b *testing.B) { if stupidgcm.BuiltWithoutOpenssl { b.Skip("openssl has been disabled at compile-time") } - bDecrypt(b, stupidgcm.NewAES256GCM(randBytes(32), false)) + bDecrypt(b, stupidgcm.NewAES256GCM(randBytes(32))) } func BenchmarkGoGCM(b *testing.B) { diff --git a/internal/stupidgcm/gcm.go b/internal/stupidgcm/gcm.go index c38dd5f..00819dd 100644 --- a/internal/stupidgcm/gcm.go +++ b/internal/stupidgcm/gcm.go @@ -26,7 +26,7 @@ type stupidGCM struct { // NewAES256GCM returns a new AES-256-GCM cipher that satisfies the cipher.AEAD interface. // // Only 32-bytes keys and 16-byte IVs are supported. -func NewAES256GCM(keyIn []byte, forceDecode bool) cipher.AEAD { +func NewAES256GCM(keyIn []byte) cipher.AEAD { if len(keyIn) != keyLen { log.Panicf("Only %d-byte keys are supported", keyLen) } diff --git a/internal/stupidgcm/gcm_test.go b/internal/stupidgcm/gcm_test.go index b587e58..73668fa 100644 --- a/internal/stupidgcm/gcm_test.go +++ b/internal/stupidgcm/gcm_test.go @@ -13,7 +13,7 @@ import ( func TestStupidGCM(t *testing.T) { key := randBytes(32) - sGCM := NewAES256GCM(key, false) + sGCM := NewAES256GCM(key) gAES, err := aes.NewCipher(key) if err != nil { diff --git a/internal/stupidgcm/without_openssl.go b/internal/stupidgcm/without_openssl.go index 93efcb4..42604de 100644 --- a/internal/stupidgcm/without_openssl.go +++ b/internal/stupidgcm/without_openssl.go @@ -21,7 +21,7 @@ func errExit() { os.Exit(exitcodes.OpenSSL) } -func NewAES256GCM(_ []byte, _ bool) cipher.AEAD { +func NewAES256GCM(_ []byte) cipher.AEAD { errExit() return nil } diff --git a/mount.go b/mount.go index b1c76dd..dfabbc9 100644 --- a/mount.go +++ b/mount.go @@ -277,7 +277,6 @@ func initFuseFrontend(args *argContainer) (rootNode fs.InodeEmbedder, wipeKeys f LongNames: args.longnames, ConfigCustom: args._configCustom, NoPrealloc: args.noprealloc, - ForceDecode: args.forcedecode, ForceOwner: args._forceOwner, Exclude: args.exclude, ExcludeWildcard: args.excludeWildcard, @@ -323,8 +322,8 @@ func initFuseFrontend(args *argContainer) (rootNode fs.InodeEmbedder, wipeKeys f } // Init crypto backend - cCore := cryptocore.New(masterkey, cryptoBackend, IVBits, args.hkdf, args.forcedecode) - cEnc := contentenc.New(cCore, contentenc.DefaultBS, args.forcedecode) + cCore := cryptocore.New(masterkey, cryptoBackend, IVBits, args.hkdf) + cEnc := contentenc.New(cCore, contentenc.DefaultBS) nameTransform := nametransform.New(cCore.EMECipher, frontendArgs.LongNames, args.raw64, []string(args.badname), frontendArgs.DeterministicNames) // After the crypto backend is initialized, @@ -408,10 +407,6 @@ func initGoFuse(rootNode fs.InodeEmbedder, args *argContainer) *fuse.Server { if args.acl { mOpts.EnableAcl = true } - if args.forcedecode { - tlog.Info.Printf(tlog.ColorYellow + "THE OPTION \"-forcedecode\" IS ACTIVE. GOCRYPTFS WILL RETURN CORRUPT DATA!" + - tlog.ColorReset) - } // fusermount from libfuse 3.x removed the "nonempty" option and exits // with an error if it sees it. Only add it to the options on libfuse 2.x. if args.nonempty && haveFusermount2() { From a85e39f68288af3b9238f7caeefe6eb420b44f02 Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Fri, 10 Sep 2021 12:17:22 +0200 Subject: [PATCH 48/59] Update README & MANPAGE --- Documentation/MANPAGE.md | 4 +++- README.md | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Documentation/MANPAGE.md b/Documentation/MANPAGE.md index e4e8f2e..01e4b5a 100644 --- a/Documentation/MANPAGE.md +++ b/Documentation/MANPAGE.md @@ -256,9 +256,11 @@ of a case where this may be useful is a situation where content is stored on a filesystem that doesn't properly support UNIX ownership and permissions. #### -forcedecode - Obsolete and ignored on gocryptfs v2.2 and later. +See https://github.com/rfjakob/gocryptfs/commit/d023cd6c95fcbc6b5056ba1f425d2ac3df4abc5a +for what it was and why it was dropped. + #### -fsname string Override the filesystem name (first column in df -T). Can also be passed as "-o fsname=" and is equivalent to libfuse's option of the diff --git a/README.md b/README.md index e1104b8..0926095 100644 --- a/README.md +++ b/README.md @@ -200,8 +200,8 @@ v2.2, IN PROGRESS ([#151](https://github.com/rfjakob/gocryptfs/issues/151), [#402](https://github.com/rfjakob/gocryptfs/issues/402), [#592](https://github.com/rfjakob/gocryptfs/pull/592)) * New feature flag! You need gocryptfs v2.2 or higher to mount a filesystem that uses this flag. * `-xchacha`: new option for `-init` (forward mode only). Selects XChaCha20-Poly1305 for content encryption. - Gives [better performance on embedded CPUs](https://gist.github.com/rfjakob/b28383f4c84263ac7c5388ccc262e38b) - ([#452](https://github.com/rfjakob/gocryptfs/issues/452)) + Gives *much* better performance on CPUs without AES acceleration + ([#452](https://github.com/rfjakob/gocryptfs/issues/452)). * New feature flag! You need gocryptfs v2.2 or higher to mount a filesystem that uses this flag. * Test with `gocryptfs -speed` what is fastest for your CPU, or read [here](https://github.com/rfjakob/gocryptfs/issues/452#issuecomment-908559414) * `-serialize_reads`: get rid of delay logic by taking advantage of the kernel flag @@ -209,6 +209,7 @@ v2.2, IN PROGRESS ([go-fuse commit](https://github.com/hanwen/go-fuse/commit/15a8bb029a4e1a51e10043c370970596b1fbb737), [gocryptfs commit](https://github.com/rfjakob/gocryptfs/commit/a99051b32452c9a781efe248c0014b65d4abddf7)) * Make obsolete `-devrandom` flag a no-op ([commit](https://github.com/rfjakob/gocryptfs/commit/61ef6b00a675456ee05d40f1ce44d693bc4be350)) +* Make `-forcedecode` flag a no-op ([commit](https://github.com/rfjakob/gocryptfs/commit/d023cd6c95fcbc6b5056ba1f425d2ac3df4abc5a)) v2.1, 2021-08-18 * `-fido2`: do not request PIN on `gocryptfs -init` fixing `FIDO_ERR_UNSUPPORTED_OPTION` with YubiKey From ee561035707527dee48cecfc9466ab4921666b43 Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Fri, 10 Sep 2021 12:22:02 +0200 Subject: [PATCH 49/59] README: update changelog for v2.2-beta1 --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 0926095..919d2ee 100644 --- a/README.md +++ b/README.md @@ -204,6 +204,8 @@ v2.2, IN PROGRESS ([#452](https://github.com/rfjakob/gocryptfs/issues/452)). * New feature flag! You need gocryptfs v2.2 or higher to mount a filesystem that uses this flag. * Test with `gocryptfs -speed` what is fastest for your CPU, or read [here](https://github.com/rfjakob/gocryptfs/issues/452#issuecomment-908559414) +* Rewrite [OpenSSL backend](https://pkg.go.dev/github.com/rfjakob/gocryptfs/v2@master/internal/stupidgcm) + for better performance on AES-GCM-256-OpenSSL and XChaCha20-Poly1305-OpenSSL * `-serialize_reads`: get rid of delay logic by taking advantage of the kernel flag `FUSE_CAP_ASYNC_READ` ([go-fuse commit](https://github.com/hanwen/go-fuse/commit/15a8bb029a4e1a51e10043c370970596b1fbb737), From c9b825c58a9f996379108926754513bca03bb306 Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Fri, 10 Sep 2021 17:17:16 +0200 Subject: [PATCH 50/59] inomap: deterministically set root device We used to have "first Translate() wins". This is not deterministic, as the LOOKUP for the root directory does not seem to reach us, so the first user LOOKUP would win, which may be on a mountpoint. --- internal/fusefrontend/root_node.go | 12 +++++++++++- internal/fusefrontend_reverse/root_node.go | 17 +++++++++++------ internal/inomap/inomap.go | 13 +++++++++++-- internal/inomap/inomap_test.go | 17 ++++++----------- 4 files changed, 39 insertions(+), 20 deletions(-) diff --git a/internal/fusefrontend/root_node.go b/internal/fusefrontend/root_node.go index 7d37520..7221be6 100644 --- a/internal/fusefrontend/root_node.go +++ b/internal/fusefrontend/root_node.go @@ -62,18 +62,28 @@ type RootNode struct { } func NewRootNode(args Args, c *contentenc.ContentEnc, n *nametransform.NameTransform) *RootNode { + var rootDev uint64 + var st syscall.Stat_t + if err := syscall.Stat(args.Cipherdir, &st); err != nil { + tlog.Warn.Printf("Could not stat backing directory %q: %v", args.Cipherdir, err) + } else { + rootDev = uint64(st.Dev) + } + if len(args.Exclude) > 0 { tlog.Warn.Printf("Forward mode does not support -exclude") } + ivLen := nametransform.DirIVLen if args.PlaintextNames { ivLen = 0 } + rn := &RootNode{ args: args, nameTransform: n, contentEnc: c, - inoMap: inomap.New(), + inoMap: inomap.New(rootDev), dirCache: dirCache{ivLen: ivLen}, quirks: syscallcompat.DetectQuirks(args.Cipherdir), } diff --git a/internal/fusefrontend_reverse/root_node.go b/internal/fusefrontend_reverse/root_node.go index d4c1e37..e15ddb0 100644 --- a/internal/fusefrontend_reverse/root_node.go +++ b/internal/fusefrontend_reverse/root_node.go @@ -2,10 +2,13 @@ package fusefrontend_reverse import ( "log" + "os" "path/filepath" "strings" "syscall" + "github.com/rfjakob/gocryptfs/v2/internal/exitcodes" + "github.com/rfjakob/gocryptfs/v2/internal/tlog" "golang.org/x/sys/unix" @@ -46,12 +49,14 @@ type RootNode struct { // ReverseFS provides an encrypted view. func NewRootNode(args fusefrontend.Args, c *contentenc.ContentEnc, n *nametransform.NameTransform) *RootNode { var rootDev uint64 - if args.OneFileSystem { - var st syscall.Stat_t - err := syscall.Stat(args.Cipherdir, &st) - if err != nil { - log.Panicf("Could not stat backing directory %q: %v", args.Cipherdir, err) + var st syscall.Stat_t + if err := syscall.Stat(args.Cipherdir, &st); err != nil { + tlog.Warn.Printf("Could not stat backing directory %q: %v", args.Cipherdir, err) + if args.OneFileSystem { + tlog.Fatal.Printf("This is a fatal error in combination with -one-file-system") + os.Exit(exitcodes.CipherDir) } + } else { rootDev = uint64(st.Dev) } @@ -59,7 +64,7 @@ func NewRootNode(args fusefrontend.Args, c *contentenc.ContentEnc, n *nametransf args: args, nameTransform: n, contentEnc: c, - inoMap: inomap.New(), + inoMap: inomap.New(rootDev), rootDev: rootDev, } if len(args.Exclude) > 0 || len(args.ExcludeWildcard) > 0 || len(args.ExcludeFrom) > 0 { diff --git a/internal/inomap/inomap.go b/internal/inomap/inomap.go index 97f9b61..997ea9b 100644 --- a/internal/inomap/inomap.go +++ b/internal/inomap/inomap.go @@ -49,13 +49,22 @@ type InoMap struct { } // New returns a new InoMap. -func New() *InoMap { - return &InoMap{ +// Inode numbers on device `rootDev` will be passed through as-is. +// If `rootDev` is zero, the first Translate() call decides the effective +// rootDev. +func New(rootDev uint64) *InoMap { + m := &InoMap{ namespaceMap: make(map[namespaceData]uint16), namespaceNext: 0, spillMap: make(map[QIno]uint64), spillNext: 0, } + if rootDev > 0 { + // Reserve namespace 0 for rootDev + m.namespaceMap[namespaceData{rootDev, 0}] = 0 + m.namespaceNext = 1 + } + return m } var spillWarn sync.Once diff --git a/internal/inomap/inomap_test.go b/internal/inomap/inomap_test.go index 6fbb4ef..9ec2932 100644 --- a/internal/inomap/inomap_test.go +++ b/internal/inomap/inomap_test.go @@ -6,7 +6,7 @@ import ( ) func TestTranslate(t *testing.T) { - m := New() + m := New(0) q := QIno{Ino: 1} out := m.Translate(q) if out != 1 { @@ -25,12 +25,7 @@ func TestTranslate(t *testing.T) { func TestTranslateStress(t *testing.T) { const baseDev = 12345 - m := New() - - // Make sure baseDev gets namespace id zero - var q QIno - q.Dev = baseDev - m.Translate(q) + m := New(baseDev) var wg sync.WaitGroup wg.Add(4) @@ -100,7 +95,7 @@ func TestTranslateStress(t *testing.T) { } func TestSpill(t *testing.T) { - m := New() + m := New(0) var q QIno q.Ino = maxPassthruIno + 1 out1 := m.Translate(q) @@ -119,7 +114,7 @@ func TestSpill(t *testing.T) { // TestUniqueness checks that unique (Dev, Flags, Ino) tuples get unique inode // numbers func TestUniqueness(t *testing.T) { - m := New() + m := New(0) var q QIno outMap := make(map[uint64]struct{}) for q.Dev = 0; q.Dev < 10; q.Dev++ { @@ -141,7 +136,7 @@ func TestUniqueness(t *testing.T) { } func BenchmarkTranslateSingleDev(b *testing.B) { - m := New() + m := New(0) var q QIno for n := 0; n < b.N; n++ { q.Ino = uint64(n % 1000) @@ -150,7 +145,7 @@ func BenchmarkTranslateSingleDev(b *testing.B) { } func BenchmarkTranslateManyDevs(b *testing.B) { - m := New() + m := New(0) var q QIno for n := 0; n < b.N; n++ { q.Dev = uint64(n % 10) From 2a4380ac258a8cb7d68ba5b165917bb816006320 Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Fri, 10 Sep 2021 17:19:51 +0200 Subject: [PATCH 51/59] README: update changelog --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 919d2ee..cbaf8a2 100644 --- a/README.md +++ b/README.md @@ -212,6 +212,7 @@ v2.2, IN PROGRESS [gocryptfs commit](https://github.com/rfjakob/gocryptfs/commit/a99051b32452c9a781efe248c0014b65d4abddf7)) * Make obsolete `-devrandom` flag a no-op ([commit](https://github.com/rfjakob/gocryptfs/commit/61ef6b00a675456ee05d40f1ce44d693bc4be350)) * Make `-forcedecode` flag a no-op ([commit](https://github.com/rfjakob/gocryptfs/commit/d023cd6c95fcbc6b5056ba1f425d2ac3df4abc5a)) +* Fix reverse mode sometimes remapping most inode numbers to >281474976710656 ([commit](https://github.com/rfjakob/gocryptfs/commit/c9b825c58a9f996379108926754513bca03bb306)) v2.1, 2021-08-18 * `-fido2`: do not request PIN on `gocryptfs -init` fixing `FIDO_ERR_UNSUPPORTED_OPTION` with YubiKey From d0cba59f6b2b8bd62c4c09b6f511d6a9f857ae3b Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Sun, 12 Sep 2021 18:12:10 +0200 Subject: [PATCH 52/59] README: highlight changes in v2.2, simplify pkg.go.dev link --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index cbaf8a2..a0edd33 100644 --- a/README.md +++ b/README.md @@ -195,16 +195,16 @@ Changelog --------- v2.2, IN PROGRESS -* `-deterministic-names`: new option for `-init`, both for reverse and forward mode. +* **`-deterministic-names`: new option for `-init`**, both for reverse and forward mode. Disables file name randomisation & `gocryptfs.diriv` files ([#151](https://github.com/rfjakob/gocryptfs/issues/151), [#402](https://github.com/rfjakob/gocryptfs/issues/402), [#592](https://github.com/rfjakob/gocryptfs/pull/592)) * New feature flag! You need gocryptfs v2.2 or higher to mount a filesystem that uses this flag. -* `-xchacha`: new option for `-init` (forward mode only). Selects XChaCha20-Poly1305 for content encryption. +* **`-xchacha`: new option for `-init`** (forward mode only). Selects XChaCha20-Poly1305 for content encryption. Gives *much* better performance on CPUs without AES acceleration ([#452](https://github.com/rfjakob/gocryptfs/issues/452)). * New feature flag! You need gocryptfs v2.2 or higher to mount a filesystem that uses this flag. * Test with `gocryptfs -speed` what is fastest for your CPU, or read [here](https://github.com/rfjakob/gocryptfs/issues/452#issuecomment-908559414) -* Rewrite [OpenSSL backend](https://pkg.go.dev/github.com/rfjakob/gocryptfs/v2@master/internal/stupidgcm) +* Rewrite [OpenSSL backend](https://pkg.go.dev/github.com/rfjakob/gocryptfs/v2/internal/stupidgcm) for better performance on AES-GCM-256-OpenSSL and XChaCha20-Poly1305-OpenSSL * `-serialize_reads`: get rid of delay logic by taking advantage of the kernel flag `FUSE_CAP_ASYNC_READ` From cdbc48fe297388392d95c0757a6099d01a50347b Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Tue, 14 Sep 2021 10:15:18 +0200 Subject: [PATCH 53/59] -speed: drop useless tab at end of line --- internal/speed/speed.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/speed/speed.go b/internal/speed/speed.go index 9958697..231a982 100644 --- a/internal/speed/speed.go +++ b/internal/speed/speed.go @@ -49,7 +49,7 @@ func Run() { if b.preferred { fmt.Printf("\t(selected in auto mode)\n") } else { - fmt.Printf("\t\n") + fmt.Printf("\n") } } } From 52b044498529c4abd1d2044eabd709047e1194b5 Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Tue, 14 Sep 2021 10:18:24 +0200 Subject: [PATCH 54/59] README: update example -speed output --- README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index a0edd33..e4ea9a7 100644 --- a/README.md +++ b/README.md @@ -168,11 +168,12 @@ Example for a CPU with AES-NI: ``` $ ./gocryptfs -speed -gocryptfs v2.0; go-fuse v2.1.1-0.20210508151621-62c5aa1919a7; 2021-06-06 go1.16.5 linux/amd64 -AES-GCM-256-OpenSSL 536.63 MB/s -AES-GCM-256-Go 831.84 MB/s (selected in auto mode) -AES-SIV-512-Go 155.85 MB/s -XChaCha20-Poly1305-Go 700.02 MB/s (benchmark only, not selectable yet) +gocryptfs v2.2.0-beta1-4-gcdbc48f; go-fuse v2.1.1-0.20210825171523-3ab5d95a30ae; 2021-09-14 go1.17.1 linux/amd64 +AES-GCM-256-OpenSSL 868.09 MB/s +AES-GCM-256-Go 997.97 MB/s (selected in auto mode) +AES-SIV-512-Go 160.72 MB/s +XChaCha20-Poly1305-OpenSSL 722.14 MB/s +XChaCha20-Poly1305-Go 841.89 MB/s (selected in auto mode) ``` You can run `./benchmark.bash` to run gocryptfs' canonical set of From 61e37b2439f0b8a7c16458e73cb57c7428fe61f2 Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Tue, 14 Sep 2021 18:47:07 +0200 Subject: [PATCH 55/59] stupidgcm: add CpuHasAES() Makes the code clearer, and will be used in the next commit. --- internal/stupidgcm/prefer.go | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/internal/stupidgcm/prefer.go b/internal/stupidgcm/prefer.go index ffbaf58..bb613c3 100644 --- a/internal/stupidgcm/prefer.go +++ b/internal/stupidgcm/prefer.go @@ -21,17 +21,11 @@ func PreferOpenSSLAES256GCM() bool { if BuiltWithoutOpenssl { return false } - // Safe to call on other architectures - will just read false. - if cpu.X86.HasAES || cpu.ARM64.HasAES { - // Go stdlib is probably faster + // If the CPU has AES acceleration, Go stdlib is faster + if CpuHasAES() { return false } - // On the Apple M1, Go stdlib is faster than OpenSSL, despite cpu.ARM64.HasAES - // reading false: https://github.com/rfjakob/gocryptfs/issues/556#issuecomment-848079309 - if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" { - return false - } - // OpenSSL is probably faster + // Otherwise OpenSSL is probably faster return true } @@ -49,3 +43,18 @@ func PreferOpenSSLXchacha20poly1305() bool { // On arm64 and arm, OpenSSL is faster. Probably everwhere else too. return true } + +// CpuHasAES tells you if the CPU we are running has AES acceleration that is +// usable by the Go crypto library. +func CpuHasAES() bool { + // Safe to call on other architectures - will just read false. + if cpu.X86.HasAES || cpu.ARM64.HasAES { + return true + } + // On the Apple M1, the CPU has AES acceleration, despite cpu.ARM64.HasAES + // reading false: https://github.com/rfjakob/gocryptfs/issues/556#issuecomment-848079309 + if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" { + return true + } + return false +} From 2d0ba24ecab375f276f024bc014faad9a7ef169c Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Tue, 14 Sep 2021 18:47:41 +0200 Subject: [PATCH 56/59] -speed: print cpu model When somebody posts "gocryptfs -speed" results, they are most helpful together with the CPU model. Add the cpu model to the output. Example: $ ./gocryptfs -speed gocryptfs v2.2.0-beta1-5-g52b0444-dirty; go-fuse v2.1.1-0.20210825171523-3ab5d95a30ae; 2021-09-14 go1.17.1 linux/amd64 cpu: Intel(R) Core(TM) i5-3470 CPU @ 3.20GHz; with AES acceleration AES-GCM-256-OpenSSL 862.79 MB/s AES-GCM-256-Go 997.71 MB/s (selected in auto mode) AES-SIV-512-Go 159.58 MB/s XChaCha20-Poly1305-OpenSSL 729.65 MB/s XChaCha20-Poly1305-Go 843.97 MB/s (selected in auto mode) --- README.md | 13 +++++----- internal/speed/cpuinfo.go | 54 +++++++++++++++++++++++++++++++++++++++ internal/speed/speed.go | 10 ++++++++ 3 files changed, 71 insertions(+), 6 deletions(-) create mode 100644 internal/speed/cpuinfo.go diff --git a/README.md b/README.md index e4ea9a7..69a3416 100644 --- a/README.md +++ b/README.md @@ -168,12 +168,13 @@ Example for a CPU with AES-NI: ``` $ ./gocryptfs -speed -gocryptfs v2.2.0-beta1-4-gcdbc48f; go-fuse v2.1.1-0.20210825171523-3ab5d95a30ae; 2021-09-14 go1.17.1 linux/amd64 -AES-GCM-256-OpenSSL 868.09 MB/s -AES-GCM-256-Go 997.97 MB/s (selected in auto mode) -AES-SIV-512-Go 160.72 MB/s -XChaCha20-Poly1305-OpenSSL 722.14 MB/s -XChaCha20-Poly1305-Go 841.89 MB/s (selected in auto mode) +gocryptfs v2.2.0-beta1-5-g52b0444-dirty; go-fuse v2.1.1-0.20210825171523-3ab5d95a30ae; 2021-09-14 go1.17.1 linux/amd64 +cpu: Intel(R) Core(TM) i5-3470 CPU @ 3.20GHz; with AES acceleration +AES-GCM-256-OpenSSL 862.79 MB/s +AES-GCM-256-Go 997.71 MB/s (selected in auto mode) +AES-SIV-512-Go 159.58 MB/s +XChaCha20-Poly1305-OpenSSL 729.65 MB/s +XChaCha20-Poly1305-Go 843.97 MB/s (selected in auto mode) ``` You can run `./benchmark.bash` to run gocryptfs' canonical set of diff --git a/internal/speed/cpuinfo.go b/internal/speed/cpuinfo.go new file mode 100644 index 0000000..09e7a89 --- /dev/null +++ b/internal/speed/cpuinfo.go @@ -0,0 +1,54 @@ +package speed + +import ( + "io/ioutil" + "os" + "runtime" + "strings" +) + +// cpuModelName returns the "model name" acc. to /proc/cpuinfo, or "" +// on error. +// +// Examples: On my desktop PC: +// +// $ grep "model name" /proc/cpuinfo +// model name : Intel(R) Core(TM) i5-3470 CPU @ 3.20GHz +// +// --> Returns "Intel(R) Core(TM) i5-3470 CPU @ 3.20GHz". +// +// On a Raspberry Pi 4: +// +// $ grep "model name" /proc/cpuinfo +// (empty) +// $ grep Hardware /proc/cpuinfo +// Hardware : BCM2835 +// +// --> Returns "BCM2835" +func cpuModelName() string { + if runtime.GOOS != "linux" { + return "" + } + f, err := os.Open("/proc/cpuinfo") + if err != nil { + return "" + } + content, err := ioutil.ReadAll(f) + if err != nil { + return "" + } + lines := strings.Split(string(content), "\n") + // Look for "model name", then for "Hardware" (arm devices don't have "model name") + for _, want := range []string{"model name", "Hardware"} { + for _, line := range lines { + if strings.HasPrefix(line, want) { + parts := strings.SplitN(line, ":", 2) + if len(parts) != 2 { + continue + } + return strings.TrimSpace(parts[1]) + } + } + } + return "" +} diff --git a/internal/speed/speed.go b/internal/speed/speed.go index 231a982..0b1a51a 100644 --- a/internal/speed/speed.go +++ b/internal/speed/speed.go @@ -27,6 +27,16 @@ const blockSize = 4096 // Run - run the speed the test and print the results. func Run() { + cpu := cpuModelName() + if cpu == "" { + cpu = "unknown" + } + aes := "; no AES acceleration" + if stupidgcm.CpuHasAES() { + aes = "; with AES acceleration" + } + fmt.Printf("cpu: %s%s\n", cpu, aes) + bTable := []struct { name string f func(*testing.B) From 53d51acd2b321ac50638998f81d268ea58236410 Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Wed, 15 Sep 2021 16:09:26 +0200 Subject: [PATCH 57/59] README: release will be called v2.2.0 instead of v2.2 pkg.go.dev really wants that we want to comply with https://golang.org/doc/modules/version-numbers . Trying v2.2-beta1 as in https://pkg.go.dev/github.com/rfjakob/gocryptfs/v2@v2.2-beta1 said "v2.2-beta1 is not a valid semantic version.". --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 69a3416..c2011af 100644 --- a/README.md +++ b/README.md @@ -196,7 +196,7 @@ RM: 2,367 Changelog --------- -v2.2, IN PROGRESS +v2.2.0, IN PROGRESS * **`-deterministic-names`: new option for `-init`**, both for reverse and forward mode. Disables file name randomisation & `gocryptfs.diriv` files ([#151](https://github.com/rfjakob/gocryptfs/issues/151), [#402](https://github.com/rfjakob/gocryptfs/issues/402), [#592](https://github.com/rfjakob/gocryptfs/pull/592)) @@ -215,6 +215,9 @@ v2.2, IN PROGRESS * Make obsolete `-devrandom` flag a no-op ([commit](https://github.com/rfjakob/gocryptfs/commit/61ef6b00a675456ee05d40f1ce44d693bc4be350)) * Make `-forcedecode` flag a no-op ([commit](https://github.com/rfjakob/gocryptfs/commit/d023cd6c95fcbc6b5056ba1f425d2ac3df4abc5a)) * Fix reverse mode sometimes remapping most inode numbers to >281474976710656 ([commit](https://github.com/rfjakob/gocryptfs/commit/c9b825c58a9f996379108926754513bca03bb306)) +* This version will be called v2.2.0 (instead of v2.2) to comply with + the [Go module versioning](https://golang.org/doc/modules/version-numbers) convention. + Later releases will also follow the convention. v2.1, 2021-08-18 * `-fido2`: do not request PIN on `gocryptfs -init` fixing `FIDO_ERR_UNSUPPORTED_OPTION` with YubiKey From eceeaaad1f56569ac74de5a6780280a7d813d8d9 Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Sat, 25 Sep 2021 16:44:06 +0200 Subject: [PATCH 58/59] README: make changelog entries subheadings This allows to anchor-link in to each release. --- README.md | 80 +++++++++++++++++++++++++++---------------------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index c2011af..af26567 100644 --- a/README.md +++ b/README.md @@ -196,7 +196,7 @@ RM: 2,367 Changelog --------- -v2.2.0, IN PROGRESS +#### v2.2.0, IN PROGRESS * **`-deterministic-names`: new option for `-init`**, both for reverse and forward mode. Disables file name randomisation & `gocryptfs.diriv` files ([#151](https://github.com/rfjakob/gocryptfs/issues/151), [#402](https://github.com/rfjakob/gocryptfs/issues/402), [#592](https://github.com/rfjakob/gocryptfs/pull/592)) @@ -219,7 +219,7 @@ v2.2.0, IN PROGRESS the [Go module versioning](https://golang.org/doc/modules/version-numbers) convention. Later releases will also follow the convention. -v2.1, 2021-08-18 +#### v2.1, 2021-08-18 * `-fido2`: do not request PIN on `gocryptfs -init` fixing `FIDO_ERR_UNSUPPORTED_OPTION` with YubiKey ([#571](https://github.com/rfjakob/gocryptfs/issues/571)) * `-sharedstorage`: present stable inode numbers, fixing getcwd failures @@ -238,19 +238,19 @@ v2.1, 2021-08-18 * Drop support for Go 1.11 & Go 1.12 ([commit](https://github.com/rfjakob/gocryptfs/commit/a5f88e86d186cdbc67e1efabd7aacf389775e027)) * You must have Go 1.13 or newer now -v2.0.1, 2021-06-07 +#### v2.0.1, 2021-06-07 * Fix symlink creation reporting the wrong size, causing git to report it as modified ([#574](https://github.com/rfjakob/gocryptfs/issues/574)) -v2.0, 2021-06-05 +#### v2.0, 2021-06-05 * Fix a few [issues discovered by xfstests](https://github.com/rfjakob/fuse-xfstests/wiki/results_2021-05-19) * Biggest change: rewrite SEEK_HOLE / SEEK_DATA logic (now emulates 4k alignment) -v2.0-beta4, 2021-05-15 +#### v2.0-beta4, 2021-05-15 * **Make ACLs *actually* work (pass `-acl` to enable)** ([#536](https://github.com/rfjakob/gocryptfs/issues/536)) * Blocklist `RENAME_EXCHANGE` and `RENAME_WHITEOUT` (broken as discovered by [fuse-xfstest/gocryptfs-2019-12](https://github.com/rfjakob/fuse-xfstests/tree/gocryptfs-2019-12)) -v2.0-beta3, 2021-04-24 +#### v2.0-beta3, 2021-04-24 * MANPAGE: Split options into sections acc. to where they apply ([#517](https://github.com/rfjakob/gocryptfs/issues/517)) * `-idle`: count cwd inside the mount as busy ([#533](https://github.com/rfjakob/gocryptfs/issues/533)) * Make `gocryptfs.diriv` and `gocryptfs.xxx.name` files world-readable to make encrypted backups easier @@ -269,11 +269,11 @@ v2.0-beta3, 2021-04-24 * Add directory fd caching for 2x - 3x speed boost in small file ops compared to v2.0-beta2 ([performance numbers](https://github.com/rfjakob/gocryptfs/blob/5cb1e55714aa92a848c0fb5fc3fa7b91625210fe/Documentation/performance.txt#L73)) -v2.0-beta2, 2020-11-14 +#### v2.0-beta2, 2020-11-14 * Improve [performance](Documentation/performance.txt#L69) * Fix [GETATTR panic](https://github.com/rfjakob/gocryptfs/issues/519#issuecomment-718790790) in reverse mode -v2.0-beta1, 2020-10-15 +#### v2.0-beta1, 2020-10-15 * **Switch to the improved go-fuse [v2 API](https://pkg.go.dev/github.com/hanwen/go-fuse/v2@v2.0.3/fs)** * This is a big change, a lot of code has been reorganized or rewritten to fit the v2 API model. @@ -292,7 +292,7 @@ v2.0-beta1, 2020-10-15 ([go-fuse #276](https://github.com/hanwen/go-fuse/issues/276), [gocryptfs commit ec74d1d](https://github.com/rfjakob/gocryptfs/commit/ec74d1d2f4217a9a337d1db9902f32ae2aecaf33)) -v1.8.0, 2020-05-09 +#### v1.8.0, 2020-05-09 * Enable ACL support ([#453](https://github.com/rfjakob/gocryptfs/issues/453)) * **Warning 2021-02-07**: This feature is incomplete! Do not use ACLs before gocryptfs v2.0 final! Reading and writing ACLs works, but they are not enforced or inherited ([#542](https://github.com/rfjakob/gocryptfs/issues/542)) @@ -317,7 +317,7 @@ v1.8.0, 2020-05-09 * Has been disabled since v1.7 due to issues a third-party module. * Please use FIDO2 instead (gocryptfs v2.0) -v1.7.1, 2019-10-06 +#### v1.7.1, 2019-10-06 * Support wild cards in reverse mode via `--exclude-wildcard` ([#367](https://github.com/rfjakob/gocryptfs/pull/367)). Thanks @ekalin! * Create `gocryptfs.diriv` files with 0440 permissions to make it easier to @@ -338,7 +338,7 @@ v1.7.1, 2019-10-06 * tests: use /var/tmp instead of /tmp by default ([commit 8c4429](https://github.com/rfjakob/gocryptfs/commit/8c4429408716d9890a98a48c246d616dbfea7e31)) -v1.7, 2019-03-17 +#### v1.7, 2019-03-17 * **Fix possible symlink race attacks in forward mode** when using allow_other + plaintextnames * If you use *both* `-allow_other` *and* `-plaintextnames`, you should upgrade. Malicious users could trick gocryptfs into modifying files outside of `CIPHERDIR`, @@ -371,11 +371,11 @@ v1.7, 2019-03-17 * Trezor support has been broken since Sept 2018 due to issues in a third-party module ([#261](https://github.com/rfjakob/gocryptfs/issues/261)) -v1.6.1, 2018-12-12 +#### v1.6.1, 2018-12-12 * Fix "Operation not supported" chmod errors on Go 1.11 ([#271](https://github.com/rfjakob/gocryptfs/issues/271)) -v1.6, 2018-08-18 +#### v1.6, 2018-08-18 * **Add `-e` / `-exclude` option** for reverse mode ([#235](https://github.com/rfjakob/gocryptfs/issues/235), [commit](https://github.com/rfjakob/gocryptfs/commit/ec2fdc19cf9358ae7ba09c528a5807b6b0760f9b)) @@ -389,7 +389,7 @@ v1.6, 2018-08-18 * Fall back to buffered IO even when passed `O_DIRECT` ([commit](https://github.com/rfjakob/gocryptfs/commit/893e41149ed353f355047003b89eeff456990e76)) -v1.5, 2018-06-12 +#### v1.5, 2018-06-12 * **Support extended attributes (xattr)** in forward mode ([#217](https://github.com/rfjakob/gocryptfs/issues/217)). Older gocryptfs versions will ignore the extended attributes. @@ -409,7 +409,7 @@ v1.5, 2018-06-12 * Stop printing the help text on a "flag provided but not defined" error ([commit](https://github.com/rfjakob/gocryptfs/commit/5ad26495fc86527bbfe75ac6b46528d49a373676)) -v1.4.4, 2018-03-18 +#### v1.4.4, 2018-03-18 * Overwrite secrets in memory with zeros as soon as possible ([#211](https://github.com/rfjakob/gocryptfs/issues/211)) * Fix Getdents problems on i386 and mips64le @@ -422,7 +422,7 @@ v1.4.4, 2018-03-18 [commit](https://github.com/hanwen/go-fuse/commit/a9ddcb8a4b609500fc59c89ccc9ee05f00a5fefd)) * Fix various test issues on MacOS -v1.4.3, 2018-01-21 +#### v1.4.3, 2018-01-21 * **Fix several symlink race attacks** in connection with reverse mode and allow_other. Thanks to @slackner for reporting and helping to fix the issues: @@ -440,7 +440,7 @@ v1.4.3, 2018-01-21 * MacOS: let OSXFuse create the mountpoint if it does not exist ([issue #194](https://github.com/rfjakob/gocryptfs/issues/194)) -v1.4.2, 2017-11-01 +#### v1.4.2, 2017-11-01 * Add `Gopkg.toml` file for `dep` vendoring and reproducible builds ([issue #142](https://github.com/rfjakob/gocryptfs/issues/142)) * MacOS: deal with `.DS_Store` files inside CIPHERDIR @@ -453,7 +453,7 @@ v1.4.2, 2017-11-01 * Fix a startup hang when `$PATH` contains the mountpoint ([issue #146](https://github.com/rfjakob/gocryptfs/issues/146)) -v1.4.1, 2017-08-21 +#### v1.4.1, 2017-08-21 * **Use memory pools for buffer handling** ( [3c6fe98](https://github.com/rfjakob/gocryptfs/commit/3c6fe98), [b2a23e9](https://github.com/rfjakob/gocryptfs/commit/b2a23e9), @@ -477,7 +477,7 @@ v1.4.1, 2017-08-21 * Enable writing to write-only files ([issue #125](https://github.com/rfjakob/gocryptfs/issues/125)) -v1.4, 2017-06-20 +#### v1.4, 2017-06-20 * **Switch to static binary releases** * From gocryptfs v1.4, I will only release statically-built binaries. These support all Linux distributions but cannot use OpenSSL. @@ -502,7 +502,7 @@ v1.4, 2017-06-20 ([commit 80516ed](https://github.com/rfjakob/gocryptfs/commit/80516ed3351477793eec882508969b6b29b69b0a)) * Add `-info` option to pretty-print infos about a filesystem. -v1.3, 2017-04-29 +#### v1.3, 2017-04-29 * **Use HKDF to derive separate keys for GCM and EME** * New feature flag: `HKDF` (enabled by default) * This is a forwards-compatible change. gocryptfs v1.3 can mount @@ -525,14 +525,14 @@ v1.3, 2017-04-29 that were compiled without Large File Support. * Passing "--" now also blocks "-o" parsing -v1.2.1, 2017-02-26 +#### v1.2.1, 2017-02-26 * Add an integrated speed test, `gocryptfs -speed` * Limit password size to 1000 bytes and reject trailing garbage after the newline * Make the test suite work on [Mac OS X](https://github.com/rfjakob/gocryptfs/issues/15) * Handle additional corner cases in `-ctlsock` path sanitization * Use dedicated exit code 12 on "password incorrect" -v1.2, 2016-12-04 +#### v1.2, 2016-12-04 * Add a control socket interface. Allows to encrypt and decrypt filenames. For details see [backintime#644](https://github.com/bit-team/backintime/issues/644#issuecomment-259835183). * New command-line option: `-ctlsock` @@ -551,14 +551,14 @@ v1.2, 2016-12-04 * Preserve owner for symlinks an device files (fixes bug [#64](https://github.com/rfjakob/gocryptfs/issues/64)) * Include rendered man page `gocryptfs.1` in the release tarball -v1.1.1, 2016-10-30 +#### v1.1.1, 2016-10-30 * Fix a panic on setting file timestamps ([go-fuse#131](https://github.com/hanwen/go-fuse/pull/131)) * Work around an issue in tmpfs that caused a panic in xfstests generic/075 ([gocryptfs#56](https://github.com/rfjakob/gocryptfs/issues/56)) * Optimize NFS streaming writes ([commit](https://github.com/rfjakob/gocryptfs/commit/a08d55f42d5b11e265a8617bee16babceebfd026)) -v1.1, 2016-10-19 +#### v1.1, 2016-10-19 * **Add reverse mode ([#19](https://github.com/rfjakob/gocryptfs/issues/19))** * AES-SIV (RFC5297) encryption to implement deterministic encryption securely. Uses the excellent @@ -584,7 +584,7 @@ v1.1, 2016-10-19 * Enable changing the password when you only know the master key ([#28](https://github.com/rfjakob/gocryptfs/issues/28)) -v1.0, 2016-07-17 +#### v1.0, 2016-07-17 * Deprecate very old filesystems, stage 3/3 * Filesystems created by v0.6 can no longer be mounted * Drop command-line options `-gcmiv128`, `-emenames`, `-diriv`. These @@ -598,7 +598,7 @@ v1.0, 2016-07-17 * Experimental Mac OS X support. See [ticket #15](https://github.com/rfjakob/gocryptfs/issues/15) for details. -v0.12, 2016-06-19 +#### v0.12, 2016-06-19 * Deprecate very old filesystems, stage 2/3 * Filesystems created by v0.6 and older can only be mounted read-only * A [message](https://github.com/rfjakob/gocryptfs/blob/v0.12/internal/configfile/config_file.go#L120) @@ -607,7 +607,7 @@ v0.12, 2016-06-19 * Mounts the filesystem read-only * Accept password from stdin as well ([ticket #30](https://github.com/rfjakob/gocryptfs/issues/30)) -v0.11, 2016-06-10 +#### v0.11, 2016-06-10 * Deprecate very old filesystems, stage 1/3 * Filesystems created by v0.6 and older can still be mounted but a [warning](https://github.com/rfjakob/gocryptfs/blob/v0.11/internal/configfile/config_file.go#L120) @@ -619,7 +619,7 @@ v0.11, 2016-06-10 * Build release binaries with Go 1.6.2 * Big speedup for CPUs with AES-NI, see [ticket #23](https://github.com/rfjakob/gocryptfs/issues/23) -v0.10, 2016-05-30 +#### v0.10, 2016-05-30 * **Replace `spacemonkeygo/openssl` with `stupidgcm`** * gocryptfs now has its own thin wrapper to OpenSSL's GCM implementation called `stupidgcm`. @@ -639,7 +639,7 @@ v0.10, 2016-05-30 * Fix a fsstress [failure](https://github.com/hanwen/go-fuse/issues/106) in the go-fuse library. -v0.9, 2016-04-10 +#### v0.9, 2016-04-10 * **Long file name support** * gocryptfs now supports file names up to 255 characters. * This is a forwards-compatible change. gocryptfs v0.9 can mount filesystems @@ -652,25 +652,25 @@ v0.9, 2016-04-10 * `-d`: Alias for `-debug` * `-q`: Alias for `-quiet` -v0.8, 2016-01-23 +#### v0.8, 2016-01-23 * Redirect output to syslog when running in the background * New command-line option: * `-memprofile`: Write a memory allocation debugging profile the specified file -v0.7.2, 2016-01-19 +#### v0.7.2, 2016-01-19 * **Fix performance issue in small file creation** * This brings performance on-par with EncFS paranoia mode, with streaming writes significantly faster * The actual [fix](https://github.com/hanwen/go-fuse/commit/c4b6b7949716d13eec856baffc7b7941ae21778c) is in the go-fuse library. There are no code changes in gocryptfs. -v0.7.1, 2016-01-09 +#### v0.7.1, 2016-01-09 * Make the `build.bash` script compatible with Go 1.3 * Disable fallocate on OSX (system call not available) * Introduce pre-built binaries for Fedora 23 and Debian 8 -v0.7, 2015-12-20 +#### v0.7, 2015-12-20 * **Extend GCM IV size to 128 bit from Go's default of 96 bit** * This pushes back the birthday bound to make IV collisions virtually impossible @@ -679,7 +679,7 @@ v0.7, 2015-12-20 * New command-line option: * `-gcmiv128`: Use 128-bit GCM IVs (default true) -v0.6, 2015-12-08 +#### v0.6, 2015-12-08 * **Wide-block filename encryption using EME + DirIV** * EME (ECB-Mix-ECB) provides even better security than CBC as it fixes the prefix leak. The used Go EME implementation is @@ -690,11 +690,11 @@ v0.6, 2015-12-08 * New command-line option: * `-emenames`: Enable EME filename encryption (default true) -v0.5.1, 2015-12-06 +#### v0.5.1, 2015-12-06 * Fix a rename regression caused by DirIV and add test case * Use fallocate to guard against out-of-space errors -v0.5, 2015-12-04 +#### v0.5, 2015-12-04 * **Stronger filename encryption: DirIV** * Each directory gets a random 128 bit file name IV on creation, stored in `gocryptfs.diriv` @@ -710,7 +710,7 @@ v0.5, 2015-12-04 can be used for faster mounting at the cost of lower brute-force resistance. It was mainly added to speed up the automated tests. -v0.4, 2015-11-15 +#### v0.4, 2015-11-15 * New command-line options: * `-plaintextnames`: disables filename encryption, added on user request * `-extpass`: calls an external program for prompting for the password @@ -721,15 +721,15 @@ v0.4, 2015-11-15 format changes. The first user is `-plaintextnames`. * On-disk format 2 -v0.3, 2015-11-01 +#### v0.3, 2015-11-01 * **Add a random 128 bit file header to authenticate file->block ownership** * This is an on-disk-format change * On-disk format 1 -v0.2, 2015-10-11 +#### v0.2, 2015-10-11 * Replace bash daemonization wrapper with native Go implementation * Better user feedback on mount failures -v0.1, 2015-10-07 +#### v0.1, 2015-10-07 * First release * On-disk format 0 From 5e67e183c08a19652480dce375f3178f0b9e16c9 Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Sat, 25 Sep 2021 16:45:36 +0200 Subject: [PATCH 59/59] README: set v2.2.0 release date --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index af26567..9bc33ec 100644 --- a/README.md +++ b/README.md @@ -196,7 +196,7 @@ RM: 2,367 Changelog --------- -#### v2.2.0, IN PROGRESS +#### v2.2.0, 2021-09-25 * **`-deterministic-names`: new option for `-init`**, both for reverse and forward mode. Disables file name randomisation & `gocryptfs.diriv` files ([#151](https://github.com/rfjakob/gocryptfs/issues/151), [#402](https://github.com/rfjakob/gocryptfs/issues/402), [#592](https://github.com/rfjakob/gocryptfs/pull/592))