fusefrontend: implement recursive diriv caching
The new contrib/maxlen.bash showed that we have exponential
runtime with respect to directory depth.
The new recursive diriv caching is a lot smarter as it caches
intermediate lookups. maxlen.bash now completes in a few seconds.
xfstests results same as
https://github.com/rfjakob/fuse-xfstests/blob/2d158e4c82be85c15269af77498e353f928f4fab/screenlog.0 :
Failures: generic/035 generic/062 generic/080 generic/093 generic/099 generic/215 generic/285 generic/319 generic/426 generic/444 generic/467 generic/477 generic/523
Failed 13 of 580 tests
benchmark.bash results are identical:
$ ./benchmark.bash
Testing gocryptfs at /tmp/benchmark.bash.BdQ: gocryptfs v2.0.1-17-g6b09bc0; go-fuse v2.1.1-0.20210611132105-24a1dfe6b4f8; 2021-06-25 go1.16.5 linux/amd64
/tmp/benchmark.bash.BdQ.mnt is a mountpoint
WRITE: 262144000 bytes (262 MB, 250 MiB) copied, 0,4821 s, 544 MB/s
READ: 262144000 bytes (262 MB, 250 MiB) copied, 0,266061 s, 985 MB/s
UNTAR: 8,280
MD5: 4,564
LS: 1,745
RM: 2,244
2021-06-25 11:33:18 +02:00
|
|
|
package fusefrontend
|
|
|
|
|
|
|
|
import (
|
|
|
|
"sync/atomic"
|
|
|
|
"syscall"
|
|
|
|
|
2021-08-23 15:05:15 +02:00
|
|
|
"github.com/rfjakob/gocryptfs/v2/internal/tlog"
|
fusefrontend: implement recursive diriv caching
The new contrib/maxlen.bash showed that we have exponential
runtime with respect to directory depth.
The new recursive diriv caching is a lot smarter as it caches
intermediate lookups. maxlen.bash now completes in a few seconds.
xfstests results same as
https://github.com/rfjakob/fuse-xfstests/blob/2d158e4c82be85c15269af77498e353f928f4fab/screenlog.0 :
Failures: generic/035 generic/062 generic/080 generic/093 generic/099 generic/215 generic/285 generic/319 generic/426 generic/444 generic/467 generic/477 generic/523
Failed 13 of 580 tests
benchmark.bash results are identical:
$ ./benchmark.bash
Testing gocryptfs at /tmp/benchmark.bash.BdQ: gocryptfs v2.0.1-17-g6b09bc0; go-fuse v2.1.1-0.20210611132105-24a1dfe6b4f8; 2021-06-25 go1.16.5 linux/amd64
/tmp/benchmark.bash.BdQ.mnt is a mountpoint
WRITE: 262144000 bytes (262 MB, 250 MiB) copied, 0,4821 s, 544 MB/s
READ: 262144000 bytes (262 MB, 250 MiB) copied, 0,266061 s, 985 MB/s
UNTAR: 8,280
MD5: 4,564
LS: 1,745
RM: 2,244
2021-06-25 11:33:18 +02:00
|
|
|
|
|
|
|
"github.com/hanwen/go-fuse/v2/fs"
|
|
|
|
|
2021-08-23 15:05:15 +02:00
|
|
|
"github.com/rfjakob/gocryptfs/v2/internal/syscallcompat"
|
fusefrontend: implement recursive diriv caching
The new contrib/maxlen.bash showed that we have exponential
runtime with respect to directory depth.
The new recursive diriv caching is a lot smarter as it caches
intermediate lookups. maxlen.bash now completes in a few seconds.
xfstests results same as
https://github.com/rfjakob/fuse-xfstests/blob/2d158e4c82be85c15269af77498e353f928f4fab/screenlog.0 :
Failures: generic/035 generic/062 generic/080 generic/093 generic/099 generic/215 generic/285 generic/319 generic/426 generic/444 generic/467 generic/477 generic/523
Failed 13 of 580 tests
benchmark.bash results are identical:
$ ./benchmark.bash
Testing gocryptfs at /tmp/benchmark.bash.BdQ: gocryptfs v2.0.1-17-g6b09bc0; go-fuse v2.1.1-0.20210611132105-24a1dfe6b4f8; 2021-06-25 go1.16.5 linux/amd64
/tmp/benchmark.bash.BdQ.mnt is a mountpoint
WRITE: 262144000 bytes (262 MB, 250 MiB) copied, 0,4821 s, 544 MB/s
READ: 262144000 bytes (262 MB, 250 MiB) copied, 0,266061 s, 985 MB/s
UNTAR: 8,280
MD5: 4,564
LS: 1,745
RM: 2,244
2021-06-25 11:33:18 +02:00
|
|
|
)
|
|
|
|
|
|
|
|
// prepareAtSyscall returns a (dirfd, cName) pair that can be used
|
|
|
|
// with the "___at" family of system calls (openat, fstatat, unlinkat...) to
|
|
|
|
// access the backing encrypted child file.
|
|
|
|
func (n *Node) prepareAtSyscall(child string) (dirfd int, cName string, errno syscall.Errno) {
|
|
|
|
if child == "" {
|
|
|
|
tlog.Warn.Printf("BUG: prepareAtSyscall: child=%q, should have called prepareAtSyscallMyself", child)
|
|
|
|
return n.prepareAtSyscallMyself()
|
|
|
|
}
|
|
|
|
|
|
|
|
rn := n.rootNode()
|
|
|
|
|
|
|
|
// All filesystem operations go through here, so this is a good place
|
|
|
|
// to reset the idle marker.
|
|
|
|
atomic.StoreUint32(&rn.IsIdle, 0)
|
|
|
|
|
|
|
|
if n.IsRoot() && rn.isFiltered(child) {
|
|
|
|
return -1, "", syscall.EPERM
|
|
|
|
}
|
|
|
|
|
|
|
|
var encryptName func(int, string, []byte) (string, error)
|
|
|
|
if !rn.args.PlaintextNames {
|
|
|
|
encryptName = func(dirfd int, child string, iv []byte) (cName string, err error) {
|
|
|
|
// Badname allowed, try to determine filenames
|
|
|
|
if rn.nameTransform.HaveBadnamePatterns() {
|
|
|
|
return rn.nameTransform.EncryptAndHashBadName(child, iv, dirfd)
|
|
|
|
}
|
|
|
|
return rn.nameTransform.EncryptAndHashName(child, iv)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Cache lookup
|
|
|
|
var iv []byte
|
|
|
|
dirfd, iv = rn.dirCache.Lookup(n)
|
|
|
|
if dirfd > 0 {
|
|
|
|
if rn.args.PlaintextNames {
|
|
|
|
return dirfd, child, 0
|
|
|
|
}
|
|
|
|
var err error
|
|
|
|
cName, err = encryptName(dirfd, child, iv)
|
|
|
|
if err != nil {
|
|
|
|
syscall.Close(dirfd)
|
|
|
|
return -1, "", fs.ToErrno(err)
|
|
|
|
}
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
// Slowpath: Open ourselves & read diriv
|
|
|
|
parentDirfd, myCName, errno := n.prepareAtSyscallMyself()
|
|
|
|
if errno != 0 {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
defer syscall.Close(parentDirfd)
|
|
|
|
|
|
|
|
dirfd, err := syscallcompat.Openat(parentDirfd, myCName, syscall.O_NOFOLLOW|syscall.O_DIRECTORY|syscallcompat.O_PATH, 0)
|
2021-07-31 10:53:32 +02:00
|
|
|
if err != nil {
|
|
|
|
return -1, "", fs.ToErrno(err)
|
|
|
|
}
|
fusefrontend: implement recursive diriv caching
The new contrib/maxlen.bash showed that we have exponential
runtime with respect to directory depth.
The new recursive diriv caching is a lot smarter as it caches
intermediate lookups. maxlen.bash now completes in a few seconds.
xfstests results same as
https://github.com/rfjakob/fuse-xfstests/blob/2d158e4c82be85c15269af77498e353f928f4fab/screenlog.0 :
Failures: generic/035 generic/062 generic/080 generic/093 generic/099 generic/215 generic/285 generic/319 generic/426 generic/444 generic/467 generic/477 generic/523
Failed 13 of 580 tests
benchmark.bash results are identical:
$ ./benchmark.bash
Testing gocryptfs at /tmp/benchmark.bash.BdQ: gocryptfs v2.0.1-17-g6b09bc0; go-fuse v2.1.1-0.20210611132105-24a1dfe6b4f8; 2021-06-25 go1.16.5 linux/amd64
/tmp/benchmark.bash.BdQ.mnt is a mountpoint
WRITE: 262144000 bytes (262 MB, 250 MiB) copied, 0,4821 s, 544 MB/s
READ: 262144000 bytes (262 MB, 250 MiB) copied, 0,266061 s, 985 MB/s
UNTAR: 8,280
MD5: 4,564
LS: 1,745
RM: 2,244
2021-06-25 11:33:18 +02:00
|
|
|
|
|
|
|
// Cache store
|
|
|
|
if !rn.args.PlaintextNames {
|
|
|
|
var err error
|
2021-08-20 10:57:26 +02:00
|
|
|
iv, err = rn.nameTransform.ReadDirIVAt(dirfd)
|
fusefrontend: implement recursive diriv caching
The new contrib/maxlen.bash showed that we have exponential
runtime with respect to directory depth.
The new recursive diriv caching is a lot smarter as it caches
intermediate lookups. maxlen.bash now completes in a few seconds.
xfstests results same as
https://github.com/rfjakob/fuse-xfstests/blob/2d158e4c82be85c15269af77498e353f928f4fab/screenlog.0 :
Failures: generic/035 generic/062 generic/080 generic/093 generic/099 generic/215 generic/285 generic/319 generic/426 generic/444 generic/467 generic/477 generic/523
Failed 13 of 580 tests
benchmark.bash results are identical:
$ ./benchmark.bash
Testing gocryptfs at /tmp/benchmark.bash.BdQ: gocryptfs v2.0.1-17-g6b09bc0; go-fuse v2.1.1-0.20210611132105-24a1dfe6b4f8; 2021-06-25 go1.16.5 linux/amd64
/tmp/benchmark.bash.BdQ.mnt is a mountpoint
WRITE: 262144000 bytes (262 MB, 250 MiB) copied, 0,4821 s, 544 MB/s
READ: 262144000 bytes (262 MB, 250 MiB) copied, 0,266061 s, 985 MB/s
UNTAR: 8,280
MD5: 4,564
LS: 1,745
RM: 2,244
2021-06-25 11:33:18 +02:00
|
|
|
if err != nil {
|
|
|
|
syscall.Close(dirfd)
|
|
|
|
return -1, "", fs.ToErrno(err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
rn.dirCache.Store(n, dirfd, iv)
|
|
|
|
|
|
|
|
if rn.args.PlaintextNames {
|
|
|
|
return dirfd, child, 0
|
|
|
|
}
|
|
|
|
|
|
|
|
cName, err = encryptName(dirfd, child, iv)
|
|
|
|
if err != nil {
|
|
|
|
syscall.Close(dirfd)
|
|
|
|
return -1, "", fs.ToErrno(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
func (n *Node) prepareAtSyscallMyself() (dirfd int, cName string, errno syscall.Errno) {
|
|
|
|
dirfd = -1
|
|
|
|
|
|
|
|
// Handle root node
|
|
|
|
if n.IsRoot() {
|
|
|
|
var err error
|
|
|
|
rn := n.rootNode()
|
2021-06-26 18:44:52 +02:00
|
|
|
// Open cipherdir (following symlinks)
|
|
|
|
dirfd, err = syscallcompat.Open(rn.args.Cipherdir, syscall.O_DIRECTORY|syscallcompat.O_PATH, 0)
|
fusefrontend: implement recursive diriv caching
The new contrib/maxlen.bash showed that we have exponential
runtime with respect to directory depth.
The new recursive diriv caching is a lot smarter as it caches
intermediate lookups. maxlen.bash now completes in a few seconds.
xfstests results same as
https://github.com/rfjakob/fuse-xfstests/blob/2d158e4c82be85c15269af77498e353f928f4fab/screenlog.0 :
Failures: generic/035 generic/062 generic/080 generic/093 generic/099 generic/215 generic/285 generic/319 generic/426 generic/444 generic/467 generic/477 generic/523
Failed 13 of 580 tests
benchmark.bash results are identical:
$ ./benchmark.bash
Testing gocryptfs at /tmp/benchmark.bash.BdQ: gocryptfs v2.0.1-17-g6b09bc0; go-fuse v2.1.1-0.20210611132105-24a1dfe6b4f8; 2021-06-25 go1.16.5 linux/amd64
/tmp/benchmark.bash.BdQ.mnt is a mountpoint
WRITE: 262144000 bytes (262 MB, 250 MiB) copied, 0,4821 s, 544 MB/s
READ: 262144000 bytes (262 MB, 250 MiB) copied, 0,266061 s, 985 MB/s
UNTAR: 8,280
MD5: 4,564
LS: 1,745
RM: 2,244
2021-06-25 11:33:18 +02:00
|
|
|
if err != nil {
|
2021-06-26 18:44:52 +02:00
|
|
|
return -1, "", fs.ToErrno(err)
|
fusefrontend: implement recursive diriv caching
The new contrib/maxlen.bash showed that we have exponential
runtime with respect to directory depth.
The new recursive diriv caching is a lot smarter as it caches
intermediate lookups. maxlen.bash now completes in a few seconds.
xfstests results same as
https://github.com/rfjakob/fuse-xfstests/blob/2d158e4c82be85c15269af77498e353f928f4fab/screenlog.0 :
Failures: generic/035 generic/062 generic/080 generic/093 generic/099 generic/215 generic/285 generic/319 generic/426 generic/444 generic/467 generic/477 generic/523
Failed 13 of 580 tests
benchmark.bash results are identical:
$ ./benchmark.bash
Testing gocryptfs at /tmp/benchmark.bash.BdQ: gocryptfs v2.0.1-17-g6b09bc0; go-fuse v2.1.1-0.20210611132105-24a1dfe6b4f8; 2021-06-25 go1.16.5 linux/amd64
/tmp/benchmark.bash.BdQ.mnt is a mountpoint
WRITE: 262144000 bytes (262 MB, 250 MiB) copied, 0,4821 s, 544 MB/s
READ: 262144000 bytes (262 MB, 250 MiB) copied, 0,266061 s, 985 MB/s
UNTAR: 8,280
MD5: 4,564
LS: 1,745
RM: 2,244
2021-06-25 11:33:18 +02:00
|
|
|
}
|
2021-06-26 18:44:52 +02:00
|
|
|
return dirfd, ".", 0
|
fusefrontend: implement recursive diriv caching
The new contrib/maxlen.bash showed that we have exponential
runtime with respect to directory depth.
The new recursive diriv caching is a lot smarter as it caches
intermediate lookups. maxlen.bash now completes in a few seconds.
xfstests results same as
https://github.com/rfjakob/fuse-xfstests/blob/2d158e4c82be85c15269af77498e353f928f4fab/screenlog.0 :
Failures: generic/035 generic/062 generic/080 generic/093 generic/099 generic/215 generic/285 generic/319 generic/426 generic/444 generic/467 generic/477 generic/523
Failed 13 of 580 tests
benchmark.bash results are identical:
$ ./benchmark.bash
Testing gocryptfs at /tmp/benchmark.bash.BdQ: gocryptfs v2.0.1-17-g6b09bc0; go-fuse v2.1.1-0.20210611132105-24a1dfe6b4f8; 2021-06-25 go1.16.5 linux/amd64
/tmp/benchmark.bash.BdQ.mnt is a mountpoint
WRITE: 262144000 bytes (262 MB, 250 MiB) copied, 0,4821 s, 544 MB/s
READ: 262144000 bytes (262 MB, 250 MiB) copied, 0,266061 s, 985 MB/s
UNTAR: 8,280
MD5: 4,564
LS: 1,745
RM: 2,244
2021-06-25 11:33:18 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// Otherwise convert to prepareAtSyscall of parent node
|
|
|
|
myName, p1 := n.Parent()
|
|
|
|
if p1 == nil || myName == "" {
|
|
|
|
errno = syscall.ENOENT
|
|
|
|
return
|
|
|
|
}
|
|
|
|
parent := toNode(p1.Operations())
|
|
|
|
return parent.prepareAtSyscall(myName)
|
|
|
|
}
|