Add single-element cache for DirIV lookup

Another 3x performance boost for applications that walk the
directory tree.

Excerpt from performance.txt:

VERSION         UNTAR    LS     RM
v0.4               48     1.5    5
v0.5-rc1           56     7     19
v0.5-rc1-1         54     4.1    9
v0.5-rc1-2         45     1.7	 3.4  <---- THIS VERSION
This commit is contained in:
Jakob Unterwurzacher 2015-11-29 21:41:38 +01:00
parent 1d0a442405
commit 20b058a333
5 changed files with 74 additions and 7 deletions

View File

@ -26,6 +26,8 @@ type CryptFS struct {
// Stores an all-zero block of size cipherBS // Stores an all-zero block of size cipherBS
allZeroBlock []byte allZeroBlock []byte
plaintextNames bool plaintextNames bool
// DirIV cache for filename encryption
DirIVCacheEnc DirIVCache
} }
func NewCryptFS(key []byte, useOpenssl bool, plaintextNames bool) *CryptFS { func NewCryptFS(key []byte, useOpenssl bool, plaintextNames bool) *CryptFS {

View File

@ -1,10 +1,10 @@
package cryptfs package cryptfs
import ( import (
"os"
"math"
"fmt" "fmt"
"golang.org/x/crypto/scrypt" "golang.org/x/crypto/scrypt"
"math"
"os"
) )
const ( const (
@ -50,5 +50,5 @@ func (s *scryptKdf) DeriveKey(pw string) []byte {
// LogN - N is saved as 2^LogN, but LogN is much easier to work with. // LogN - N is saved as 2^LogN, but LogN is much easier to work with.
// This function gives you LogN = Log2(N). // This function gives you LogN = Log2(N).
func (s *scryptKdf) LogN() int { func (s *scryptKdf) LogN() int {
return int(math.Log2(float64(s.N))+0.5) return int(math.Log2(float64(s.N)) + 0.5)
} }

View File

@ -5,8 +5,47 @@ import (
"io/ioutil" "io/ioutil"
"path/filepath" "path/filepath"
"strings" "strings"
"sync"
) )
// A simple one-entry DirIV cache
type DirIVCache struct {
// Invalidated?
cleared bool
// The DirIV
iv []byte
// Directory the DirIV belongs to
dir string
// Ecrypted version of "dir"
translatedDir string
// Synchronisation
lock sync.RWMutex
}
func (c *DirIVCache) lookup(dir string) (bool, []byte, string) {
c.lock.RLock()
defer c.lock.RUnlock()
if !c.cleared && c.dir == dir {
return true, c.iv, c.translatedDir
}
return false, nil, ""
}
func (c *DirIVCache) store(dir string, iv []byte, translatedDir string) {
c.lock.Lock()
defer c.lock.Unlock()
c.cleared = false
c.iv = iv
c.dir = dir
c.translatedDir = translatedDir
}
func (c *DirIVCache) Clear() {
c.lock.Lock()
defer c.lock.Unlock()
c.cleared = true
}
// readDirIV - read the "gocryptfs.diriv" file from "dir" (absolute ciphertext path) // readDirIV - read the "gocryptfs.diriv" file from "dir" (absolute ciphertext path)
func (be *CryptFS) ReadDirIV(dir string) (iv []byte, err error) { func (be *CryptFS) ReadDirIV(dir string) (iv []byte, err error) {
ivfile := filepath.Join(dir, DIRIV_FILENAME) ivfile := filepath.Join(dir, DIRIV_FILENAME)
@ -41,11 +80,23 @@ func (be *CryptFS) EncryptPathDirIV(plainPath string, rootDir string) (string, e
if plainPath == "" { if plainPath == "" {
return plainPath, nil return plainPath, nil
} }
// Check if the DirIV is cached
parentDir := filepath.Dir(plainPath)
found, iv, cParentDir := be.DirIVCacheEnc.lookup(parentDir)
if found {
//fmt.Print("h")
baseName := filepath.Base(plainPath)
cBaseName := be.encryptName(baseName, iv)
cPath := cParentDir + "/" + cBaseName
return cPath, nil
}
// Walk the directory tree
var wd = rootDir var wd = rootDir
var encryptedNames []string var encryptedNames []string
var err error
plainNames := strings.Split(plainPath, "/") plainNames := strings.Split(plainPath, "/")
for _, plainName := range plainNames { for _, plainName := range plainNames {
iv, err := be.ReadDirIV(wd) iv, err = be.ReadDirIV(wd)
if err != nil { if err != nil {
return "", err return "", err
} }
@ -53,7 +104,11 @@ func (be *CryptFS) EncryptPathDirIV(plainPath string, rootDir string) (string, e
encryptedNames = append(encryptedNames, encryptedName) encryptedNames = append(encryptedNames, encryptedName)
wd = filepath.Join(wd, encryptedName) wd = filepath.Join(wd, encryptedName)
} }
return filepath.Join(encryptedNames...), nil // Cache the final DirIV
cPath := strings.Join(encryptedNames, "/")
cParentDir = filepath.Dir(cPath)
be.DirIVCacheEnc.store(parentDir, iv, cParentDir)
return cPath, nil
} }
// DecryptPathDirIV - encrypt path using CBC with DirIV // DecryptPathDirIV - encrypt path using CBC with DirIV

View File

@ -251,6 +251,8 @@ func (fs *FS) Mkdir(relPath string, mode uint32, context *fuse.Context) (code fu
if err != nil { if err != nil {
return fuse.ToStatus(err) return fuse.ToStatus(err)
} }
// The new directory may take the place of an older one that is still in the cache
fs.CryptFS.DirIVCacheEnc.Clear()
// Create directory // Create directory
fs.dirIVLock.Lock() fs.dirIVLock.Lock()
defer fs.dirIVLock.Unlock() defer fs.dirIVLock.Unlock()
@ -315,7 +317,9 @@ func (fs *FS) Rmdir(name string, context *fuse.Context) (code fuse.Status) {
tmpName := fmt.Sprintf("gocryptfs.diriv.rmdir.%d", st.Ino) tmpName := fmt.Sprintf("gocryptfs.diriv.rmdir.%d", st.Ino)
tmpDirivPath := filepath.Join(parentDir, tmpName) tmpDirivPath := filepath.Join(parentDir, tmpName)
cryptfs.Debug.Printf("Rmdir: Renaming %s to %s\n", cryptfs.DIRIV_FILENAME, tmpDirivPath) cryptfs.Debug.Printf("Rmdir: Renaming %s to %s\n", cryptfs.DIRIV_FILENAME, tmpDirivPath)
fs.dirIVLock.Lock() // directory will be in an inconsistent state after the rename // The directory is in an inconsistent state between rename and rmdir. Protect against
// concurrent readers.
fs.dirIVLock.Lock()
defer fs.dirIVLock.Unlock() defer fs.dirIVLock.Unlock()
err = os.Rename(dirivPath, tmpDirivPath) err = os.Rename(dirivPath, tmpDirivPath)
if err != nil { if err != nil {
@ -338,7 +342,8 @@ func (fs *FS) Rmdir(name string, context *fuse.Context) (code fuse.Status) {
if err != nil { if err != nil {
cryptfs.Warn.Printf("Rmdir: Could not clean up %s: %v\n", tmpName, err) cryptfs.Warn.Printf("Rmdir: Could not clean up %s: %v\n", tmpName, err)
} }
// The now-deleted directory may have been in the DirIV cache. Clear it.
fs.CryptFS.DirIVCacheEnc.Clear()
return fuse.OK return fuse.OK
} }
@ -382,6 +387,10 @@ func (fs *FS) Rename(oldPath string, newPath string, context *fuse.Context) (cod
if err != nil { if err != nil {
return fuse.ToStatus(err) return fuse.ToStatus(err)
} }
// The Rename may cause a directory to take the place of another directory.
// That directory may still be in the DirIV cache, clear it.
fs.CryptFS.DirIVCacheEnc.Clear()
return fs.FileSystem.Rename(cOldPath, cNewPath, context) return fs.FileSystem.Rename(cOldPath, cNewPath, context)
} }

View File

@ -8,3 +8,4 @@ VERSION UNTAR LS RM
v0.4 48 1.5 5 v0.4 48 1.5 5
v0.5-rc1 56 7 19 v0.5-rc1 56 7 19
v0.5-rc1-1 54 4.1 9 v0.5-rc1-1 54 4.1 9
v0.5-rc1-2 45 1.7 3.4