Implement -deterministic-names: extended -zerodiriv

-deterministc-names uses all-zero dirivs but does not write
them to disk anymore.

cli_args.go

@@ -30,7 +30,7 @@ type argContainer struct {
 	plaintextnames, quiet, nosyslog, wpanic,
 	longnames, allow_other, reverse, aessiv, nonempty, raw64,
 	noprealloc, speed, hkdf, serialize_reads, forcedecode, hh, info,
-	sharedstorage, devrandom, fsck, one_file_system, zerodiriv bool
+	sharedstorage, devrandom, fsck, one_file_system, deterministic_names bool
 	// Mount options with opposites
 	dev, nodev, suid, nosuid, exec, noexec, rw, ro, kernel_cache, acl bool
 	masterkey, mountpoint, cipherdir, cpuprofile,
@@ -179,7 +179,7 @@ func parseCliOpts(osArgs []string) (args argContainer) {
 	flagSet.BoolVar(&args.devrandom, "devrandom", false, "Use /dev/random for generating master key")
 	flagSet.BoolVar(&args.fsck, "fsck", false, "Run a filesystem check on CIPHERDIR")
 	flagSet.BoolVar(&args.one_file_system, "one-file-system", false, "Don't cross filesystem boundaries")
-	flagSet.BoolVar(&args.zerodiriv, "zerodiriv", false, "Create diriv as all-zero files")
+	flagSet.BoolVar(&args.deterministic_names, "deterministic-names", false, "Disable diriv file name randomisation")
 
 	// Mount options with opposites
 	flagSet.BoolVar(&args.dev, "dev", false, "Allow device files")

init_dir.go

@@ -99,11 +99,11 @@ func initDir(args *argContainer) {
 	}
 	// Forward mode with filename encryption enabled needs a gocryptfs.diriv file
 	// in the root dir
-	if !args.plaintextnames && !args.reverse {
+	if !args.plaintextnames && !args.reverse && !args.deterministic_names {
 		// Open cipherdir (following symlinks)
 		dirfd, err := syscall.Open(args.cipherdir, syscall.O_DIRECTORY|syscallcompat.O_PATH, 0)
 		if err == nil {
-			err = nametransform.WriteDirIVAt(dirfd, !args.zerodiriv)
+			err = nametransform.WriteDirIVAt(dirfd)
 			syscall.Close(dirfd)
 		}
 		if err != nil {

internal/fusefrontend/args.go

@@ -53,6 +53,6 @@ type Args struct {
 	// like rsync's `--one-file-system` does.
 	// Only applicable to reverse mode.
 	OneFileSystem bool
-	// ZeroDirIV creates diriv files as all-zero files
-	ZeroDirIV bool
+	// DeterministicNames disables gocryptfs.diriv files
+	DeterministicNames bool
 }

internal/fusefrontend/ctlsock_interface.go

@@ -32,7 +32,7 @@ func (rn *RootNode) EncryptPath(plainPath string) (cipherPath string, err error)
 	parts := strings.Split(plainPath, "/")
 	wd := dirfd
 	for i, part := range parts {
-		dirIV, err := nametransform.ReadDirIVAt(wd)
+		dirIV, err := rn.nameTransform.ReadDirIVAt(wd)
 		if err != nil {
 			return "", err
 		}
@@ -78,7 +78,7 @@ func (rn *RootNode) DecryptPath(cipherPath string) (plainPath string, err error)
 	parts := strings.Split(cipherPath, "/")
 	wd := dirfd
 	for i, part := range parts {
-		dirIV, err := nametransform.ReadDirIVAt(wd)
+		dirIV, err := rn.nameTransform.ReadDirIVAt(wd)
 		if err != nil {
 			return "", err
 		}

internal/fusefrontend/node_dir_ops.go

@@ -34,9 +34,14 @@ func haveDsstore(entries []fuse.DirEntry) bool {
 // mkdirWithIv - create a new directory and corresponding diriv file. dirfd
 // should be a handle to the parent directory, cName is the name of the new
 // directory and mode specifies the access permissions to use.
+// If DeterministicNames is set, the diriv file is NOT created.
 func (n *Node) mkdirWithIv(dirfd int, cName string, mode uint32, context *fuse.Context) error {
-
 	rn := n.rootNode()
+
+	if rn.args.DeterministicNames {
+		return syscallcompat.MkdiratUser(dirfd, cName, mode, context)
+	}
+
 	// Between the creation of the directory and the creation of gocryptfs.diriv
 	// the directory is inconsistent. Take the lock to prevent other readers
 	// from seeing it.
@@ -49,7 +54,7 @@ func (n *Node) mkdirWithIv(dirfd int, cName string, mode uint32, context *fuse.C
 	dirfd2, err := syscallcompat.Openat(dirfd, cName, syscall.O_DIRECTORY|syscall.O_NOFOLLOW|syscallcompat.O_PATH, 0)
 	if err == nil {
 		// Create gocryptfs.diriv
-		err = nametransform.WriteDirIVAt(dirfd2, !rn.args.ZeroDirIV)
+		err = nametransform.WriteDirIVAt(dirfd2)
 		syscall.Close(dirfd2)
 	}
 	if err != nil {
@@ -90,7 +95,13 @@ func (n *Node) Mkdir(ctx context.Context, name string, mode uint32, out *fuse.En
 			return nil, fs.ToErrno(err)
 		}
 		st = syscallcompat.Unix2syscall(ust)
-	} else {
+
+		// Create child node & return
+		ch := n.newChild(ctx, &st, out)
+		return ch, 0
+
+	}
+
 	// We need write and execute permissions to create gocryptfs.diriv.
 	// Also, we need read permissions to open the directory (to avoid
 	// race-conditions between getting and setting the mode).
@@ -104,8 +115,7 @@ func (n *Node) Mkdir(ctx context.Context, name string, mode uint32, out *fuse.En
 		if err != nil {
 			return nil, fs.ToErrno(err)
 		}
-
-			// Create directory
+		// Create directory & rollback .name file on error
 		err = rn.mkdirWithIv(dirfd, cName, mode, context)
 		if err != nil {
 			nametransform.DeleteLongNameAt(dirfd, cName)
@@ -118,6 +128,7 @@ func (n *Node) Mkdir(ctx context.Context, name string, mode uint32, out *fuse.En
 		}
 	}
 
+	// Fill `st`
 	fd, err := syscallcompat.Openat(dirfd, cName,
 		syscall.O_RDONLY|syscall.O_DIRECTORY|syscall.O_NOFOLLOW, 0)
 	if err != nil {
@@ -140,12 +151,11 @@ func (n *Node) Mkdir(ctx context.Context, name string, mode uint32, out *fuse.En
 		if err != nil {
 			tlog.Warn.Printf("Mkdir %q: Fchmod %#o -> %#o failed: %v", cName, mode, origMode, err)
 		}
-		}
+
 	}
 
-	// Create child node
+	// Create child node & return
 	ch := n.newChild(ctx, &st, out)
-
 	return ch, 0
 }
 
@@ -175,7 +185,7 @@ func (n *Node) Readdir(ctx context.Context) (fs.DirStream, syscall.Errno) {
 	rn := n.rootNode()
 	if !rn.args.PlaintextNames {
 		// Read the DirIV from disk
-		cachedIV, err = nametransform.ReadDirIVAt(fd)
+		cachedIV, err = rn.nameTransform.ReadDirIVAt(fd)
 		if err != nil {
 			tlog.Warn.Printf("OpenDir %q: could not read %s: %v", cDirName, nametransform.DirIVFilename, err)
 			return nil, syscall.EIO
@@ -196,7 +206,7 @@ func (n *Node) Readdir(ctx context.Context) (fs.DirStream, syscall.Errno) {
 			plain = append(plain, cipherEntries[i])
 			continue
 		}
-		if cName == nametransform.DirIVFilename {
+		if !rn.args.DeterministicNames && cName == nametransform.DirIVFilename {
 			// silently ignore "gocryptfs.diriv" everywhere if dirIV is enabled
 			continue
 		}
@@ -249,6 +259,15 @@ func (n *Node) Rmdir(ctx context.Context, name string) (code syscall.Errno) {
 		err := unix.Unlinkat(parentDirFd, cName, unix.AT_REMOVEDIR)
 		return fs.ToErrno(err)
 	}
+	if rn.args.DeterministicNames {
+		if err := unix.Unlinkat(parentDirFd, cName, unix.AT_REMOVEDIR); err != nil {
+			return fs.ToErrno(err)
+		}
+		if nametransform.IsLongContent(cName) {
+			nametransform.DeleteLongNameAt(parentDirFd, cName)
+		}
+		return 0
+	}
 	// Unless we are running as root, we need read, write and execute permissions
 	// to handle gocryptfs.diriv.
 	permWorkaround := false

internal/fusefrontend/node_prepare_syscall.go

@@ -8,7 +8,6 @@ import (
 
 	"github.com/hanwen/go-fuse/v2/fs"
 
-	"github.com/rfjakob/gocryptfs/internal/nametransform"
 	"github.com/rfjakob/gocryptfs/internal/syscallcompat"
 )
 
@@ -73,7 +72,7 @@ func (n *Node) prepareAtSyscall(child string) (dirfd int, cName string, errno sy
 	// Cache store
 	if !rn.args.PlaintextNames {
 		var err error
-		iv, err = nametransform.ReadDirIVAt(dirfd)
+		iv, err = rn.nameTransform.ReadDirIVAt(dirfd)
 		if err != nil {
 			syscall.Close(dirfd)
 			return -1, "", fs.ToErrno(err)

internal/fusefrontend/xattr_unit_test.go

@@ -19,7 +19,7 @@ func newTestFS(args Args) *RootNode {
 	key := make([]byte, cryptocore.KeyLen)
 	cCore := cryptocore.New(key, cryptocore.BackendGoGCM, contentenc.DefaultIVBits, true, false)
 	cEnc := contentenc.New(cCore, contentenc.DefaultBS, false)
-	n := nametransform.New(cCore.EMECipher, true, true, nil)
+	n := nametransform.New(cCore.EMECipher, true, true, nil, false)
 	rn := NewRootNode(args, cEnc, n)
 	oneSec := time.Second
 	options := &fs.Options{

internal/nametransform/diriv.go

@@ -1,6 +1,7 @@
 package nametransform
 
 import (
+	"bytes"
 	"fmt"
 	"io"
 	"os"
@@ -22,7 +23,11 @@ const (
 // ReadDirIVAt reads "gocryptfs.diriv" from the directory that is opened as "dirfd".
 // Using the dirfd makes it immune to concurrent renames of the directory.
 // Retries on EINTR.
-func ReadDirIVAt(dirfd int) (iv []byte, err error) {
+// If deterministicNames is set it returns an all-zero slice.
+func (n *NameTransform) ReadDirIVAt(dirfd int) (iv []byte, err error) {
+	if n.deterministicNames {
+		return make([]byte, DirIVLen), nil
+	}
 	fdRaw, err := syscallcompat.Openat(dirfd, DirIVFilename,
 		syscall.O_RDONLY|syscall.O_NOFOLLOW, 0)
 	if err != nil {
@@ -33,6 +38,9 @@ func ReadDirIVAt(dirfd int) (iv []byte, err error) {
 	return fdReadDirIV(fd)
 }
 
+// allZeroDirIV is preallocated to quickly check if the data read from disk is all zero
+var allZeroDirIV = make([]byte, DirIVLen)
+
 // fdReadDirIV reads and verifies the DirIV from an opened gocryptfs.diriv file.
 func fdReadDirIV(fd *os.File) (iv []byte, err error) {
 	// We want to detect if the file is bigger than DirIVLen, so
@@ -46,6 +54,9 @@ func fdReadDirIV(fd *os.File) (iv []byte, err error) {
 	if len(iv) != DirIVLen {
 		return nil, fmt.Errorf("wanted %d bytes, got %d", DirIVLen, len(iv))
 	}
+	if bytes.Equal(iv, allZeroDirIV) {
+		return nil, fmt.Errorf("diriv is all-zero")
+	}
 	return iv, nil
 }
 
@@ -53,13 +64,8 @@ func fdReadDirIV(fd *os.File) (iv []byte, err error) {
 // "dirfd". On error we try to delete the incomplete file.
 // This function is exported because it is used from fusefrontend, main,
 // and also the automated tests.
-func WriteDirIVAt(dirfd int, randomInitialization bool) error {
-	var iv []byte
-	if randomInitialization {
-		iv = cryptocore.RandBytes(DirIVLen)
-	} else {
-		iv = make([]byte, DirIVLen)
-	}
+func WriteDirIVAt(dirfd int) error {
+	iv := cryptocore.RandBytes(DirIVLen)
 	// 0400 permissions: gocryptfs.diriv should never be modified after creation.
 	// Don't use "ioutil.WriteFile", it causes trouble on NFS:
 	// https://github.com/rfjakob/gocryptfs/commit/7d38f80a78644c8ec4900cc990bfb894387112ed

internal/nametransform/longnames.go

@@ -114,7 +114,7 @@ func ReadLongNameAt(dirfd int, cName string) (string, error) {
 func DeleteLongNameAt(dirfd int, hashName string) error {
 	err := syscallcompat.Unlinkat(dirfd, hashName+LongNameSuffix, 0)
 	if err != nil {
-		tlog.Warn.Printf("DeleteLongName: %v", err)
+		tlog.Warn.Printf("DeleteLongNameAt: %v", err)
 	}
 	return err
 }
@@ -128,7 +128,7 @@ func (n *NameTransform) WriteLongNameAt(dirfd int, hashName string, plainName st
 	plainName = filepath.Base(plainName)
 
 	// Encrypt the basename
-	dirIV, err := ReadDirIVAt(dirfd)
+	dirIV, err := n.ReadDirIVAt(dirfd)
 	if err != nil {
 		return err
 	}

internal/nametransform/names.go

@@ -26,10 +26,11 @@ type NameTransform struct {
 	B64 *base64.Encoding
 	// Patterns to bypass decryption
 	badnamePatterns    []string
+	deterministicNames bool
 }
 
 // New returns a new NameTransform instance.
-func New(e *eme.EMECipher, longNames bool, raw64 bool, badname []string) *NameTransform {
+func New(e *eme.EMECipher, longNames bool, raw64 bool, badname []string, deterministicNames bool) *NameTransform {
 	tlog.Debug.Printf("nametransform.New: longNames=%v, raw64=%v, badname=%q",
 		longNames, raw64, badname)
 
@@ -42,6 +43,7 @@ func New(e *eme.EMECipher, longNames bool, raw64 bool, badname []string) *NameTr
 		longNames:          longNames,
 		B64:                b64,
 		badnamePatterns:    badname,
+		deterministicNames: deterministicNames,
 	}
 }
 

mount.go

@@ -276,7 +276,7 @@ func initFuseFrontend(args *argContainer) (rootNode fs.InodeEmbedder, wipeKeys f
 		KernelCache:        args.kernel_cache,
 		SharedStorage:      args.sharedstorage,
 		OneFileSystem:      args.one_file_system,
-		ZeroDirIV:       args.zerodiriv,
+		DeterministicNames: args.deterministic_names,
 	}
 	// confFile is nil when "-zerokey" or "-masterkey" was used
 	if confFile != nil {
@@ -300,7 +300,8 @@ func initFuseFrontend(args *argContainer) (rootNode fs.InodeEmbedder, wipeKeys f
 	// Init crypto backend
 	cCore := cryptocore.New(masterkey, cryptoBackend, contentenc.DefaultIVBits, args.hkdf, args.forcedecode)
 	cEnc := contentenc.New(cCore, contentenc.DefaultBS, args.forcedecode)
-	nameTransform := nametransform.New(cCore.EMECipher, frontendArgs.LongNames, args.raw64, []string(args.badname))
+	nameTransform := nametransform.New(cCore.EMECipher, frontendArgs.LongNames,
+		args.raw64, []string(args.badname), frontendArgs.DeterministicNames)
 	// After the crypto backend is initialized,
 	// we can purge the master key from memory.
 	for i := range masterkey {

tests/deterministic_names/deterministic_names_test.go

@@ -0,0 +1,79 @@
+package deterministic_names
+
+// integration tests that target "-deterministic-names" specifically
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/rfjakob/gocryptfs/tests/test_helpers"
+)
+
+var cDir string
+var pDir string
+
+var testPw = []byte("test")
+
+// Create and mount "-deterministic-names" fs
+func TestMain(m *testing.M) {
+	cDir = test_helpers.InitFS(nil, "-deterministic-names")
+	pDir = cDir + ".mnt"
+	test_helpers.MountOrExit(cDir, pDir, "-deterministic-names", "-extpass", "echo test")
+	r := m.Run()
+	test_helpers.UnmountPanic(pDir)
+	os.Exit(r)
+}
+
+// TestDeterministicNames checks that a file with the same plaintext name
+// always encrypts to the same ciphertext name
+func TestDeterministicNames(t *testing.T) {
+	// "foo" should encrypt to the same name in both directories
+	if err := os.MkdirAll(pDir+"/x/foo", 0700); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.MkdirAll(pDir+"/y/foo", 0700); err != nil {
+		t.Fatal(err)
+	}
+	matches, err := filepath.Glob(cDir + "/*/*")
+	if err != nil || len(matches) != 2 {
+		t.Fatal(matches, err)
+	}
+	if filepath.Base(matches[0]) != filepath.Base(matches[1]) {
+		t.Error(matches)
+	}
+	fooEncrypted := filepath.Base(matches[0])
+
+	// "foo" should also encrypt to the same name in the root directory
+	if err := os.Mkdir(pDir+"/foo", 0700); err != nil {
+		t.Fatal(err)
+	}
+	_, err = os.Stat(cDir + "/" + fooEncrypted)
+	if err != nil {
+		t.Error(err)
+	}
+
+	// Replace directory with file
+	if err := os.RemoveAll(pDir + "/foo"); err != nil {
+		t.Fatal(err)
+	}
+	if err := ioutil.WriteFile(pDir+"/foo", nil, 0700); err != nil {
+		t.Fatal(err)
+	}
+	_, err = os.Stat(cDir + "/" + fooEncrypted)
+	if err != nil {
+		t.Error(err)
+	}
+
+	// Rename back and forth, name should stay the same
+	if err := os.Rename(pDir+"/foo", pDir+"/foo.tmp"); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Rename(pDir+"/foo.tmp", pDir+"/foo"); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := os.Stat(cDir + "/" + fooEncrypted); err != nil {
+		t.Error(err)
+	}
+}

tests/matrix/matrix_test.go

@@ -55,20 +55,28 @@ var matrix = []testcaseMatrix{
 	// -serialize_reads
 	{false, "auto", false, false, []string{"-serialize_reads"}},
 	{false, "auto", false, false, []string{"-sharedstorage"}},
+	{false, "auto", false, false, []string{"-deterministic-names"}},
 }
 
 // This is the entry point for the tests
 func TestMain(m *testing.M) {
 	// Make "testing.Verbose()" return the correct value
 	flag.Parse()
-	for _, testcase = range matrix {
+	var i int
+	for i, testcase = range matrix {
 		if testcase.openssl == "true" && stupidgcm.BuiltWithoutOpenssl {
 			continue
 		}
 		if testing.Verbose() {
 			fmt.Printf("matrix: testcase = %#v\n", testcase)
 		}
-		test_helpers.ResetTmpDir(!testcase.plaintextnames)
+		createDirIV := true
+		if testcase.plaintextnames {
+			createDirIV = false
+		} else if len(testcase.extraArgs) == 1 && testcase.extraArgs[0] == "-deterministic-names" {
+			createDirIV = false
+		}
+		test_helpers.ResetTmpDir(createDirIV)
 		opts := []string{"-zerokey"}
 		//opts = append(opts, "-fusedebug")
 		opts = append(opts, fmt.Sprintf("-openssl=%v", testcase.openssl))
@@ -90,6 +98,7 @@ func TestMain(m *testing.M) {
 		}
 		test_helpers.UnmountPanic(test_helpers.DefaultPlainDir)
 		if r != 0 {
+			fmt.Printf("TestMain: matrix[%d] = %#v failed\n", i, testcase)
 			os.Exit(r)
 		}
 	}

tests/test_helpers/helpers.go

@@ -110,7 +110,7 @@ func ResetTmpDir(createDirIV bool) {
 		// Open cipherdir (following symlinks)
 		dirfd, err := syscall.Open(DefaultCipherDir, syscall.O_DIRECTORY|syscallcompat.O_PATH, 0)
 		if err == nil {
-			err = nametransform.WriteDirIVAt(dirfd, true)
+			err = nametransform.WriteDirIVAt(dirfd)
 			syscall.Close(dirfd)
 		}
 		if err != nil {

tests/zerodiriv/zerodiriv_test.go

@@ -1,85 +0,0 @@
-package zerodiriv
-
-// integration tests that target zerodiriv specifically
-
-import (
-	"bytes"
-	"path/filepath"
-	"io/ioutil"
-	"os"
-	"testing"
-
-	"github.com/rfjakob/gocryptfs/tests/test_helpers"
-)
-
-var cDir string
-var pDir string
-
-var testPw = []byte("test")
-
-// Create and mount "-zerodiriv" fs
-func TestMain(m *testing.M) {
-	cDir = test_helpers.InitFS(nil, "-zerodiriv")
-	pDir = cDir + ".mnt"
-	test_helpers.MountOrExit(cDir, pDir, "-zerodiriv", "-extpass", "echo test")
-	r := m.Run()
-	test_helpers.UnmountPanic(pDir)
-	os.Exit(r)
-}
-
-// diriv should be all-zero on newly created dirs
-func TestZeroDirIV(t *testing.T) {
-	// Create /dir1, move it and create it again
-	var dirPath = pDir+"/dir1"
-	var err = os.Mkdir(dirPath, 0777)
-	if err != nil {
-		t.Error(err)
-	}
-	err = os.Rename(dirPath, dirPath + ".bak")
-	if err != nil {
-		t.Error(err)
-	}
-	err = os.Mkdir(dirPath, 0777)
-	if err != nil {
-		t.Error(err)
-	}
-
-	var matches []string
-	matches, err = filepath.Glob(cDir+"/*/gocryptfs.diriv")
-	if err != nil {
-		t.Error(err)
-	}
-
-	// The contents of the both diriv files must be the same
-	var diriv0 []byte
-	diriv0, err = ioutil.ReadFile(matches[0])
-	if err != nil {
-		t.Error(err)
-	}
-	var diriv1 []byte
-	diriv1, err = ioutil.ReadFile(matches[1])
-	if err != nil {
-		t.Error(err)
-	}
-	if !bytes.Equal(diriv0, diriv1) {
-		t.Errorf("both dirivs should have the same value")
-	}
-	// And equal to zero
-	zerodiriv := make([]byte, len(diriv0))
-	if !bytes.Equal(diriv0, zerodiriv) {
-		t.Errorf("both dirivs should be all-zero")
-	}
-}
-
-// root diriv should be all-zero
-func TestZeroRootDirIV(t *testing.T) {
-	// The contents of the diriv file must be zero
-	diriv, err := ioutil.ReadFile(cDir+"/gocryptfs.diriv")
-	if err != nil {
-		t.Error(err)
-	}
-	zerodiriv := make([]byte, len(diriv))
-	if !bytes.Equal(diriv, zerodiriv) {
-		t.Errorf("root diriv should be all-zero")
-	}
-}