fusefronted, openfiletable: move the open file table to its own package

The open file table code needs some room to grow for the upcoming
FD multiplexing implementation.

internal/fusefrontend/file.go

@@ -8,7 +8,6 @@ import (
 	"log"
 	"os"
 	"sync"
-	"sync/atomic"
 	"syscall"
 	"time"
 
@@ -16,6 +15,7 @@ import (
 	"github.com/hanwen/go-fuse/fuse/nodefs"
 
 	"github.com/rfjakob/gocryptfs/internal/contentenc"
+	"github.com/rfjakob/gocryptfs/internal/openfiletable"
 	"github.com/rfjakob/gocryptfs/internal/serialize_reads"
 	"github.com/rfjakob/gocryptfs/internal/stupidgcm"
 	"github.com/rfjakob/gocryptfs/internal/syscallcompat"
@@ -40,9 +40,9 @@ type file struct {
 	// Content encryption helper
 	contentEnc *contentenc.ContentEnc
 	// Device and inode number uniquely identify the backing file
-	devIno DevInoStruct
+	qIno openfiletable.QIno
-	// Entry in the open file map
+	// Entry in the open file table
-	fileTableEntry *openFileEntryT
+	fileTableEntry *openfiletable.Entry
 	// go-fuse nodefs.loopbackFile
 	loopbackFile nodefs.File
 	// Store where the last byte was written
@@ -66,15 +66,15 @@ func NewFile(fd *os.File, writeOnly bool, fs *FS) (nodefs.File, fuse.Status) {
 		tlog.Warn.Printf("NewFile: Fstat on fd %d failed: %v\n", fd.Fd(), err)
 		return nil, fuse.ToStatus(err)
 	}
-	di := DevInoFromStat(&st)
+	qi := openfiletable.QInoFromStat(&st)
-	t := openFileMap.register(di)
+	e := openfiletable.Register(qi)
 
 	return &file{
 		fd:             fd,
 		writeOnly:      writeOnly,
 		contentEnc:     fs.contentEnc,
-		devIno:         di,
+		qIno:           qi,
-		fileTableEntry: t,
+		fileTableEntry: e,
 		loopbackFile:   nodefs.NewLoopbackFile(fd),
 		fs:             fs,
 		File:           nodefs.NewDefaultFile(),
@@ -106,7 +106,7 @@ func (f *file) readFileID() ([]byte, error) {
 	if err != nil {
 		if err == io.EOF && n != 0 {
 			tlog.Warn.Printf("ino%d: readFileID: incomplete file, got %d instead of %d bytes",
-				f.devIno.ino, n, readLen)
+				f.qIno.Ino, n, readLen)
 		}
 		return nil, err
 	}
@@ -128,7 +128,7 @@ func (f *file) createHeader() (fileID []byte, err error) {
 	if !f.fs.args.NoPrealloc {
 		err = syscallcompat.EnospcPrealloc(int(f.fd.Fd()), 0, contentenc.HeaderLen)
 		if err != nil {
-			tlog.Warn.Printf("ino%d: createHeader: prealloc failed: %s\n", f.devIno.ino, err.Error())
+			tlog.Warn.Printf("ino%d: createHeader: prealloc failed: %s\n", f.qIno.Ino, err.Error())
 			return nil, err
 		}
 	}
@@ -200,10 +200,10 @@ func (f *file) doRead(off uint64, length uint64) ([]byte, fuse.Status) {
 			// We do not have the information which block was corrupt here anymore,
 			// but DecryptBlocks() has already logged it anyway.
 			tlog.Warn.Printf("ino%d: doRead off=%d len=%d: returning corrupt data due to forcedecode",
-				f.devIno.ino, off, length)
+				f.qIno.Ino, off, length)
 		} else {
 			curruptBlockNo := firstBlockNo + f.contentEnc.PlainOffToBlockNo(uint64(len(plaintext)))
-			tlog.Warn.Printf("ino%d: doRead: corrupt block #%d: %v", f.devIno.ino, curruptBlockNo, err)
+			tlog.Warn.Printf("ino%d: doRead: corrupt block #%d: %v", f.qIno.Ino, curruptBlockNo, err)
 			return nil, fuse.EIO
 		}
 	}
@@ -227,10 +227,10 @@ func (f *file) Read(buf []byte, off int64) (resultData fuse.ReadResult, code fus
 	f.fdLock.RLock()
 	defer f.fdLock.RUnlock()
 
-	tlog.Debug.Printf("ino%d: FUSE Read: offset=%d length=%d", f.devIno.ino, len(buf), off)
+	tlog.Debug.Printf("ino%d: FUSE Read: offset=%d length=%d", f.qIno.Ino, len(buf), off)
 
 	if f.writeOnly {
-		tlog.Warn.Printf("ino%d: Tried to read from write-only file", f.devIno.ino)
+		tlog.Warn.Printf("ino%d: Tried to read from write-only file", f.qIno.Ino)
 		return nil, fuse.EBADF
 	}
 
@@ -245,13 +245,13 @@ func (f *file) Read(buf []byte, off int64) (resultData fuse.ReadResult, code fus
 	}
 
 	if status == fuse.EIO {
-		tlog.Warn.Printf("ino%d: Read: returning EIO, offset=%d, length=%d", f.devIno.ino, len(buf), off)
+		tlog.Warn.Printf("ino%d: Read: returning EIO, offset=%d, length=%d", f.qIno.Ino, len(buf), off)
 	}
 	if status != fuse.OK {
 		return nil, status
 	}
 
-	tlog.Debug.Printf("ino%d: Read: status %v, returning %d bytes", f.devIno.ino, status, len(out))
+	tlog.Debug.Printf("ino%d: Read: status %v, returning %d bytes", f.qIno.Ino, status, len(out))
 	return fuse.ReadResultData(out), status
 }
 
@@ -302,7 +302,7 @@ func (f *file) doWrite(data []byte, off int64) (uint32, fuse.Status) {
 			var oldData []byte
 			oldData, status = f.doRead(o, f.contentEnc.PlainBS())
 			if status != fuse.OK {
-				tlog.Warn.Printf("ino%d fh%d: RMW read failed: %s", f.devIno.ino, f.intFd(), status.String())
+				tlog.Warn.Printf("ino%d fh%d: RMW read failed: %s", f.qIno.Ino, f.intFd(), status.String())
 				return 0, status
 			}
 			// Modify
@@ -312,7 +312,7 @@ func (f *file) doWrite(data []byte, off int64) (uint32, fuse.Status) {
 		// Encrypt
 		blockData = f.contentEnc.EncryptBlock(blockData, b.BlockNo, fileID)
 		tlog.Debug.Printf("ino%d: Writing %d bytes to block #%d",
-			f.devIno.ino, uint64(len(blockData))-f.contentEnc.BlockOverhead(), b.BlockNo)
+			f.qIno.Ino, uint64(len(blockData))-f.contentEnc.BlockOverhead(), b.BlockNo)
 		// Store output data in the writeChain
 		writeChain[i] = blockData
 		numOutBytes += len(blockData)
@@ -330,7 +330,7 @@ func (f *file) doWrite(data []byte, off int64) (uint32, fuse.Status) {
 	if !f.fs.args.NoPrealloc {
 		err = syscallcompat.EnospcPrealloc(int(f.fd.Fd()), int64(cOff), int64(writeBuf.Len()))
 		if err != nil {
-			tlog.Warn.Printf("ino%d fh%d: doWrite: prealloc failed: %s", f.devIno.ino, f.intFd(), err.Error())
+			tlog.Warn.Printf("ino%d fh%d: doWrite: prealloc failed: %s", f.qIno.Ino, f.intFd(), err.Error())
 			return 0, fuse.ToStatus(err)
 		}
 	}
@@ -349,7 +349,7 @@ func (f *file) doWrite(data []byte, off int64) (uint32, fuse.Status) {
 // Stat() call is very expensive.
 // The caller must "wlock.lock(f.devIno.ino)" otherwise this check would be racy.
 func (f *file) isConsecutiveWrite(off int64) bool {
-	opCount := atomic.LoadUint64(&openFileMap.opCount)
+	opCount := openfiletable.WriteLockCount()
 	return opCount == f.lastOpCount+1 && off == f.lastWrittenOffset+1
 }
 
@@ -363,12 +363,12 @@ func (f *file) Write(data []byte, off int64) (uint32, fuse.Status) {
 		// The file descriptor has been closed concurrently, which also means
 		// the wlock has been freed. Exit here so we don't crash trying to access
 		// it.
-		tlog.Warn.Printf("ino%d fh%d: Write on released file", f.devIno.ino, f.intFd())
+		tlog.Warn.Printf("ino%d fh%d: Write on released file", f.qIno.Ino, f.intFd())
 		return 0, fuse.EBADF
 	}
-	f.fileTableEntry.writeLock.Lock()
+	f.fileTableEntry.WriteLock.Lock()
-	defer f.fileTableEntry.writeLock.Unlock()
+	defer f.fileTableEntry.WriteLock.Unlock()
-	tlog.Debug.Printf("ino%d: FUSE Write: offset=%d length=%d", f.devIno.ino, off, len(data))
+	tlog.Debug.Printf("ino%d: FUSE Write: offset=%d length=%d", f.qIno.Ino, off, len(data))
 	// If the write creates a file hole, we have to zero-pad the last block.
 	// But if the write directly follows an earlier write, it cannot create a
 	// hole, and we can save one Stat() call.
@@ -380,7 +380,7 @@ func (f *file) Write(data []byte, off int64) (uint32, fuse.Status) {
 	}
 	n, status := f.doWrite(data, off)
 	if status.Ok() {
-		f.lastOpCount = atomic.LoadUint64(&openFileMap.opCount)
+		f.lastOpCount = openfiletable.WriteLockCount()
 		f.lastWrittenOffset = off + int64(len(data)) - 1
 	}
 	return n, status
@@ -390,13 +390,13 @@ func (f *file) Write(data []byte, off int64) (uint32, fuse.Status) {
 func (f *file) Release() {
 	f.fdLock.Lock()
 	if f.released {
-		log.Panicf("ino%d fh%d: double release", f.devIno.ino, f.intFd())
+		log.Panicf("ino%d fh%d: double release", f.qIno.Ino, f.intFd())
 	}
 	f.fd.Close()
 	f.released = true
 	f.fdLock.Unlock()
 
-	openFileMap.unregister(f.devIno)
+	openfiletable.Unregister(f.qIno)
 }
 
 // Flush - FUSE call

internal/fusefrontend/file_allocate_truncate.go

@@ -50,8 +50,8 @@ func (f *file) Allocate(off uint64, sz uint64, mode uint32) fuse.Status {
 	if f.released {
 		return fuse.EBADF
 	}
-	f.fileTableEntry.writeLock.Lock()
+	f.fileTableEntry.WriteLock.Lock()
-	defer f.fileTableEntry.writeLock.Unlock()
+	defer f.fileTableEntry.WriteLock.Unlock()
 
 	blocks := f.contentEnc.ExplodePlainRange(off, sz)
 	firstBlock := blocks[0]
@@ -97,17 +97,17 @@ func (f *file) Truncate(newSize uint64) fuse.Status {
 	defer f.fdLock.RUnlock()
 	if f.released {
 		// The file descriptor has been closed concurrently.
-		tlog.Warn.Printf("ino%d fh%d: Truncate on released file", f.devIno.ino, f.intFd())
+		tlog.Warn.Printf("ino%d fh%d: Truncate on released file", f.qIno.Ino, f.intFd())
 		return fuse.EBADF
 	}
-	f.fileTableEntry.writeLock.Lock()
+	f.fileTableEntry.WriteLock.Lock()
-	defer f.fileTableEntry.writeLock.Unlock()
+	defer f.fileTableEntry.WriteLock.Unlock()
 	var err error
 	// Common case first: Truncate to zero
 	if newSize == 0 {
 		err = syscall.Ftruncate(int(f.fd.Fd()), 0)
 		if err != nil {
-			tlog.Warn.Printf("ino%d fh%d: Ftruncate(fd, 0) returned error: %v", f.devIno.ino, f.intFd(), err)
+			tlog.Warn.Printf("ino%d fh%d: Ftruncate(fd, 0) returned error: %v", f.qIno.Ino, f.intFd(), err)
 			return fuse.ToStatus(err)
 		}
 		// Truncate to zero kills the file header
@@ -125,7 +125,7 @@ func (f *file) Truncate(newSize uint64) fuse.Status {
 
 	oldB := float32(oldSize) / float32(f.contentEnc.PlainBS())
 	newB := float32(newSize) / float32(f.contentEnc.PlainBS())
-	tlog.Debug.Printf("ino%d: FUSE Truncate from %.2f to %.2f blocks (%d to %d bytes)", f.devIno.ino, oldB, newB, oldSize, newSize)
+	tlog.Debug.Printf("ino%d: FUSE Truncate from %.2f to %.2f blocks (%d to %d bytes)", f.qIno.Ino, oldB, newB, oldSize, newSize)
 
 	// File size stays the same - nothing to do
 	if newSize == oldSize {
@@ -168,7 +168,7 @@ func (f *file) Truncate(newSize uint64) fuse.Status {
 func (f *file) statPlainSize() (uint64, error) {
 	fi, err := f.fd.Stat()
 	if err != nil {
-		tlog.Warn.Printf("ino%d fh%d: statPlainSize: %v", f.devIno.ino, f.intFd(), err)
+		tlog.Warn.Printf("ino%d fh%d: statPlainSize: %v", f.qIno.Ino, f.intFd(), err)
 		return 0, err
 	}
 	cipherSz := uint64(fi.Size())

internal/fusefrontend/open_file_table.go

@@ -1,101 +0,0 @@
-package fusefrontend
-
-import (
-	"sync"
-	"sync/atomic"
-	"syscall"
-)
-
-// DevInoStruct uniquely identifies a backing file through device number and
-// inode number.
-type DevInoStruct struct {
-	dev uint64
-	ino uint64
-}
-
-// DevInoFromStat fills a new DevInoStruct with the passed Stat_t info
-func DevInoFromStat(st *syscall.Stat_t) DevInoStruct {
-	// Explicit cast to uint64 to prevent build problems on 32-bit platforms
-	return DevInoStruct{
-		dev: uint64(st.Dev),
-		ino: uint64(st.Ino),
-	}
-}
-
-func init() {
-	openFileMap.entries = make(map[DevInoStruct]*openFileEntryT)
-}
-
-// wlock - serializes write accesses to each file (identified by inode number)
-// Writing partial blocks means we have to do read-modify-write cycles. We
-// really don't want concurrent writes there.
-// Concurrent full-block writes could actually be allowed, but are not to
-// keep the locking simple.
-var openFileMap openFileMapT
-
-// wlockMap - usage:
-// 1) register
-// 2) lock ... unlock ...
-// 3) unregister
-type openFileMapT struct {
-	// opCount counts writeLock.Lock() calls. As every operation that modifies a file should
-	// call it, this effectively serves as a write-operation counter.
-	// The variable is accessed without holding any locks so atomic operations
-	// must be used. It must be the first element of the struct to guarantee
-	// 64-bit alignment.
-	opCount uint64
-	// Protects map access
-	sync.Mutex
-	entries map[DevInoStruct]*openFileEntryT
-}
-
-type opCountMutex struct {
-	sync.Mutex
-	// Points to the opCount variable of the parent openFileMapT
-	opCount *uint64
-}
-
-func (o *opCountMutex) Lock() {
-	o.Mutex.Lock()
-	atomic.AddUint64(o.opCount, 1)
-}
-
-// refCntMutex - mutex with reference count
-type openFileEntryT struct {
-	// Reference count
-	refCnt int
-	// Write lock for this inode
-	writeLock *opCountMutex
-	// ID is the file ID in the file header.
-	ID     []byte
-	IDLock sync.RWMutex
-}
-
-// register creates an entry for "ino", or incrementes the reference count
-// if the entry already exists.
-func (w *openFileMapT) register(di DevInoStruct) *openFileEntryT {
-	w.Lock()
-	defer w.Unlock()
-
-	r := w.entries[di]
-	if r == nil {
-		o := opCountMutex{opCount: &w.opCount}
-		r = &openFileEntryT{writeLock: &o}
-		w.entries[di] = r
-	}
-	r.refCnt++
-	return r
-}
-
-// unregister decrements the reference count for "di" and deletes the entry if
-// the reference count has reached 0.
-func (w *openFileMapT) unregister(di DevInoStruct) {
-	w.Lock()
-	defer w.Unlock()
-
-	r := w.entries[di]
-	r.refCnt--
-	if r.refCnt == 0 {
-		delete(w.entries, di)
-	}
-}

internal/openfiletable/open_file_table.go

@@ -0,0 +1,111 @@
+// Package openfiletable maintains a table of currently opened files, identified
+// by the device number + inode number pair. This table is used by fusefrontend
+// to centrally store the current file ID and to lock files against concurrent
+// writes.
+package openfiletable
+
+import (
+	"sync"
+	"sync/atomic"
+	"syscall"
+)
+
+// QIno = Qualified Inode number.
+// Uniquely identifies a backing file through the device number,
+// inode number pair.
+type QIno struct {
+	// Stat_t.{Dev,Ino} is uint64 on 32- and 64-bit Linux
+	Dev uint64
+	Ino uint64
+}
+
+// QInoFromStat fills a new QIno struct with the passed Stat_t info.
+func QInoFromStat(st *syscall.Stat_t) QIno {
+	return QIno{
+		// There are some architectures that use 32-bit values here
+		// (darwin, freebsd-32, maybe others). Add and explicit cast to make
+		// this function work everywhere.
+		Dev: uint64(st.Dev),
+		Ino: uint64(st.Ino),
+	}
+}
+
+// wlock - serializes write accesses to each file (identified by inode number)
+// Writing partial blocks means we have to do read-modify-write cycles. We
+// really don't want concurrent writes there.
+// Concurrent full-block writes could actually be allowed, but are not to
+// keep the locking simple.
+var t table
+
+func init() {
+	t.entries = make(map[QIno]*Entry)
+}
+
+type table struct {
+	// writeLockCount counts entry.writeLock.Lock() calls. As every operation that
+	// modifies a file should
+	// call it, this effectively serves as a write-operation counter.
+	// The variable is accessed without holding any locks so atomic operations
+	// must be used. It must be the first element of the struct to guarantee
+	// 64-bit alignment.
+	writeLockCount uint64
+	// Protects map access
+	sync.Mutex
+	// Table entries
+	entries map[QIno]*Entry
+}
+
+// Entry is an entry in the open file table
+type Entry struct {
+	// Reference count
+	refCount int
+	// Write lock for this inode
+	WriteLock countingMutex
+	// ID is the file ID in the file header.
+	ID     []byte
+	IDLock sync.RWMutex
+}
+
+// Register creates an open file table entry for "qi" (or incrementes the
+// reference count if the entry already exists) and returns the entry.
+func Register(qi QIno) *Entry {
+	t.Lock()
+	defer t.Unlock()
+
+	e := t.entries[qi]
+	if e == nil {
+		e = &Entry{}
+		t.entries[qi] = e
+	}
+	e.refCount++
+	return e
+}
+
+// Unregister decrements the reference count for "qi" and deletes the entry from
+// the open file table if the reference count reaches 0.
+func Unregister(qi QIno) {
+	t.Lock()
+	defer t.Unlock()
+
+	e := t.entries[qi]
+	e.refCount--
+	if e.refCount == 0 {
+		delete(t.entries, qi)
+	}
+}
+
+// countingMutex incrementes t.writeLockCount on each Lock() call.
+type countingMutex struct {
+	sync.Mutex
+}
+
+func (c *countingMutex) Lock() {
+	c.Mutex.Lock()
+	atomic.AddUint64(&t.writeLockCount, 1)
+}
+
+// WriteLockCount returns the write lock counter value. This value is encremented
+// each time writeLock.Lock() on a file table entry is called.
+func WriteLockCount() uint64 {
+	return atomic.LoadUint64(&t.writeLockCount)
+}