From 514f515dd7196e26ca8df6886ac4a34e928e50dd Mon Sep 17 00:00:00 2001 From: Jakob Unterwurzacher Date: Mon, 1 May 2017 17:26:50 +0200 Subject: [PATCH] fusefronted, openfiletable: move the open file table to its own package The open file table code needs some room to grow for the upcoming FD multiplexing implementation. --- internal/fusefrontend/file.go | 54 ++++----- .../fusefrontend/file_allocate_truncate.go | 16 +-- internal/fusefrontend/open_file_table.go | 101 ---------------- internal/openfiletable/open_file_table.go | 111 ++++++++++++++++++ 4 files changed, 146 insertions(+), 136 deletions(-) delete mode 100644 internal/fusefrontend/open_file_table.go create mode 100644 internal/openfiletable/open_file_table.go diff --git a/internal/fusefrontend/file.go b/internal/fusefrontend/file.go index ab35a59..92b1496 100644 --- a/internal/fusefrontend/file.go +++ b/internal/fusefrontend/file.go @@ -8,7 +8,6 @@ import ( "log" "os" "sync" - "sync/atomic" "syscall" "time" @@ -16,6 +15,7 @@ import ( "github.com/hanwen/go-fuse/fuse/nodefs" "github.com/rfjakob/gocryptfs/internal/contentenc" + "github.com/rfjakob/gocryptfs/internal/openfiletable" "github.com/rfjakob/gocryptfs/internal/serialize_reads" "github.com/rfjakob/gocryptfs/internal/stupidgcm" "github.com/rfjakob/gocryptfs/internal/syscallcompat" @@ -40,9 +40,9 @@ type file struct { // Content encryption helper contentEnc *contentenc.ContentEnc // Device and inode number uniquely identify the backing file - devIno DevInoStruct - // Entry in the open file map - fileTableEntry *openFileEntryT + qIno openfiletable.QIno + // Entry in the open file table + fileTableEntry *openfiletable.Entry // go-fuse nodefs.loopbackFile loopbackFile nodefs.File // Store where the last byte was written @@ -66,15 +66,15 @@ func NewFile(fd *os.File, writeOnly bool, fs *FS) (nodefs.File, fuse.Status) { tlog.Warn.Printf("NewFile: Fstat on fd %d failed: %v\n", fd.Fd(), err) return nil, fuse.ToStatus(err) } - di := DevInoFromStat(&st) - t := openFileMap.register(di) + qi := openfiletable.QInoFromStat(&st) + e := openfiletable.Register(qi) return &file{ fd: fd, writeOnly: writeOnly, contentEnc: fs.contentEnc, - devIno: di, - fileTableEntry: t, + qIno: qi, + fileTableEntry: e, loopbackFile: nodefs.NewLoopbackFile(fd), fs: fs, File: nodefs.NewDefaultFile(), @@ -106,7 +106,7 @@ func (f *file) readFileID() ([]byte, error) { if err != nil { if err == io.EOF && n != 0 { tlog.Warn.Printf("ino%d: readFileID: incomplete file, got %d instead of %d bytes", - f.devIno.ino, n, readLen) + f.qIno.Ino, n, readLen) } return nil, err } @@ -128,7 +128,7 @@ func (f *file) createHeader() (fileID []byte, err error) { if !f.fs.args.NoPrealloc { err = syscallcompat.EnospcPrealloc(int(f.fd.Fd()), 0, contentenc.HeaderLen) if err != nil { - tlog.Warn.Printf("ino%d: createHeader: prealloc failed: %s\n", f.devIno.ino, err.Error()) + tlog.Warn.Printf("ino%d: createHeader: prealloc failed: %s\n", f.qIno.Ino, err.Error()) return nil, err } } @@ -200,10 +200,10 @@ func (f *file) doRead(off uint64, length uint64) ([]byte, fuse.Status) { // We do not have the information which block was corrupt here anymore, // but DecryptBlocks() has already logged it anyway. tlog.Warn.Printf("ino%d: doRead off=%d len=%d: returning corrupt data due to forcedecode", - f.devIno.ino, off, length) + f.qIno.Ino, off, length) } else { curruptBlockNo := firstBlockNo + f.contentEnc.PlainOffToBlockNo(uint64(len(plaintext))) - tlog.Warn.Printf("ino%d: doRead: corrupt block #%d: %v", f.devIno.ino, curruptBlockNo, err) + tlog.Warn.Printf("ino%d: doRead: corrupt block #%d: %v", f.qIno.Ino, curruptBlockNo, err) return nil, fuse.EIO } } @@ -227,10 +227,10 @@ func (f *file) Read(buf []byte, off int64) (resultData fuse.ReadResult, code fus f.fdLock.RLock() defer f.fdLock.RUnlock() - tlog.Debug.Printf("ino%d: FUSE Read: offset=%d length=%d", f.devIno.ino, len(buf), off) + tlog.Debug.Printf("ino%d: FUSE Read: offset=%d length=%d", f.qIno.Ino, len(buf), off) if f.writeOnly { - tlog.Warn.Printf("ino%d: Tried to read from write-only file", f.devIno.ino) + tlog.Warn.Printf("ino%d: Tried to read from write-only file", f.qIno.Ino) return nil, fuse.EBADF } @@ -245,13 +245,13 @@ func (f *file) Read(buf []byte, off int64) (resultData fuse.ReadResult, code fus } if status == fuse.EIO { - tlog.Warn.Printf("ino%d: Read: returning EIO, offset=%d, length=%d", f.devIno.ino, len(buf), off) + tlog.Warn.Printf("ino%d: Read: returning EIO, offset=%d, length=%d", f.qIno.Ino, len(buf), off) } if status != fuse.OK { return nil, status } - tlog.Debug.Printf("ino%d: Read: status %v, returning %d bytes", f.devIno.ino, status, len(out)) + tlog.Debug.Printf("ino%d: Read: status %v, returning %d bytes", f.qIno.Ino, status, len(out)) return fuse.ReadResultData(out), status } @@ -302,7 +302,7 @@ func (f *file) doWrite(data []byte, off int64) (uint32, fuse.Status) { var oldData []byte oldData, status = f.doRead(o, f.contentEnc.PlainBS()) if status != fuse.OK { - tlog.Warn.Printf("ino%d fh%d: RMW read failed: %s", f.devIno.ino, f.intFd(), status.String()) + tlog.Warn.Printf("ino%d fh%d: RMW read failed: %s", f.qIno.Ino, f.intFd(), status.String()) return 0, status } // Modify @@ -312,7 +312,7 @@ func (f *file) doWrite(data []byte, off int64) (uint32, fuse.Status) { // Encrypt blockData = f.contentEnc.EncryptBlock(blockData, b.BlockNo, fileID) tlog.Debug.Printf("ino%d: Writing %d bytes to block #%d", - f.devIno.ino, uint64(len(blockData))-f.contentEnc.BlockOverhead(), b.BlockNo) + f.qIno.Ino, uint64(len(blockData))-f.contentEnc.BlockOverhead(), b.BlockNo) // Store output data in the writeChain writeChain[i] = blockData numOutBytes += len(blockData) @@ -330,7 +330,7 @@ func (f *file) doWrite(data []byte, off int64) (uint32, fuse.Status) { if !f.fs.args.NoPrealloc { err = syscallcompat.EnospcPrealloc(int(f.fd.Fd()), int64(cOff), int64(writeBuf.Len())) if err != nil { - tlog.Warn.Printf("ino%d fh%d: doWrite: prealloc failed: %s", f.devIno.ino, f.intFd(), err.Error()) + tlog.Warn.Printf("ino%d fh%d: doWrite: prealloc failed: %s", f.qIno.Ino, f.intFd(), err.Error()) return 0, fuse.ToStatus(err) } } @@ -349,7 +349,7 @@ func (f *file) doWrite(data []byte, off int64) (uint32, fuse.Status) { // Stat() call is very expensive. // The caller must "wlock.lock(f.devIno.ino)" otherwise this check would be racy. func (f *file) isConsecutiveWrite(off int64) bool { - opCount := atomic.LoadUint64(&openFileMap.opCount) + opCount := openfiletable.WriteLockCount() return opCount == f.lastOpCount+1 && off == f.lastWrittenOffset+1 } @@ -363,12 +363,12 @@ func (f *file) Write(data []byte, off int64) (uint32, fuse.Status) { // The file descriptor has been closed concurrently, which also means // the wlock has been freed. Exit here so we don't crash trying to access // it. - tlog.Warn.Printf("ino%d fh%d: Write on released file", f.devIno.ino, f.intFd()) + tlog.Warn.Printf("ino%d fh%d: Write on released file", f.qIno.Ino, f.intFd()) return 0, fuse.EBADF } - f.fileTableEntry.writeLock.Lock() - defer f.fileTableEntry.writeLock.Unlock() - tlog.Debug.Printf("ino%d: FUSE Write: offset=%d length=%d", f.devIno.ino, off, len(data)) + f.fileTableEntry.WriteLock.Lock() + defer f.fileTableEntry.WriteLock.Unlock() + tlog.Debug.Printf("ino%d: FUSE Write: offset=%d length=%d", f.qIno.Ino, off, len(data)) // If the write creates a file hole, we have to zero-pad the last block. // But if the write directly follows an earlier write, it cannot create a // hole, and we can save one Stat() call. @@ -380,7 +380,7 @@ func (f *file) Write(data []byte, off int64) (uint32, fuse.Status) { } n, status := f.doWrite(data, off) if status.Ok() { - f.lastOpCount = atomic.LoadUint64(&openFileMap.opCount) + f.lastOpCount = openfiletable.WriteLockCount() f.lastWrittenOffset = off + int64(len(data)) - 1 } return n, status @@ -390,13 +390,13 @@ func (f *file) Write(data []byte, off int64) (uint32, fuse.Status) { func (f *file) Release() { f.fdLock.Lock() if f.released { - log.Panicf("ino%d fh%d: double release", f.devIno.ino, f.intFd()) + log.Panicf("ino%d fh%d: double release", f.qIno.Ino, f.intFd()) } f.fd.Close() f.released = true f.fdLock.Unlock() - openFileMap.unregister(f.devIno) + openfiletable.Unregister(f.qIno) } // Flush - FUSE call diff --git a/internal/fusefrontend/file_allocate_truncate.go b/internal/fusefrontend/file_allocate_truncate.go index ae3dd41..acde76f 100644 --- a/internal/fusefrontend/file_allocate_truncate.go +++ b/internal/fusefrontend/file_allocate_truncate.go @@ -50,8 +50,8 @@ func (f *file) Allocate(off uint64, sz uint64, mode uint32) fuse.Status { if f.released { return fuse.EBADF } - f.fileTableEntry.writeLock.Lock() - defer f.fileTableEntry.writeLock.Unlock() + f.fileTableEntry.WriteLock.Lock() + defer f.fileTableEntry.WriteLock.Unlock() blocks := f.contentEnc.ExplodePlainRange(off, sz) firstBlock := blocks[0] @@ -97,17 +97,17 @@ func (f *file) Truncate(newSize uint64) fuse.Status { defer f.fdLock.RUnlock() if f.released { // The file descriptor has been closed concurrently. - tlog.Warn.Printf("ino%d fh%d: Truncate on released file", f.devIno.ino, f.intFd()) + tlog.Warn.Printf("ino%d fh%d: Truncate on released file", f.qIno.Ino, f.intFd()) return fuse.EBADF } - f.fileTableEntry.writeLock.Lock() - defer f.fileTableEntry.writeLock.Unlock() + f.fileTableEntry.WriteLock.Lock() + defer f.fileTableEntry.WriteLock.Unlock() var err error // Common case first: Truncate to zero if newSize == 0 { err = syscall.Ftruncate(int(f.fd.Fd()), 0) if err != nil { - tlog.Warn.Printf("ino%d fh%d: Ftruncate(fd, 0) returned error: %v", f.devIno.ino, f.intFd(), err) + tlog.Warn.Printf("ino%d fh%d: Ftruncate(fd, 0) returned error: %v", f.qIno.Ino, f.intFd(), err) return fuse.ToStatus(err) } // Truncate to zero kills the file header @@ -125,7 +125,7 @@ func (f *file) Truncate(newSize uint64) fuse.Status { oldB := float32(oldSize) / float32(f.contentEnc.PlainBS()) newB := float32(newSize) / float32(f.contentEnc.PlainBS()) - tlog.Debug.Printf("ino%d: FUSE Truncate from %.2f to %.2f blocks (%d to %d bytes)", f.devIno.ino, oldB, newB, oldSize, newSize) + tlog.Debug.Printf("ino%d: FUSE Truncate from %.2f to %.2f blocks (%d to %d bytes)", f.qIno.Ino, oldB, newB, oldSize, newSize) // File size stays the same - nothing to do if newSize == oldSize { @@ -168,7 +168,7 @@ func (f *file) Truncate(newSize uint64) fuse.Status { func (f *file) statPlainSize() (uint64, error) { fi, err := f.fd.Stat() if err != nil { - tlog.Warn.Printf("ino%d fh%d: statPlainSize: %v", f.devIno.ino, f.intFd(), err) + tlog.Warn.Printf("ino%d fh%d: statPlainSize: %v", f.qIno.Ino, f.intFd(), err) return 0, err } cipherSz := uint64(fi.Size()) diff --git a/internal/fusefrontend/open_file_table.go b/internal/fusefrontend/open_file_table.go deleted file mode 100644 index 053298d..0000000 --- a/internal/fusefrontend/open_file_table.go +++ /dev/null @@ -1,101 +0,0 @@ -package fusefrontend - -import ( - "sync" - "sync/atomic" - "syscall" -) - -// DevInoStruct uniquely identifies a backing file through device number and -// inode number. -type DevInoStruct struct { - dev uint64 - ino uint64 -} - -// DevInoFromStat fills a new DevInoStruct with the passed Stat_t info -func DevInoFromStat(st *syscall.Stat_t) DevInoStruct { - // Explicit cast to uint64 to prevent build problems on 32-bit platforms - return DevInoStruct{ - dev: uint64(st.Dev), - ino: uint64(st.Ino), - } -} - -func init() { - openFileMap.entries = make(map[DevInoStruct]*openFileEntryT) -} - -// wlock - serializes write accesses to each file (identified by inode number) -// Writing partial blocks means we have to do read-modify-write cycles. We -// really don't want concurrent writes there. -// Concurrent full-block writes could actually be allowed, but are not to -// keep the locking simple. -var openFileMap openFileMapT - -// wlockMap - usage: -// 1) register -// 2) lock ... unlock ... -// 3) unregister -type openFileMapT struct { - // opCount counts writeLock.Lock() calls. As every operation that modifies a file should - // call it, this effectively serves as a write-operation counter. - // The variable is accessed without holding any locks so atomic operations - // must be used. It must be the first element of the struct to guarantee - // 64-bit alignment. - opCount uint64 - // Protects map access - sync.Mutex - entries map[DevInoStruct]*openFileEntryT -} - -type opCountMutex struct { - sync.Mutex - // Points to the opCount variable of the parent openFileMapT - opCount *uint64 -} - -func (o *opCountMutex) Lock() { - o.Mutex.Lock() - atomic.AddUint64(o.opCount, 1) -} - -// refCntMutex - mutex with reference count -type openFileEntryT struct { - // Reference count - refCnt int - // Write lock for this inode - writeLock *opCountMutex - // ID is the file ID in the file header. - ID []byte - IDLock sync.RWMutex -} - -// register creates an entry for "ino", or incrementes the reference count -// if the entry already exists. -func (w *openFileMapT) register(di DevInoStruct) *openFileEntryT { - w.Lock() - defer w.Unlock() - - r := w.entries[di] - if r == nil { - o := opCountMutex{opCount: &w.opCount} - r = &openFileEntryT{writeLock: &o} - w.entries[di] = r - } - r.refCnt++ - return r -} - -// unregister decrements the reference count for "di" and deletes the entry if -// the reference count has reached 0. -func (w *openFileMapT) unregister(di DevInoStruct) { - w.Lock() - defer w.Unlock() - - r := w.entries[di] - r.refCnt-- - if r.refCnt == 0 { - delete(w.entries, di) - } -} diff --git a/internal/openfiletable/open_file_table.go b/internal/openfiletable/open_file_table.go new file mode 100644 index 0000000..7cd401e --- /dev/null +++ b/internal/openfiletable/open_file_table.go @@ -0,0 +1,111 @@ +// Package openfiletable maintains a table of currently opened files, identified +// by the device number + inode number pair. This table is used by fusefrontend +// to centrally store the current file ID and to lock files against concurrent +// writes. +package openfiletable + +import ( + "sync" + "sync/atomic" + "syscall" +) + +// QIno = Qualified Inode number. +// Uniquely identifies a backing file through the device number, +// inode number pair. +type QIno struct { + // Stat_t.{Dev,Ino} is uint64 on 32- and 64-bit Linux + Dev uint64 + Ino uint64 +} + +// QInoFromStat fills a new QIno struct with the passed Stat_t info. +func QInoFromStat(st *syscall.Stat_t) QIno { + return QIno{ + // There are some architectures that use 32-bit values here + // (darwin, freebsd-32, maybe others). Add and explicit cast to make + // this function work everywhere. + Dev: uint64(st.Dev), + Ino: uint64(st.Ino), + } +} + +// wlock - serializes write accesses to each file (identified by inode number) +// Writing partial blocks means we have to do read-modify-write cycles. We +// really don't want concurrent writes there. +// Concurrent full-block writes could actually be allowed, but are not to +// keep the locking simple. +var t table + +func init() { + t.entries = make(map[QIno]*Entry) +} + +type table struct { + // writeLockCount counts entry.writeLock.Lock() calls. As every operation that + // modifies a file should + // call it, this effectively serves as a write-operation counter. + // The variable is accessed without holding any locks so atomic operations + // must be used. It must be the first element of the struct to guarantee + // 64-bit alignment. + writeLockCount uint64 + // Protects map access + sync.Mutex + // Table entries + entries map[QIno]*Entry +} + +// Entry is an entry in the open file table +type Entry struct { + // Reference count + refCount int + // Write lock for this inode + WriteLock countingMutex + // ID is the file ID in the file header. + ID []byte + IDLock sync.RWMutex +} + +// Register creates an open file table entry for "qi" (or incrementes the +// reference count if the entry already exists) and returns the entry. +func Register(qi QIno) *Entry { + t.Lock() + defer t.Unlock() + + e := t.entries[qi] + if e == nil { + e = &Entry{} + t.entries[qi] = e + } + e.refCount++ + return e +} + +// Unregister decrements the reference count for "qi" and deletes the entry from +// the open file table if the reference count reaches 0. +func Unregister(qi QIno) { + t.Lock() + defer t.Unlock() + + e := t.entries[qi] + e.refCount-- + if e.refCount == 0 { + delete(t.entries, qi) + } +} + +// countingMutex incrementes t.writeLockCount on each Lock() call. +type countingMutex struct { + sync.Mutex +} + +func (c *countingMutex) Lock() { + c.Mutex.Lock() + atomic.AddUint64(&t.writeLockCount, 1) +} + +// WriteLockCount returns the write lock counter value. This value is encremented +// each time writeLock.Lock() on a file table entry is called. +func WriteLockCount() uint64 { + return atomic.LoadUint64(&t.writeLockCount) +}