libgocryptfs/internal/fusefrontend/file.go

481 lines
15 KiB
Go
Raw Normal View History

package fusefrontend
// FUSE operations on file handles
import (
"bytes"
"fmt"
2015-10-04 14:36:20 +02:00
"io"
"log"
"os"
"sync"
"syscall"
"time"
"github.com/hanwen/go-fuse/fuse"
"github.com/hanwen/go-fuse/fuse/nodefs"
Major refactoring: Split up "cryptfs" into several internal packages "git status" for reference: deleted: cryptfs/cryptfs.go deleted: cryptfs/names_core.go modified: integration_tests/cli_test.go modified: integration_tests/helpers.go renamed: cryptfs/config_file.go -> internal/configfile/config_file.go renamed: cryptfs/config_test.go -> internal/configfile/config_test.go renamed: cryptfs/config_test/.gitignore -> internal/configfile/config_test/.gitignore renamed: cryptfs/config_test/PlaintextNames.conf -> internal/configfile/config_test/PlaintextNames.conf renamed: cryptfs/config_test/StrangeFeature.conf -> internal/configfile/config_test/StrangeFeature.conf renamed: cryptfs/config_test/v1.conf -> internal/configfile/config_test/v1.conf renamed: cryptfs/config_test/v2.conf -> internal/configfile/config_test/v2.conf renamed: cryptfs/kdf.go -> internal/configfile/kdf.go renamed: cryptfs/kdf_test.go -> internal/configfile/kdf_test.go renamed: cryptfs/cryptfs_content.go -> internal/contentenc/content.go new file: internal/contentenc/content_api.go renamed: cryptfs/content_test.go -> internal/contentenc/content_test.go renamed: cryptfs/file_header.go -> internal/contentenc/file_header.go renamed: cryptfs/intrablock.go -> internal/contentenc/intrablock.go renamed: cryptfs/address_translation.go -> internal/contentenc/offsets.go new file: internal/cryptocore/crypto_api.go renamed: cryptfs/gcm_go1.4.go -> internal/cryptocore/gcm_go1.4.go renamed: cryptfs/gcm_go1.5.go -> internal/cryptocore/gcm_go1.5.go renamed: cryptfs/nonce.go -> internal/cryptocore/nonce.go renamed: cryptfs/openssl_aead.go -> internal/cryptocore/openssl_aead.go renamed: cryptfs/openssl_benchmark.bash -> internal/cryptocore/openssl_benchmark.bash renamed: cryptfs/openssl_test.go -> internal/cryptocore/openssl_test.go new file: internal/nametransform/name_api.go new file: internal/nametransform/names_core.go renamed: cryptfs/names_diriv.go -> internal/nametransform/names_diriv.go renamed: cryptfs/names_noiv.go -> internal/nametransform/names_noiv.go renamed: cryptfs/names_test.go -> internal/nametransform/names_test.go new file: internal/nametransform/pad16.go renamed: cryptfs/log.go -> internal/toggledlog/log.go renamed: cryptfs/log_go1.4.go -> internal/toggledlog/log_go1.4.go renamed: cryptfs/log_go1.5.go -> internal/toggledlog/log_go1.5.go modified: main.go modified: masterkey.go modified: pathfs_frontend/file.go modified: pathfs_frontend/file_holes.go modified: pathfs_frontend/fs.go modified: pathfs_frontend/fs_dir.go modified: pathfs_frontend/names.go modified: test.bash
2016-02-06 19:20:54 +01:00
"github.com/rfjakob/gocryptfs/internal/contentenc"
"github.com/rfjakob/gocryptfs/internal/openfiletable"
"github.com/rfjakob/gocryptfs/internal/serialize_reads"
"github.com/rfjakob/gocryptfs/internal/stupidgcm"
"github.com/rfjakob/gocryptfs/internal/syscallcompat"
"github.com/rfjakob/gocryptfs/internal/tlog"
)
var _ nodefs.File = &File{} // Verify that interface is implemented.
// File - based on loopbackFile in go-fuse/fuse/nodefs/files.go
type File struct {
fd *os.File
// Has Release() already been called on this file? This also means that the
// wlock entry has been freed, so let's not crash trying to access it.
// Due to concurrency, Release can overtake other operations. These will
// return EBADF in that case.
released bool
// fdLock prevents the fd to be closed while we are in the middle of
// an operation.
// Every FUSE entrypoint should RLock(). The only user of Lock() is
// Release(), which closes the fd and sets "released" to true.
fdLock sync.RWMutex
Major refactoring: Split up "cryptfs" into several internal packages "git status" for reference: deleted: cryptfs/cryptfs.go deleted: cryptfs/names_core.go modified: integration_tests/cli_test.go modified: integration_tests/helpers.go renamed: cryptfs/config_file.go -> internal/configfile/config_file.go renamed: cryptfs/config_test.go -> internal/configfile/config_test.go renamed: cryptfs/config_test/.gitignore -> internal/configfile/config_test/.gitignore renamed: cryptfs/config_test/PlaintextNames.conf -> internal/configfile/config_test/PlaintextNames.conf renamed: cryptfs/config_test/StrangeFeature.conf -> internal/configfile/config_test/StrangeFeature.conf renamed: cryptfs/config_test/v1.conf -> internal/configfile/config_test/v1.conf renamed: cryptfs/config_test/v2.conf -> internal/configfile/config_test/v2.conf renamed: cryptfs/kdf.go -> internal/configfile/kdf.go renamed: cryptfs/kdf_test.go -> internal/configfile/kdf_test.go renamed: cryptfs/cryptfs_content.go -> internal/contentenc/content.go new file: internal/contentenc/content_api.go renamed: cryptfs/content_test.go -> internal/contentenc/content_test.go renamed: cryptfs/file_header.go -> internal/contentenc/file_header.go renamed: cryptfs/intrablock.go -> internal/contentenc/intrablock.go renamed: cryptfs/address_translation.go -> internal/contentenc/offsets.go new file: internal/cryptocore/crypto_api.go renamed: cryptfs/gcm_go1.4.go -> internal/cryptocore/gcm_go1.4.go renamed: cryptfs/gcm_go1.5.go -> internal/cryptocore/gcm_go1.5.go renamed: cryptfs/nonce.go -> internal/cryptocore/nonce.go renamed: cryptfs/openssl_aead.go -> internal/cryptocore/openssl_aead.go renamed: cryptfs/openssl_benchmark.bash -> internal/cryptocore/openssl_benchmark.bash renamed: cryptfs/openssl_test.go -> internal/cryptocore/openssl_test.go new file: internal/nametransform/name_api.go new file: internal/nametransform/names_core.go renamed: cryptfs/names_diriv.go -> internal/nametransform/names_diriv.go renamed: cryptfs/names_noiv.go -> internal/nametransform/names_noiv.go renamed: cryptfs/names_test.go -> internal/nametransform/names_test.go new file: internal/nametransform/pad16.go renamed: cryptfs/log.go -> internal/toggledlog/log.go renamed: cryptfs/log_go1.4.go -> internal/toggledlog/log_go1.4.go renamed: cryptfs/log_go1.5.go -> internal/toggledlog/log_go1.5.go modified: main.go modified: masterkey.go modified: pathfs_frontend/file.go modified: pathfs_frontend/file_holes.go modified: pathfs_frontend/fs.go modified: pathfs_frontend/fs_dir.go modified: pathfs_frontend/names.go modified: test.bash
2016-02-06 19:20:54 +01:00
// Content encryption helper
contentEnc *contentenc.ContentEnc
// Device and inode number uniquely identify the backing file
qIno openfiletable.QIno
// Entry in the open file table
fileTableEntry *openfiletable.Entry
// go-fuse nodefs.loopbackFile
loopbackFile nodefs.File
// Store where the last byte was written
lastWrittenOffset int64
// The opCount is used to judge whether "lastWrittenOffset" is still
// guaranteed to be correct.
lastOpCount uint64
// Parent filesystem
fs *FS
// We embed a nodefs.NewDefaultFile() that returns ENOSYS for every operation we
// have not implemented. This prevents build breakage when the go-fuse library
// adds new methods to the nodefs.File interface.
nodefs.File
}
2016-10-02 06:14:18 +02:00
// NewFile returns a new go-fuse File instance.
func NewFile(fd *os.File, fs *FS) (*File, fuse.Status) {
var st syscall.Stat_t
err := syscall.Fstat(int(fd.Fd()), &st)
if err != nil {
tlog.Warn.Printf("NewFile: Fstat on fd %d failed: %v\n", fd.Fd(), err)
return nil, fuse.ToStatus(err)
}
qi := openfiletable.QInoFromStat(&st)
e := openfiletable.Register(qi)
return &File{
fd: fd,
contentEnc: fs.contentEnc,
qIno: qi,
fileTableEntry: e,
loopbackFile: nodefs.NewLoopbackFile(fd),
fs: fs,
File: nodefs.NewDefaultFile(),
}, fuse.OK
}
// intFd - return the backing file descriptor as an integer. Used for debug
// messages.
func (f *File) intFd() int {
return int(f.fd.Fd())
}
// readFileID loads the file header from disk and extracts the file ID.
// Returns io.EOF if the file is empty.
func (f *File) readFileID() ([]byte, error) {
// We read +1 byte to determine if the file has actual content
// and not only the header. A header-only file will be considered empty.
// This makes File ID poisoning more difficult.
readLen := contentenc.HeaderLen + 1
buf := make([]byte, readLen)
n, err := f.fd.ReadAt(buf, 0)
if err != nil {
if err == io.EOF && n != 0 {
tlog.Warn.Printf("readFileID %d: incomplete file, got %d instead of %d bytes",
f.qIno.Ino, n, readLen)
f.fs.reportMitigatedCorruption(fmt.Sprint(f.qIno.Ino))
}
return nil, err
}
buf = buf[:contentenc.HeaderLen]
Major refactoring: Split up "cryptfs" into several internal packages "git status" for reference: deleted: cryptfs/cryptfs.go deleted: cryptfs/names_core.go modified: integration_tests/cli_test.go modified: integration_tests/helpers.go renamed: cryptfs/config_file.go -> internal/configfile/config_file.go renamed: cryptfs/config_test.go -> internal/configfile/config_test.go renamed: cryptfs/config_test/.gitignore -> internal/configfile/config_test/.gitignore renamed: cryptfs/config_test/PlaintextNames.conf -> internal/configfile/config_test/PlaintextNames.conf renamed: cryptfs/config_test/StrangeFeature.conf -> internal/configfile/config_test/StrangeFeature.conf renamed: cryptfs/config_test/v1.conf -> internal/configfile/config_test/v1.conf renamed: cryptfs/config_test/v2.conf -> internal/configfile/config_test/v2.conf renamed: cryptfs/kdf.go -> internal/configfile/kdf.go renamed: cryptfs/kdf_test.go -> internal/configfile/kdf_test.go renamed: cryptfs/cryptfs_content.go -> internal/contentenc/content.go new file: internal/contentenc/content_api.go renamed: cryptfs/content_test.go -> internal/contentenc/content_test.go renamed: cryptfs/file_header.go -> internal/contentenc/file_header.go renamed: cryptfs/intrablock.go -> internal/contentenc/intrablock.go renamed: cryptfs/address_translation.go -> internal/contentenc/offsets.go new file: internal/cryptocore/crypto_api.go renamed: cryptfs/gcm_go1.4.go -> internal/cryptocore/gcm_go1.4.go renamed: cryptfs/gcm_go1.5.go -> internal/cryptocore/gcm_go1.5.go renamed: cryptfs/nonce.go -> internal/cryptocore/nonce.go renamed: cryptfs/openssl_aead.go -> internal/cryptocore/openssl_aead.go renamed: cryptfs/openssl_benchmark.bash -> internal/cryptocore/openssl_benchmark.bash renamed: cryptfs/openssl_test.go -> internal/cryptocore/openssl_test.go new file: internal/nametransform/name_api.go new file: internal/nametransform/names_core.go renamed: cryptfs/names_diriv.go -> internal/nametransform/names_diriv.go renamed: cryptfs/names_noiv.go -> internal/nametransform/names_noiv.go renamed: cryptfs/names_test.go -> internal/nametransform/names_test.go new file: internal/nametransform/pad16.go renamed: cryptfs/log.go -> internal/toggledlog/log.go renamed: cryptfs/log_go1.4.go -> internal/toggledlog/log_go1.4.go renamed: cryptfs/log_go1.5.go -> internal/toggledlog/log_go1.5.go modified: main.go modified: masterkey.go modified: pathfs_frontend/file.go modified: pathfs_frontend/file_holes.go modified: pathfs_frontend/fs.go modified: pathfs_frontend/fs_dir.go modified: pathfs_frontend/names.go modified: test.bash
2016-02-06 19:20:54 +01:00
h, err := contentenc.ParseHeader(buf)
if err != nil {
return nil, err
}
return h.ID, nil
}
// createHeader creates a new random header and writes it to disk.
// Returns the new file ID.
// The caller must hold fileIDLock.Lock().
func (f *File) createHeader() (fileID []byte, err error) {
Major refactoring: Split up "cryptfs" into several internal packages "git status" for reference: deleted: cryptfs/cryptfs.go deleted: cryptfs/names_core.go modified: integration_tests/cli_test.go modified: integration_tests/helpers.go renamed: cryptfs/config_file.go -> internal/configfile/config_file.go renamed: cryptfs/config_test.go -> internal/configfile/config_test.go renamed: cryptfs/config_test/.gitignore -> internal/configfile/config_test/.gitignore renamed: cryptfs/config_test/PlaintextNames.conf -> internal/configfile/config_test/PlaintextNames.conf renamed: cryptfs/config_test/StrangeFeature.conf -> internal/configfile/config_test/StrangeFeature.conf renamed: cryptfs/config_test/v1.conf -> internal/configfile/config_test/v1.conf renamed: cryptfs/config_test/v2.conf -> internal/configfile/config_test/v2.conf renamed: cryptfs/kdf.go -> internal/configfile/kdf.go renamed: cryptfs/kdf_test.go -> internal/configfile/kdf_test.go renamed: cryptfs/cryptfs_content.go -> internal/contentenc/content.go new file: internal/contentenc/content_api.go renamed: cryptfs/content_test.go -> internal/contentenc/content_test.go renamed: cryptfs/file_header.go -> internal/contentenc/file_header.go renamed: cryptfs/intrablock.go -> internal/contentenc/intrablock.go renamed: cryptfs/address_translation.go -> internal/contentenc/offsets.go new file: internal/cryptocore/crypto_api.go renamed: cryptfs/gcm_go1.4.go -> internal/cryptocore/gcm_go1.4.go renamed: cryptfs/gcm_go1.5.go -> internal/cryptocore/gcm_go1.5.go renamed: cryptfs/nonce.go -> internal/cryptocore/nonce.go renamed: cryptfs/openssl_aead.go -> internal/cryptocore/openssl_aead.go renamed: cryptfs/openssl_benchmark.bash -> internal/cryptocore/openssl_benchmark.bash renamed: cryptfs/openssl_test.go -> internal/cryptocore/openssl_test.go new file: internal/nametransform/name_api.go new file: internal/nametransform/names_core.go renamed: cryptfs/names_diriv.go -> internal/nametransform/names_diriv.go renamed: cryptfs/names_noiv.go -> internal/nametransform/names_noiv.go renamed: cryptfs/names_test.go -> internal/nametransform/names_test.go new file: internal/nametransform/pad16.go renamed: cryptfs/log.go -> internal/toggledlog/log.go renamed: cryptfs/log_go1.4.go -> internal/toggledlog/log_go1.4.go renamed: cryptfs/log_go1.5.go -> internal/toggledlog/log_go1.5.go modified: main.go modified: masterkey.go modified: pathfs_frontend/file.go modified: pathfs_frontend/file_holes.go modified: pathfs_frontend/fs.go modified: pathfs_frontend/fs_dir.go modified: pathfs_frontend/names.go modified: test.bash
2016-02-06 19:20:54 +01:00
h := contentenc.RandomHeader()
buf := h.Pack()
// Prevent partially written (=corrupt) header by preallocating the space beforehand
if !f.fs.args.NoPrealloc {
err = syscallcompat.EnospcPrealloc(int(f.fd.Fd()), 0, contentenc.HeaderLen)
if err != nil {
if !syscallcompat.IsENOSPC(err) {
tlog.Warn.Printf("ino%d: createHeader: prealloc failed: %s\n", f.qIno.Ino, err.Error())
}
return nil, err
}
}
// Actually write header
_, err = f.fd.WriteAt(buf, 0)
if err != nil {
return nil, err
}
return h.ID, err
}
// doRead - read "length" plaintext bytes from plaintext offset "off" and append
// to "dst".
// Arguments "length" and "off" do not have to be block-aligned.
2015-09-30 22:36:53 +02:00
//
// doRead reads the corresponding ciphertext blocks from disk, decrypts them and
// returns the requested part of the plaintext.
//
// Called by Read() for normal reading,
// by Write() and Truncate() via doWrite() for Read-Modify-Write.
func (f *File) doRead(dst []byte, off uint64, length uint64) ([]byte, fuse.Status) {
// Get the file ID, either from the open file table, or from disk.
var fileID []byte
f.fileTableEntry.IDLock.Lock()
if f.fileTableEntry.ID != nil {
// Use the cached value in the file table
fileID = f.fileTableEntry.ID
} else {
// Not cached, we have to read it from disk.
var err error
fileID, err = f.readFileID()
if err != nil {
f.fileTableEntry.IDLock.Unlock()
if err == io.EOF {
// Empty file
return nil, fuse.OK
}
if err != nil {
tlog.Warn.Printf("doRead %d: corrupt header: %v", f.qIno.Ino, err)
return nil, fuse.EIO
}
}
// Save into the file table
f.fileTableEntry.ID = fileID
}
f.fileTableEntry.IDLock.Unlock()
if fileID == nil {
log.Panicf("fileID=%v", fileID)
}
// Read the backing ciphertext in one go
Major refactoring: Split up "cryptfs" into several internal packages "git status" for reference: deleted: cryptfs/cryptfs.go deleted: cryptfs/names_core.go modified: integration_tests/cli_test.go modified: integration_tests/helpers.go renamed: cryptfs/config_file.go -> internal/configfile/config_file.go renamed: cryptfs/config_test.go -> internal/configfile/config_test.go renamed: cryptfs/config_test/.gitignore -> internal/configfile/config_test/.gitignore renamed: cryptfs/config_test/PlaintextNames.conf -> internal/configfile/config_test/PlaintextNames.conf renamed: cryptfs/config_test/StrangeFeature.conf -> internal/configfile/config_test/StrangeFeature.conf renamed: cryptfs/config_test/v1.conf -> internal/configfile/config_test/v1.conf renamed: cryptfs/config_test/v2.conf -> internal/configfile/config_test/v2.conf renamed: cryptfs/kdf.go -> internal/configfile/kdf.go renamed: cryptfs/kdf_test.go -> internal/configfile/kdf_test.go renamed: cryptfs/cryptfs_content.go -> internal/contentenc/content.go new file: internal/contentenc/content_api.go renamed: cryptfs/content_test.go -> internal/contentenc/content_test.go renamed: cryptfs/file_header.go -> internal/contentenc/file_header.go renamed: cryptfs/intrablock.go -> internal/contentenc/intrablock.go renamed: cryptfs/address_translation.go -> internal/contentenc/offsets.go new file: internal/cryptocore/crypto_api.go renamed: cryptfs/gcm_go1.4.go -> internal/cryptocore/gcm_go1.4.go renamed: cryptfs/gcm_go1.5.go -> internal/cryptocore/gcm_go1.5.go renamed: cryptfs/nonce.go -> internal/cryptocore/nonce.go renamed: cryptfs/openssl_aead.go -> internal/cryptocore/openssl_aead.go renamed: cryptfs/openssl_benchmark.bash -> internal/cryptocore/openssl_benchmark.bash renamed: cryptfs/openssl_test.go -> internal/cryptocore/openssl_test.go new file: internal/nametransform/name_api.go new file: internal/nametransform/names_core.go renamed: cryptfs/names_diriv.go -> internal/nametransform/names_diriv.go renamed: cryptfs/names_noiv.go -> internal/nametransform/names_noiv.go renamed: cryptfs/names_test.go -> internal/nametransform/names_test.go new file: internal/nametransform/pad16.go renamed: cryptfs/log.go -> internal/toggledlog/log.go renamed: cryptfs/log_go1.4.go -> internal/toggledlog/log_go1.4.go renamed: cryptfs/log_go1.5.go -> internal/toggledlog/log_go1.5.go modified: main.go modified: masterkey.go modified: pathfs_frontend/file.go modified: pathfs_frontend/file_holes.go modified: pathfs_frontend/fs.go modified: pathfs_frontend/fs_dir.go modified: pathfs_frontend/names.go modified: test.bash
2016-02-06 19:20:54 +01:00
blocks := f.contentEnc.ExplodePlainRange(off, length)
alignedOffset, alignedLength := blocks[0].JointCiphertextRange(blocks)
skip := blocks[0].Skip
tlog.Debug.Printf("doRead: off=%d len=%d -> off=%d len=%d skip=%d\n",
off, length, alignedOffset, alignedLength, skip)
ciphertext := f.fs.contentEnc.CReqPool.Get()
ciphertext = ciphertext[:int(alignedLength)]
n, err := f.fd.ReadAt(ciphertext, int64(alignedOffset))
if err != nil && err != io.EOF {
tlog.Warn.Printf("read: ReadAt: %s", err.Error())
return nil, fuse.ToStatus(err)
}
// The ReadAt came back empty. We can skip all the decryption and return early.
if n == 0 {
f.fs.contentEnc.CReqPool.Put(ciphertext)
return dst, fuse.OK
}
// Truncate ciphertext buffer down to actually read bytes
ciphertext = ciphertext[0:n]
firstBlockNo := blocks[0].BlockNo
tlog.Debug.Printf("ReadAt offset=%d bytes (%d blocks), want=%d, got=%d", alignedOffset, firstBlockNo, alignedLength, n)
// Decrypt it
plaintext, err := f.contentEnc.DecryptBlocks(ciphertext, firstBlockNo, fileID)
f.fs.contentEnc.CReqPool.Put(ciphertext)
if err != nil {
if f.fs.args.ForceDecode && err == stupidgcm.ErrAuth {
// We do not have the information which block was corrupt here anymore,
// but DecryptBlocks() has already logged it anyway.
tlog.Warn.Printf("doRead %d: off=%d len=%d: returning corrupt data due to forcedecode",
f.qIno.Ino, off, length)
} else {
curruptBlockNo := firstBlockNo + f.contentEnc.PlainOffToBlockNo(uint64(len(plaintext)))
tlog.Warn.Printf("doRead %d: corrupt block #%d: %v", f.qIno.Ino, curruptBlockNo, err)
return nil, fuse.EIO
}
}
// Crop down to the relevant part
var out []byte
lenHave := len(plaintext)
lenWant := int(skip + length)
if lenHave > lenWant {
out = plaintext[skip:lenWant]
} else if lenHave > int(skip) {
out = plaintext[skip:lenHave]
}
// else: out stays empty, file was smaller than the requested offset
out = append(dst, out...)
f.fs.contentEnc.PReqPool.Put(plaintext)
return out, fuse.OK
}
// Read - FUSE call
func (f *File) Read(buf []byte, off int64) (resultData fuse.ReadResult, code fuse.Status) {
if len(buf) > fuse.MAX_KERNEL_WRITE {
// This would crash us due to our fixed-size buffer pool
tlog.Warn.Printf("Read: rejecting oversized request with EMSGSIZE, len=%d", len(buf))
return nil, fuse.Status(syscall.EMSGSIZE)
}
f.fdLock.RLock()
defer f.fdLock.RUnlock()
f.fileTableEntry.ContentLock.RLock()
defer f.fileTableEntry.ContentLock.RUnlock()
tlog.Debug.Printf("ino%d: FUSE Read: offset=%d length=%d", f.qIno.Ino, len(buf), off)
if f.fs.args.SerializeReads {
serialize_reads.Wait(off, len(buf))
}
out, status := f.doRead(buf[:0], uint64(off), uint64(len(buf)))
if f.fs.args.SerializeReads {
serialize_reads.Done()
}
if status != fuse.OK {
return nil, status
}
tlog.Debug.Printf("ino%d: Read: status %v, returning %d bytes", f.qIno.Ino, status, len(out))
return fuse.ReadResultData(out), status
}
// doWrite - encrypt "data" and write it to plaintext offset "off"
//
// Arguments do not have to be block-aligned, read-modify-write is
// performed internally as necessary
//
// Called by Write() for normal writing,
// and by Truncate() to rewrite the last file block.
//
// Empty writes do nothing and are allowed.
func (f *File) doWrite(data []byte, off int64) (uint32, fuse.Status) {
fileWasEmpty := false
// Get the file ID, create a new one if it does not exist yet.
var fileID []byte
// The caller has exclusively locked ContentLock, which blocks all other
// readers and writers. No need to take IDLock.
if f.fileTableEntry.ID != nil {
fileID = f.fileTableEntry.ID
} else {
// If the file ID is not cached, read it from disk
var err error
fileID, err = f.readFileID()
// Write a new file header if the file is empty
if err == io.EOF {
fileID, err = f.createHeader()
fileWasEmpty = true
}
if err != nil {
return 0, fuse.ToStatus(err)
}
f.fileTableEntry.ID = fileID
}
fusefrontend: coalesce 4kB writes This improves performance on hdds running ext4, and improves streaming write performance on hdds running btrfs. Tar extract slows down on btrfs for some reason. See https://github.com/rfjakob/gocryptfs/issues/63 Benchmarks: encfs v1.9.1 ============ $ ./benchmark.bash -encfs /mnt/hdd-ext4 Testing EncFS at /mnt/hdd-ext4/benchmark.bash.u0g WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 1,48354 s, 88,4 MB/s UNTAR: 20.79 LS: 3.04 RM: 6.62 $ ./benchmark.bash -encfs /mnt/hdd-btrfs Testing EncFS at /mnt/hdd-btrfs/benchmark.bash.h40 WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 1,52552 s, 85,9 MB/s UNTAR: 24.51 LS: 2.73 RM: 5.32 gocryptfs v1.1.1-26-g4a7f8ef ============================ $ ./benchmark.bash /mnt/hdd-ext4 Testing gocryptfs at /mnt/hdd-ext4/benchmark.bash.1KG WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 1,55782 s, 84,1 MB/s UNTAR: 22.23 LS: 1.47 RM: 4.17 $ ./benchmark.bash /mnt/hdd-btrfs Testing gocryptfs at /mnt/hdd-btrfs/benchmark.bash.2t8 WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 6,87206 s, 19,1 MB/s UNTAR: 69.87 LS: 1.52 RM: 5.33 gocryptfs v1.1.1-32 =================== $ ./benchmark.bash /mnt/hdd-ext4 Testing gocryptfs at /mnt/hdd-ext4/benchmark.bash.Qt3 WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 1,22577 s, 107 MB/s UNTAR: 23.46 LS: 1.46 RM: 4.67 $ ./benchmark.bash /mnt/hdd-btrfs/ Testing gocryptfs at /mnt/hdd-btrfs//benchmark.bash.XVk WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 3,68735 s, 35,5 MB/s UNTAR: 116.87 LS: 1.84 RM: 6.34
2016-11-24 00:03:30 +01:00
// Handle payload data
dataBuf := bytes.NewBuffer(data)
Major refactoring: Split up "cryptfs" into several internal packages "git status" for reference: deleted: cryptfs/cryptfs.go deleted: cryptfs/names_core.go modified: integration_tests/cli_test.go modified: integration_tests/helpers.go renamed: cryptfs/config_file.go -> internal/configfile/config_file.go renamed: cryptfs/config_test.go -> internal/configfile/config_test.go renamed: cryptfs/config_test/.gitignore -> internal/configfile/config_test/.gitignore renamed: cryptfs/config_test/PlaintextNames.conf -> internal/configfile/config_test/PlaintextNames.conf renamed: cryptfs/config_test/StrangeFeature.conf -> internal/configfile/config_test/StrangeFeature.conf renamed: cryptfs/config_test/v1.conf -> internal/configfile/config_test/v1.conf renamed: cryptfs/config_test/v2.conf -> internal/configfile/config_test/v2.conf renamed: cryptfs/kdf.go -> internal/configfile/kdf.go renamed: cryptfs/kdf_test.go -> internal/configfile/kdf_test.go renamed: cryptfs/cryptfs_content.go -> internal/contentenc/content.go new file: internal/contentenc/content_api.go renamed: cryptfs/content_test.go -> internal/contentenc/content_test.go renamed: cryptfs/file_header.go -> internal/contentenc/file_header.go renamed: cryptfs/intrablock.go -> internal/contentenc/intrablock.go renamed: cryptfs/address_translation.go -> internal/contentenc/offsets.go new file: internal/cryptocore/crypto_api.go renamed: cryptfs/gcm_go1.4.go -> internal/cryptocore/gcm_go1.4.go renamed: cryptfs/gcm_go1.5.go -> internal/cryptocore/gcm_go1.5.go renamed: cryptfs/nonce.go -> internal/cryptocore/nonce.go renamed: cryptfs/openssl_aead.go -> internal/cryptocore/openssl_aead.go renamed: cryptfs/openssl_benchmark.bash -> internal/cryptocore/openssl_benchmark.bash renamed: cryptfs/openssl_test.go -> internal/cryptocore/openssl_test.go new file: internal/nametransform/name_api.go new file: internal/nametransform/names_core.go renamed: cryptfs/names_diriv.go -> internal/nametransform/names_diriv.go renamed: cryptfs/names_noiv.go -> internal/nametransform/names_noiv.go renamed: cryptfs/names_test.go -> internal/nametransform/names_test.go new file: internal/nametransform/pad16.go renamed: cryptfs/log.go -> internal/toggledlog/log.go renamed: cryptfs/log_go1.4.go -> internal/toggledlog/log_go1.4.go renamed: cryptfs/log_go1.5.go -> internal/toggledlog/log_go1.5.go modified: main.go modified: masterkey.go modified: pathfs_frontend/file.go modified: pathfs_frontend/file_holes.go modified: pathfs_frontend/fs.go modified: pathfs_frontend/fs_dir.go modified: pathfs_frontend/names.go modified: test.bash
2016-02-06 19:20:54 +01:00
blocks := f.contentEnc.ExplodePlainRange(uint64(off), uint64(len(data)))
toEncrypt := make([][]byte, len(blocks))
fusefrontend: coalesce 4kB writes This improves performance on hdds running ext4, and improves streaming write performance on hdds running btrfs. Tar extract slows down on btrfs for some reason. See https://github.com/rfjakob/gocryptfs/issues/63 Benchmarks: encfs v1.9.1 ============ $ ./benchmark.bash -encfs /mnt/hdd-ext4 Testing EncFS at /mnt/hdd-ext4/benchmark.bash.u0g WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 1,48354 s, 88,4 MB/s UNTAR: 20.79 LS: 3.04 RM: 6.62 $ ./benchmark.bash -encfs /mnt/hdd-btrfs Testing EncFS at /mnt/hdd-btrfs/benchmark.bash.h40 WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 1,52552 s, 85,9 MB/s UNTAR: 24.51 LS: 2.73 RM: 5.32 gocryptfs v1.1.1-26-g4a7f8ef ============================ $ ./benchmark.bash /mnt/hdd-ext4 Testing gocryptfs at /mnt/hdd-ext4/benchmark.bash.1KG WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 1,55782 s, 84,1 MB/s UNTAR: 22.23 LS: 1.47 RM: 4.17 $ ./benchmark.bash /mnt/hdd-btrfs Testing gocryptfs at /mnt/hdd-btrfs/benchmark.bash.2t8 WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 6,87206 s, 19,1 MB/s UNTAR: 69.87 LS: 1.52 RM: 5.33 gocryptfs v1.1.1-32 =================== $ ./benchmark.bash /mnt/hdd-ext4 Testing gocryptfs at /mnt/hdd-ext4/benchmark.bash.Qt3 WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 1,22577 s, 107 MB/s UNTAR: 23.46 LS: 1.46 RM: 4.67 $ ./benchmark.bash /mnt/hdd-btrfs/ Testing gocryptfs at /mnt/hdd-btrfs//benchmark.bash.XVk WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 3,68735 s, 35,5 MB/s UNTAR: 116.87 LS: 1.84 RM: 6.34
2016-11-24 00:03:30 +01:00
for i, b := range blocks {
blockData := dataBuf.Next(int(b.Length))
// Incomplete block -> Read-Modify-Write
if b.IsPartial() {
// Read
oldData, status := f.doRead(nil, b.BlockPlainOff(), f.contentEnc.PlainBS())
if status != fuse.OK {
tlog.Warn.Printf("ino%d fh%d: RMW read failed: %s", f.qIno.Ino, f.intFd(), status.String())
fusefrontend: coalesce 4kB writes This improves performance on hdds running ext4, and improves streaming write performance on hdds running btrfs. Tar extract slows down on btrfs for some reason. See https://github.com/rfjakob/gocryptfs/issues/63 Benchmarks: encfs v1.9.1 ============ $ ./benchmark.bash -encfs /mnt/hdd-ext4 Testing EncFS at /mnt/hdd-ext4/benchmark.bash.u0g WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 1,48354 s, 88,4 MB/s UNTAR: 20.79 LS: 3.04 RM: 6.62 $ ./benchmark.bash -encfs /mnt/hdd-btrfs Testing EncFS at /mnt/hdd-btrfs/benchmark.bash.h40 WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 1,52552 s, 85,9 MB/s UNTAR: 24.51 LS: 2.73 RM: 5.32 gocryptfs v1.1.1-26-g4a7f8ef ============================ $ ./benchmark.bash /mnt/hdd-ext4 Testing gocryptfs at /mnt/hdd-ext4/benchmark.bash.1KG WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 1,55782 s, 84,1 MB/s UNTAR: 22.23 LS: 1.47 RM: 4.17 $ ./benchmark.bash /mnt/hdd-btrfs Testing gocryptfs at /mnt/hdd-btrfs/benchmark.bash.2t8 WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 6,87206 s, 19,1 MB/s UNTAR: 69.87 LS: 1.52 RM: 5.33 gocryptfs v1.1.1-32 =================== $ ./benchmark.bash /mnt/hdd-ext4 Testing gocryptfs at /mnt/hdd-ext4/benchmark.bash.Qt3 WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 1,22577 s, 107 MB/s UNTAR: 23.46 LS: 1.46 RM: 4.67 $ ./benchmark.bash /mnt/hdd-btrfs/ Testing gocryptfs at /mnt/hdd-btrfs//benchmark.bash.XVk WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 3,68735 s, 35,5 MB/s UNTAR: 116.87 LS: 1.84 RM: 6.34
2016-11-24 00:03:30 +01:00
return 0, status
}
// Modify
Major refactoring: Split up "cryptfs" into several internal packages "git status" for reference: deleted: cryptfs/cryptfs.go deleted: cryptfs/names_core.go modified: integration_tests/cli_test.go modified: integration_tests/helpers.go renamed: cryptfs/config_file.go -> internal/configfile/config_file.go renamed: cryptfs/config_test.go -> internal/configfile/config_test.go renamed: cryptfs/config_test/.gitignore -> internal/configfile/config_test/.gitignore renamed: cryptfs/config_test/PlaintextNames.conf -> internal/configfile/config_test/PlaintextNames.conf renamed: cryptfs/config_test/StrangeFeature.conf -> internal/configfile/config_test/StrangeFeature.conf renamed: cryptfs/config_test/v1.conf -> internal/configfile/config_test/v1.conf renamed: cryptfs/config_test/v2.conf -> internal/configfile/config_test/v2.conf renamed: cryptfs/kdf.go -> internal/configfile/kdf.go renamed: cryptfs/kdf_test.go -> internal/configfile/kdf_test.go renamed: cryptfs/cryptfs_content.go -> internal/contentenc/content.go new file: internal/contentenc/content_api.go renamed: cryptfs/content_test.go -> internal/contentenc/content_test.go renamed: cryptfs/file_header.go -> internal/contentenc/file_header.go renamed: cryptfs/intrablock.go -> internal/contentenc/intrablock.go renamed: cryptfs/address_translation.go -> internal/contentenc/offsets.go new file: internal/cryptocore/crypto_api.go renamed: cryptfs/gcm_go1.4.go -> internal/cryptocore/gcm_go1.4.go renamed: cryptfs/gcm_go1.5.go -> internal/cryptocore/gcm_go1.5.go renamed: cryptfs/nonce.go -> internal/cryptocore/nonce.go renamed: cryptfs/openssl_aead.go -> internal/cryptocore/openssl_aead.go renamed: cryptfs/openssl_benchmark.bash -> internal/cryptocore/openssl_benchmark.bash renamed: cryptfs/openssl_test.go -> internal/cryptocore/openssl_test.go new file: internal/nametransform/name_api.go new file: internal/nametransform/names_core.go renamed: cryptfs/names_diriv.go -> internal/nametransform/names_diriv.go renamed: cryptfs/names_noiv.go -> internal/nametransform/names_noiv.go renamed: cryptfs/names_test.go -> internal/nametransform/names_test.go new file: internal/nametransform/pad16.go renamed: cryptfs/log.go -> internal/toggledlog/log.go renamed: cryptfs/log_go1.4.go -> internal/toggledlog/log_go1.4.go renamed: cryptfs/log_go1.5.go -> internal/toggledlog/log_go1.5.go modified: main.go modified: masterkey.go modified: pathfs_frontend/file.go modified: pathfs_frontend/file_holes.go modified: pathfs_frontend/fs.go modified: pathfs_frontend/fs_dir.go modified: pathfs_frontend/names.go modified: test.bash
2016-02-06 19:20:54 +01:00
blockData = f.contentEnc.MergeBlocks(oldData, blockData, int(b.Skip))
tlog.Debug.Printf("len(oldData)=%d len(blockData)=%d", len(oldData), len(blockData))
}
tlog.Debug.Printf("ino%d: Writing %d bytes to block #%d",
f.qIno.Ino, uint64(len(blockData))-f.contentEnc.BlockOverhead(), b.BlockNo)
// Write into the to-encrypt list
toEncrypt[i] = blockData
fusefrontend: coalesce 4kB writes This improves performance on hdds running ext4, and improves streaming write performance on hdds running btrfs. Tar extract slows down on btrfs for some reason. See https://github.com/rfjakob/gocryptfs/issues/63 Benchmarks: encfs v1.9.1 ============ $ ./benchmark.bash -encfs /mnt/hdd-ext4 Testing EncFS at /mnt/hdd-ext4/benchmark.bash.u0g WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 1,48354 s, 88,4 MB/s UNTAR: 20.79 LS: 3.04 RM: 6.62 $ ./benchmark.bash -encfs /mnt/hdd-btrfs Testing EncFS at /mnt/hdd-btrfs/benchmark.bash.h40 WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 1,52552 s, 85,9 MB/s UNTAR: 24.51 LS: 2.73 RM: 5.32 gocryptfs v1.1.1-26-g4a7f8ef ============================ $ ./benchmark.bash /mnt/hdd-ext4 Testing gocryptfs at /mnt/hdd-ext4/benchmark.bash.1KG WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 1,55782 s, 84,1 MB/s UNTAR: 22.23 LS: 1.47 RM: 4.17 $ ./benchmark.bash /mnt/hdd-btrfs Testing gocryptfs at /mnt/hdd-btrfs/benchmark.bash.2t8 WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 6,87206 s, 19,1 MB/s UNTAR: 69.87 LS: 1.52 RM: 5.33 gocryptfs v1.1.1-32 =================== $ ./benchmark.bash /mnt/hdd-ext4 Testing gocryptfs at /mnt/hdd-ext4/benchmark.bash.Qt3 WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 1,22577 s, 107 MB/s UNTAR: 23.46 LS: 1.46 RM: 4.67 $ ./benchmark.bash /mnt/hdd-btrfs/ Testing gocryptfs at /mnt/hdd-btrfs//benchmark.bash.XVk WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 3,68735 s, 35,5 MB/s UNTAR: 116.87 LS: 1.84 RM: 6.34
2016-11-24 00:03:30 +01:00
}
// Encrypt all blocks
ciphertext := f.contentEnc.EncryptBlocks(toEncrypt, blocks[0].BlockNo, f.fileTableEntry.ID)
fusefrontend: coalesce 4kB writes This improves performance on hdds running ext4, and improves streaming write performance on hdds running btrfs. Tar extract slows down on btrfs for some reason. See https://github.com/rfjakob/gocryptfs/issues/63 Benchmarks: encfs v1.9.1 ============ $ ./benchmark.bash -encfs /mnt/hdd-ext4 Testing EncFS at /mnt/hdd-ext4/benchmark.bash.u0g WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 1,48354 s, 88,4 MB/s UNTAR: 20.79 LS: 3.04 RM: 6.62 $ ./benchmark.bash -encfs /mnt/hdd-btrfs Testing EncFS at /mnt/hdd-btrfs/benchmark.bash.h40 WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 1,52552 s, 85,9 MB/s UNTAR: 24.51 LS: 2.73 RM: 5.32 gocryptfs v1.1.1-26-g4a7f8ef ============================ $ ./benchmark.bash /mnt/hdd-ext4 Testing gocryptfs at /mnt/hdd-ext4/benchmark.bash.1KG WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 1,55782 s, 84,1 MB/s UNTAR: 22.23 LS: 1.47 RM: 4.17 $ ./benchmark.bash /mnt/hdd-btrfs Testing gocryptfs at /mnt/hdd-btrfs/benchmark.bash.2t8 WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 6,87206 s, 19,1 MB/s UNTAR: 69.87 LS: 1.52 RM: 5.33 gocryptfs v1.1.1-32 =================== $ ./benchmark.bash /mnt/hdd-ext4 Testing gocryptfs at /mnt/hdd-ext4/benchmark.bash.Qt3 WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 1,22577 s, 107 MB/s UNTAR: 23.46 LS: 1.46 RM: 4.67 $ ./benchmark.bash /mnt/hdd-btrfs/ Testing gocryptfs at /mnt/hdd-btrfs//benchmark.bash.XVk WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 3,68735 s, 35,5 MB/s UNTAR: 116.87 LS: 1.84 RM: 6.34
2016-11-24 00:03:30 +01:00
// Preallocate so we cannot run out of space in the middle of the write.
// This prevents partially written (=corrupt) blocks.
var err error
cOff := int64(blocks[0].BlockCipherOff())
if !f.fs.args.NoPrealloc {
err = syscallcompat.EnospcPrealloc(int(f.fd.Fd()), cOff, int64(len(ciphertext)))
if err != nil {
if !syscallcompat.IsENOSPC(err) {
tlog.Warn.Printf("ino%d fh%d: doWrite: prealloc failed: %v", f.qIno.Ino, f.intFd(), err)
}
if fileWasEmpty {
// Kill the file header again
f.fileTableEntry.ID = nil
err2 := syscall.Ftruncate(int(f.fd.Fd()), 0)
if err2 != nil {
tlog.Warn.Printf("ino%d fh%d: doWrite: rollback failed: %v", f.qIno.Ino, f.intFd(), err2)
}
}
return 0, fuse.ToStatus(err)
}
fusefrontend: coalesce 4kB writes This improves performance on hdds running ext4, and improves streaming write performance on hdds running btrfs. Tar extract slows down on btrfs for some reason. See https://github.com/rfjakob/gocryptfs/issues/63 Benchmarks: encfs v1.9.1 ============ $ ./benchmark.bash -encfs /mnt/hdd-ext4 Testing EncFS at /mnt/hdd-ext4/benchmark.bash.u0g WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 1,48354 s, 88,4 MB/s UNTAR: 20.79 LS: 3.04 RM: 6.62 $ ./benchmark.bash -encfs /mnt/hdd-btrfs Testing EncFS at /mnt/hdd-btrfs/benchmark.bash.h40 WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 1,52552 s, 85,9 MB/s UNTAR: 24.51 LS: 2.73 RM: 5.32 gocryptfs v1.1.1-26-g4a7f8ef ============================ $ ./benchmark.bash /mnt/hdd-ext4 Testing gocryptfs at /mnt/hdd-ext4/benchmark.bash.1KG WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 1,55782 s, 84,1 MB/s UNTAR: 22.23 LS: 1.47 RM: 4.17 $ ./benchmark.bash /mnt/hdd-btrfs Testing gocryptfs at /mnt/hdd-btrfs/benchmark.bash.2t8 WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 6,87206 s, 19,1 MB/s UNTAR: 69.87 LS: 1.52 RM: 5.33 gocryptfs v1.1.1-32 =================== $ ./benchmark.bash /mnt/hdd-ext4 Testing gocryptfs at /mnt/hdd-ext4/benchmark.bash.Qt3 WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 1,22577 s, 107 MB/s UNTAR: 23.46 LS: 1.46 RM: 4.67 $ ./benchmark.bash /mnt/hdd-btrfs/ Testing gocryptfs at /mnt/hdd-btrfs//benchmark.bash.XVk WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 3,68735 s, 35,5 MB/s UNTAR: 116.87 LS: 1.84 RM: 6.34
2016-11-24 00:03:30 +01:00
}
// Write
_, err = f.fd.WriteAt(ciphertext, cOff)
// Return memory to CReqPool
f.fs.contentEnc.CReqPool.Put(ciphertext)
fusefrontend: coalesce 4kB writes This improves performance on hdds running ext4, and improves streaming write performance on hdds running btrfs. Tar extract slows down on btrfs for some reason. See https://github.com/rfjakob/gocryptfs/issues/63 Benchmarks: encfs v1.9.1 ============ $ ./benchmark.bash -encfs /mnt/hdd-ext4 Testing EncFS at /mnt/hdd-ext4/benchmark.bash.u0g WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 1,48354 s, 88,4 MB/s UNTAR: 20.79 LS: 3.04 RM: 6.62 $ ./benchmark.bash -encfs /mnt/hdd-btrfs Testing EncFS at /mnt/hdd-btrfs/benchmark.bash.h40 WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 1,52552 s, 85,9 MB/s UNTAR: 24.51 LS: 2.73 RM: 5.32 gocryptfs v1.1.1-26-g4a7f8ef ============================ $ ./benchmark.bash /mnt/hdd-ext4 Testing gocryptfs at /mnt/hdd-ext4/benchmark.bash.1KG WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 1,55782 s, 84,1 MB/s UNTAR: 22.23 LS: 1.47 RM: 4.17 $ ./benchmark.bash /mnt/hdd-btrfs Testing gocryptfs at /mnt/hdd-btrfs/benchmark.bash.2t8 WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 6,87206 s, 19,1 MB/s UNTAR: 69.87 LS: 1.52 RM: 5.33 gocryptfs v1.1.1-32 =================== $ ./benchmark.bash /mnt/hdd-ext4 Testing gocryptfs at /mnt/hdd-ext4/benchmark.bash.Qt3 WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 1,22577 s, 107 MB/s UNTAR: 23.46 LS: 1.46 RM: 4.67 $ ./benchmark.bash /mnt/hdd-btrfs/ Testing gocryptfs at /mnt/hdd-btrfs//benchmark.bash.XVk WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 3,68735 s, 35,5 MB/s UNTAR: 116.87 LS: 1.84 RM: 6.34
2016-11-24 00:03:30 +01:00
if err != nil {
tlog.Warn.Printf("ino%d fh%d: doWrite: WriteAt off=%d len=%d failed: %v",
f.qIno.Ino, f.intFd(), cOff, len(ciphertext), err)
fusefrontend: coalesce 4kB writes This improves performance on hdds running ext4, and improves streaming write performance on hdds running btrfs. Tar extract slows down on btrfs for some reason. See https://github.com/rfjakob/gocryptfs/issues/63 Benchmarks: encfs v1.9.1 ============ $ ./benchmark.bash -encfs /mnt/hdd-ext4 Testing EncFS at /mnt/hdd-ext4/benchmark.bash.u0g WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 1,48354 s, 88,4 MB/s UNTAR: 20.79 LS: 3.04 RM: 6.62 $ ./benchmark.bash -encfs /mnt/hdd-btrfs Testing EncFS at /mnt/hdd-btrfs/benchmark.bash.h40 WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 1,52552 s, 85,9 MB/s UNTAR: 24.51 LS: 2.73 RM: 5.32 gocryptfs v1.1.1-26-g4a7f8ef ============================ $ ./benchmark.bash /mnt/hdd-ext4 Testing gocryptfs at /mnt/hdd-ext4/benchmark.bash.1KG WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 1,55782 s, 84,1 MB/s UNTAR: 22.23 LS: 1.47 RM: 4.17 $ ./benchmark.bash /mnt/hdd-btrfs Testing gocryptfs at /mnt/hdd-btrfs/benchmark.bash.2t8 WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 6,87206 s, 19,1 MB/s UNTAR: 69.87 LS: 1.52 RM: 5.33 gocryptfs v1.1.1-32 =================== $ ./benchmark.bash /mnt/hdd-ext4 Testing gocryptfs at /mnt/hdd-ext4/benchmark.bash.Qt3 WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 1,22577 s, 107 MB/s UNTAR: 23.46 LS: 1.46 RM: 4.67 $ ./benchmark.bash /mnt/hdd-btrfs/ Testing gocryptfs at /mnt/hdd-btrfs//benchmark.bash.XVk WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 3,68735 s, 35,5 MB/s UNTAR: 116.87 LS: 1.84 RM: 6.34
2016-11-24 00:03:30 +01:00
return 0, fuse.ToStatus(err)
}
fusefrontend: coalesce 4kB writes This improves performance on hdds running ext4, and improves streaming write performance on hdds running btrfs. Tar extract slows down on btrfs for some reason. See https://github.com/rfjakob/gocryptfs/issues/63 Benchmarks: encfs v1.9.1 ============ $ ./benchmark.bash -encfs /mnt/hdd-ext4 Testing EncFS at /mnt/hdd-ext4/benchmark.bash.u0g WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 1,48354 s, 88,4 MB/s UNTAR: 20.79 LS: 3.04 RM: 6.62 $ ./benchmark.bash -encfs /mnt/hdd-btrfs Testing EncFS at /mnt/hdd-btrfs/benchmark.bash.h40 WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 1,52552 s, 85,9 MB/s UNTAR: 24.51 LS: 2.73 RM: 5.32 gocryptfs v1.1.1-26-g4a7f8ef ============================ $ ./benchmark.bash /mnt/hdd-ext4 Testing gocryptfs at /mnt/hdd-ext4/benchmark.bash.1KG WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 1,55782 s, 84,1 MB/s UNTAR: 22.23 LS: 1.47 RM: 4.17 $ ./benchmark.bash /mnt/hdd-btrfs Testing gocryptfs at /mnt/hdd-btrfs/benchmark.bash.2t8 WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 6,87206 s, 19,1 MB/s UNTAR: 69.87 LS: 1.52 RM: 5.33 gocryptfs v1.1.1-32 =================== $ ./benchmark.bash /mnt/hdd-ext4 Testing gocryptfs at /mnt/hdd-ext4/benchmark.bash.Qt3 WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 1,22577 s, 107 MB/s UNTAR: 23.46 LS: 1.46 RM: 4.67 $ ./benchmark.bash /mnt/hdd-btrfs/ Testing gocryptfs at /mnt/hdd-btrfs//benchmark.bash.XVk WRITE: 131072000 bytes (131 MB, 125 MiB) copied, 3,68735 s, 35,5 MB/s UNTAR: 116.87 LS: 1.84 RM: 6.34
2016-11-24 00:03:30 +01:00
return uint32(len(data)), fuse.OK
}
// isConsecutiveWrite returns true if the current write
// directly (in time and space) follows the last write.
// This is an optimisation for streaming writes on NFS where a
// Stat() call is very expensive.
// The caller must "wlock.lock(f.devIno.ino)" otherwise this check would be racy.
func (f *File) isConsecutiveWrite(off int64) bool {
opCount := openfiletable.WriteOpCount()
return opCount == f.lastOpCount+1 && off == f.lastWrittenOffset+1
}
// Write - FUSE call
//
// If the write creates a hole, pads the file to the next block boundary.
func (f *File) Write(data []byte, off int64) (uint32, fuse.Status) {
if len(data) > fuse.MAX_KERNEL_WRITE {
// This would crash us due to our fixed-size buffer pool
tlog.Warn.Printf("Write: rejecting oversized request with EMSGSIZE, len=%d", len(data))
return 0, fuse.Status(syscall.EMSGSIZE)
}
f.fdLock.RLock()
defer f.fdLock.RUnlock()
if f.released {
// The file descriptor has been closed concurrently
tlog.Warn.Printf("ino%d fh%d: Write on released file", f.qIno.Ino, f.intFd())
return 0, fuse.EBADF
}
f.fileTableEntry.ContentLock.Lock()
defer f.fileTableEntry.ContentLock.Unlock()
tlog.Debug.Printf("ino%d: FUSE Write: offset=%d length=%d", f.qIno.Ino, off, len(data))
// If the write creates a file hole, we have to zero-pad the last block.
// But if the write directly follows an earlier write, it cannot create a
// hole, and we can save one Stat() call.
if !f.isConsecutiveWrite(off) {
status := f.writePadHole(off)
if !status.Ok() {
return 0, status
}
}
n, status := f.doWrite(data, off)
if status.Ok() {
f.lastOpCount = openfiletable.WriteOpCount()
f.lastWrittenOffset = off + int64(len(data)) - 1
}
return n, status
}
// Release - FUSE call, close file
func (f *File) Release() {
f.fdLock.Lock()
if f.released {
log.Panicf("ino%d fh%d: double release", f.qIno.Ino, f.intFd())
}
f.fd.Close()
f.released = true
f.fdLock.Unlock()
openfiletable.Unregister(f.qIno)
}
// Flush - FUSE call
func (f *File) Flush() fuse.Status {
f.fdLock.RLock()
defer f.fdLock.RUnlock()
// Since Flush() may be called for each dup'd fd, we don't
// want to really close the file, we just want to flush. This
// is achieved by closing a dup'd fd.
newFd, err := syscall.Dup(int(f.fd.Fd()))
if err != nil {
return fuse.ToStatus(err)
}
err = syscall.Close(newFd)
return fuse.ToStatus(err)
}
2018-07-01 22:00:06 +02:00
// Fsync FUSE call
func (f *File) Fsync(flags int) (code fuse.Status) {
f.fdLock.RLock()
defer f.fdLock.RUnlock()
return fuse.ToStatus(syscall.Fsync(int(f.fd.Fd())))
}
2018-07-01 22:00:06 +02:00
// Chmod FUSE call
func (f *File) Chmod(mode uint32) fuse.Status {
f.fdLock.RLock()
defer f.fdLock.RUnlock()
// os.File.Chmod goes through the "syscallMode" translation function that messes
// up the suid and sgid bits. So use syscall.Fchmod directly.
err := syscall.Fchmod(f.intFd(), mode)
return fuse.ToStatus(err)
}
2018-07-01 22:00:06 +02:00
// Chown FUSE call
func (f *File) Chown(uid uint32, gid uint32) fuse.Status {
f.fdLock.RLock()
defer f.fdLock.RUnlock()
return fuse.ToStatus(f.fd.Chown(int(uid), int(gid)))
}
2018-07-01 22:00:06 +02:00
// GetAttr FUSE call (like stat)
func (f *File) GetAttr(a *fuse.Attr) fuse.Status {
f.fdLock.RLock()
defer f.fdLock.RUnlock()
tlog.Debug.Printf("file.GetAttr()")
st := syscall.Stat_t{}
err := syscall.Fstat(int(f.fd.Fd()), &st)
if err != nil {
return fuse.ToStatus(err)
}
a.FromStat(&st)
Major refactoring: Split up "cryptfs" into several internal packages "git status" for reference: deleted: cryptfs/cryptfs.go deleted: cryptfs/names_core.go modified: integration_tests/cli_test.go modified: integration_tests/helpers.go renamed: cryptfs/config_file.go -> internal/configfile/config_file.go renamed: cryptfs/config_test.go -> internal/configfile/config_test.go renamed: cryptfs/config_test/.gitignore -> internal/configfile/config_test/.gitignore renamed: cryptfs/config_test/PlaintextNames.conf -> internal/configfile/config_test/PlaintextNames.conf renamed: cryptfs/config_test/StrangeFeature.conf -> internal/configfile/config_test/StrangeFeature.conf renamed: cryptfs/config_test/v1.conf -> internal/configfile/config_test/v1.conf renamed: cryptfs/config_test/v2.conf -> internal/configfile/config_test/v2.conf renamed: cryptfs/kdf.go -> internal/configfile/kdf.go renamed: cryptfs/kdf_test.go -> internal/configfile/kdf_test.go renamed: cryptfs/cryptfs_content.go -> internal/contentenc/content.go new file: internal/contentenc/content_api.go renamed: cryptfs/content_test.go -> internal/contentenc/content_test.go renamed: cryptfs/file_header.go -> internal/contentenc/file_header.go renamed: cryptfs/intrablock.go -> internal/contentenc/intrablock.go renamed: cryptfs/address_translation.go -> internal/contentenc/offsets.go new file: internal/cryptocore/crypto_api.go renamed: cryptfs/gcm_go1.4.go -> internal/cryptocore/gcm_go1.4.go renamed: cryptfs/gcm_go1.5.go -> internal/cryptocore/gcm_go1.5.go renamed: cryptfs/nonce.go -> internal/cryptocore/nonce.go renamed: cryptfs/openssl_aead.go -> internal/cryptocore/openssl_aead.go renamed: cryptfs/openssl_benchmark.bash -> internal/cryptocore/openssl_benchmark.bash renamed: cryptfs/openssl_test.go -> internal/cryptocore/openssl_test.go new file: internal/nametransform/name_api.go new file: internal/nametransform/names_core.go renamed: cryptfs/names_diriv.go -> internal/nametransform/names_diriv.go renamed: cryptfs/names_noiv.go -> internal/nametransform/names_noiv.go renamed: cryptfs/names_test.go -> internal/nametransform/names_test.go new file: internal/nametransform/pad16.go renamed: cryptfs/log.go -> internal/toggledlog/log.go renamed: cryptfs/log_go1.4.go -> internal/toggledlog/log_go1.4.go renamed: cryptfs/log_go1.5.go -> internal/toggledlog/log_go1.5.go modified: main.go modified: masterkey.go modified: pathfs_frontend/file.go modified: pathfs_frontend/file_holes.go modified: pathfs_frontend/fs.go modified: pathfs_frontend/fs_dir.go modified: pathfs_frontend/names.go modified: test.bash
2016-02-06 19:20:54 +01:00
a.Size = f.contentEnc.CipherSizeToPlainSize(a.Size)
if f.fs.args.ForceOwner != nil {
a.Owner = *f.fs.args.ForceOwner
}
return fuse.OK
}
2018-07-01 22:00:06 +02:00
// Utimens FUSE call
func (f *File) Utimens(a *time.Time, m *time.Time) fuse.Status {
f.fdLock.RLock()
defer f.fdLock.RUnlock()
return f.loopbackFile.Utimens(a, m)
}