full stack: implement HKDF support

...but keep it disabled by default for new filesystems.

We are still missing an example filesystem and CLI arguments
to explicitely enable and disable it.

README.md

@@ -139,6 +139,11 @@ Changelog
 ---------
 
 v1.3 (not released yet)
+* **Use HKDF to derive separate keys for GCM and EME**
+ * New feature flag: `HKDF` (enabled by default)
+ * This is a forwards-compatible change. gocryptfs v1.3 can mount
+   filesystems created by earlier versions but not the other way round.
+* Enable Raw64 filename encoding by default (gets rid of trailing `==` characters)
 * Drop Go 1.4 compatibility. You now need Go 1.5 (released 2015-08-19)
   or higher to build gocryptfs.
 

internal/configfile/config_file.go

@@ -56,13 +56,6 @@ func CreateConfFile(filename string, password string, plaintextNames bool, logN
 	cf.Creator = creator
 	cf.Version = contentenc.CurrentVersion
 
-	// Generate new random master key
-	key := cryptocore.RandBytes(cryptocore.KeyLen)
-
-	// Encrypt it using the password
-	// This sets ScryptObject and EncryptedKey
-	cf.EncryptKey(key, password, logN)
-
 	// Set feature flags
 	cf.FeatureFlags = append(cf.FeatureFlags, knownFlags[FlagGCMIV128])
 	if plaintextNames {
@@ -72,11 +65,22 @@ func CreateConfFile(filename string, password string, plaintextNames bool, logN
 		cf.FeatureFlags = append(cf.FeatureFlags, knownFlags[FlagEMENames])
 		cf.FeatureFlags = append(cf.FeatureFlags, knownFlags[FlagLongNames])
 		cf.FeatureFlags = append(cf.FeatureFlags, knownFlags[FlagRaw64])
+		// TODO enable this and release as v1.3-beta1 once we have enough test
+		// coverage
+		//cf.FeatureFlags = append(cf.FeatureFlags, knownFlags[FlagHKDF])
 	}
 	if aessiv {
 		cf.FeatureFlags = append(cf.FeatureFlags, knownFlags[FlagAESSIV])
 	}
 
+	// Generate new random master key
+	key := cryptocore.RandBytes(cryptocore.KeyLen)
+
+	// Encrypt it using the password
+	// This sets ScryptObject and EncryptedKey
+	// Note: this looks at the FeatureFlags, so call it AFTER setting them.
+	cf.EncryptKey(key, password, logN)
+
 	// Write file to disk
 	return cf.WriteFile()
 }
@@ -148,20 +152,13 @@ func LoadConfFile(filename string, password string) ([]byte, *ConfFile, error) {
 		// decrypt the master key. Return only the parsed config.
 		return nil, &cf, nil
 	}
+
 	// Generate derived key from password
 	scryptHash := cf.ScryptObject.DeriveKey(password)
 
 	// Unlock master key using password-based key
-	// gocryptfs v1.2 and older used 96-bit IVs for master key encryption.
-	// v1.3 and up use 128 bits, which makes EncryptedKey longer (64 bytes).
-	IVLen := contentenc.DefaultIVBits
-	if len(cf.EncryptedKey) == 60 {
-		IVLen = 96
-	}
-	// We use stock Go GCM instead of OpenSSL as speed is not
-	// important and we get better error messages
-	cc := cryptocore.New(scryptHash, cryptocore.BackendGoGCM, IVLen)
-	ce := contentenc.New(cc, 4096)
+	useHKDF := cf.IsFeatureFlagSet(FlagHKDF)
+	ce := getKeyEncrypter(scryptHash, useHKDF)
 
 	tlog.Warn.Enabled = false // Silence DecryptBlock() error messages on incorrect password
 	key, err := ce.DecryptBlock(cf.EncryptedKey, 0, nil)
@@ -184,8 +181,8 @@ func (cf *ConfFile) EncryptKey(key []byte, password string, logN int) {
 	scryptHash := cf.ScryptObject.DeriveKey(password)
 
 	// Lock master key using password-based key
-	cc := cryptocore.New(scryptHash, cryptocore.BackendGoGCM, contentenc.DefaultIVBits)
-	ce := contentenc.New(cc, 4096)
+	useHKDF := cf.IsFeatureFlagSet(FlagHKDF)
+	ce := getKeyEncrypter(scryptHash, useHKDF)
 	cf.EncryptedKey = ce.EncryptBlock(key, 0, nil)
 }
 
@@ -220,3 +217,17 @@ func (cf *ConfFile) WriteFile() error {
 	err = os.Rename(tmp, cf.filename)
 	return err
 }
+
+// getKeyEncrypter is a helper function that returns the right ContentEnc
+// instance for the "useHKDF" setting.
+func getKeyEncrypter(scryptHash []byte, useHKDF bool) *contentenc.ContentEnc {
+	IVLen := 96
+	// gocryptfs v1.2 and older used 96-bit IVs for master key encryption.
+	// v1.3 adds the "HKDF" feature flag, which also enables 128-bit nonces.
+	if useHKDF {
+		IVLen = contentenc.DefaultIVBits
+	}
+	cc := cryptocore.New(scryptHash, cryptocore.BackendGoGCM, IVLen, useHKDF)
+	ce := contentenc.New(cc, 4096)
+	return ce
+}

internal/configfile/feature_flags.go

@@ -36,7 +36,7 @@ var knownFlags = map[flagIota]string{
 	FlagLongNames:      "LongNames",
 	FlagAESSIV:         "AESSIV",
 	FlagRaw64:          "Raw64",
-	//FlagHKDF:           "HKDF",
+	FlagHKDF:           "HKDF",
 }
 
 // Filesystems that do not have these feature flags set are deprecated.

internal/contentenc/content_test.go

@@ -23,7 +23,7 @@ func TestSplitRange(t *testing.T) {
 		testRange{6654, 8945})
 
 	key := make([]byte, cryptocore.KeyLen)
-	cc := cryptocore.New(key, cryptocore.BackendOpenSSL, DefaultIVBits)
+	cc := cryptocore.New(key, cryptocore.BackendOpenSSL, DefaultIVBits, true)
 	f := New(cc, DefaultBS)
 
 	for _, r := range ranges {
@@ -51,7 +51,7 @@ func TestCiphertextRange(t *testing.T) {
 		testRange{6654, 8945})
 
 	key := make([]byte, cryptocore.KeyLen)
-	cc := cryptocore.New(key, cryptocore.BackendOpenSSL, DefaultIVBits)
+	cc := cryptocore.New(key, cryptocore.BackendOpenSSL, DefaultIVBits, true)
 	f := New(cc, DefaultBS)
 
 	for _, r := range ranges {
@@ -74,7 +74,7 @@ func TestCiphertextRange(t *testing.T) {
 
 func TestBlockNo(t *testing.T) {
 	key := make([]byte, cryptocore.KeyLen)
-	cc := cryptocore.New(key, cryptocore.BackendOpenSSL, DefaultIVBits)
+	cc := cryptocore.New(key, cryptocore.BackendOpenSSL, DefaultIVBits, true)
 	f := New(cc, DefaultBS)
 
 	b := f.CipherOffToBlockNo(788)

internal/cryptocore/cryptocore.go

@@ -51,45 +51,71 @@ type CryptoCore struct {
 // Even though the "GCMIV128" feature flag is now mandatory, we must still
 // support 96-bit IVs here because they were used for encrypting the master
 // key in gocryptfs.conf up to gocryptfs v1.2. v1.3 switched to 128 bits.
-func New(key []byte, aeadType AEADTypeEnum, IVBitLen int) *CryptoCore {
+func New(key []byte, aeadType AEADTypeEnum, IVBitLen int, useHKDF bool) *CryptoCore {
 	if len(key) != KeyLen {
 		log.Panic(fmt.Sprintf("Unsupported key length %d", len(key)))
 	}
 	// We want the IV size in bytes
 	IVLen := IVBitLen / 8
 
-	// Name encryption always uses built-in Go AES through blockCipher.
-	// Content encryption uses BlockCipher only if useOpenssl=false.
-	blockCipher, err := aes.NewCipher(key)
+	// Initialize EME for filename encryption.
+	var emeCipher *eme.EMECipher
+	{
+		emeKey := key
+		if useHKDF {
+			info := "EME filename encryption"
+			emeKey = hkdfDerive(key, info, KeyLen)
+		}
+		emeBlockCipher, err := aes.NewCipher(emeKey)
 		if err != nil {
 			log.Panic(err)
 		}
-	emeCipher := eme.New(blockCipher)
+		emeCipher = eme.New(emeBlockCipher)
+	}
 
+	// Initilize an AEAD cipher for file content encryption.
 	var aeadCipher cipher.AEAD
+	if aeadType == BackendOpenSSL || aeadType == BackendGoGCM {
+		gcmKey := key
+		if useHKDF {
+			info := "AES-GCM file content encryption"
+			gcmKey = hkdfDerive(key, info, KeyLen)
+		}
 		switch aeadType {
 		case BackendOpenSSL:
 			if IVLen != 16 {
 				log.Panic("stupidgcm only supports 128-bit IVs")
 			}
-		aeadCipher = stupidgcm.New(key)
+			aeadCipher = stupidgcm.New(gcmKey)
 		case BackendGoGCM:
-		aeadCipher, err = cipher.NewGCMWithNonceSize(blockCipher, IVLen)
-	case BackendAESSIV:
+			goGcmBlockCipher, err := aes.NewCipher(gcmKey)
+			if err != nil {
+				log.Panic(err)
+			}
+			aeadCipher, err = cipher.NewGCMWithNonceSize(goGcmBlockCipher, IVLen)
+			if err != nil {
+				log.Panic(err)
+			}
+		}
+	} else if aeadType == BackendAESSIV {
 		if IVLen != 16 {
 			// SIV supports any nonce size, but we only use 16.
 			log.Panic("AES-SIV must use 16-byte nonces")
 		}
+		var key64 []byte
+		if useHKDF {
+			info := "AES-SIV file content encryption"
+			key64 = hkdfDerive(key, info, siv_aead.KeyLen)
+		} else {
 			// AES-SIV uses 1/2 of the key for authentication, 1/2 for
 			// encryption, so we need a 64-bytes key for AES-256. Derive it from
 			// the master key by hashing it with SHA-512.
-		key64 := sha512.Sum512(key)
-		aeadCipher = siv_aead.New(key64[:])
-	default:
-		log.Panic("unknown backend cipher")
+			s := sha512.Sum512(key)
+			key64 = s[:]
 		}
-	if err != nil {
-		log.Panic(err)
+		aeadCipher = siv_aead.New(key64)
+	} else {
+		log.Panic("unknown backend cipher")
 	}
 
 	return &CryptoCore{

internal/cryptocore/cryptocore_test.go

@@ -7,20 +7,21 @@ import (
 // "New" should accept at least these param combinations
 func TestCryptoCoreNew(t *testing.T) {
 	key := make([]byte, 32)
-
-	c := New(key, BackendOpenSSL, 128)
+	for _, useHKDF := range []bool{true, false} {
+		c := New(key, BackendOpenSSL, 128, useHKDF)
 		if c.IVLen != 16 {
 			t.Fail()
 		}
-	c = New(key, BackendGoGCM, 96)
+		c = New(key, BackendGoGCM, 96, useHKDF)
 		if c.IVLen != 12 {
 			t.Fail()
 		}
-	c = New(key, BackendGoGCM, 128)
+		c = New(key, BackendGoGCM, 128, useHKDF)
 		if c.IVLen != 16 {
 			t.Fail()
 		}
 	}
+}
 
 // "New" should panic on any key not 32 bytes long
 func TestNewPanic(t *testing.T) {
@@ -31,5 +32,5 @@ func TestNewPanic(t *testing.T) {
 	}()
 
 	key := make([]byte, 16)
-	New(key, BackendOpenSSL, 128)
+	New(key, BackendOpenSSL, 128, true)
 }

internal/cryptocore/hkdf.go

@@ -0,0 +1,21 @@
+package cryptocore
+
+import (
+	"crypto/sha256"
+	"log"
+
+	"golang.org/x/crypto/hkdf"
+)
+
+// hkdfDerive derives "outLen" bytes from "masterkey" and "info" using
+// HKDF-SHA256.
+// It returns the derived bytes or panics.
+func hkdfDerive(masterkey []byte, info string, outLen int) (out []byte) {
+	h := hkdf.New(sha256.New, masterkey, nil, []byte(info))
+	out = make([]byte, outLen)
+	n, err := h.Read(out)
+	if n != outLen || err != nil {
+		log.Panicf("hkdfDerive: hkdf read failed, got %d bytes, error: %v", n, err)
+	}
+	return out
+}

internal/fusefrontend/args.go

@@ -19,8 +19,12 @@ type Args struct {
 	// to "gocryptfs.conf" in the plaintext dir.
 	ConfigCustom bool
 	// Raw64 is true when RawURLEncoding (without padding) should be used for
-	// file names
+	// file names.
+	// Corresponds to the Raw64 feature flag introduced in gocryptfs v1.2.
 	Raw64 bool
 	// NoPrealloc disables automatic preallocation before writing
 	NoPrealloc bool
+	// Use HKDF key derivation.
+	// Corresponds to the HKDF feature flag introduced in gocryptfs v1.3.
+	HKDF bool
 }

internal/fusefrontend/fs.go

@@ -40,7 +40,7 @@ var _ pathfs.FileSystem = &FS{} // Verify that interface is implemented.
 
 // NewFS returns a new encrypted FUSE overlay filesystem.
 func NewFS(args Args) *FS {
-	cryptoCore := cryptocore.New(args.Masterkey, args.CryptoBackend, contentenc.DefaultIVBits)
+	cryptoCore := cryptocore.New(args.Masterkey, args.CryptoBackend, contentenc.DefaultIVBits, args.HKDF)
 	contentEnc := contentenc.New(cryptoCore, contentenc.DefaultBS)
 	nameTransform := nametransform.New(cryptoCore.EMECipher, args.LongNames, args.Raw64)
 

internal/fusefrontend_reverse/rfs.go

@@ -57,7 +57,7 @@ func NewFS(args fusefrontend.Args) *ReverseFS {
 		log.Panic("reverse mode must use AES-SIV, everything else is insecure")
 	}
 	initLongnameCache()
-	cryptoCore := cryptocore.New(args.Masterkey, args.CryptoBackend, contentenc.DefaultIVBits)
+	cryptoCore := cryptocore.New(args.Masterkey, args.CryptoBackend, contentenc.DefaultIVBits, args.HKDF)
 	contentEnc := contentenc.New(cryptoCore, contentenc.DefaultBS)
 	nameTransform := nametransform.New(cryptoCore.EMECipher, args.LongNames, args.Raw64)
 

internal/siv_aead/correctness_test.go

@@ -15,7 +15,7 @@ func TestKeyLens(t *testing.T) {
 	plaintext := []byte("foobar")
 	for _, keyLen := range keyLens {
 		key := make([]byte, keyLen)
-		a := New(key)
+		a := new2(key)
 		ciphertext2 := a.Seal(nil, nonce, plaintext, nil)
 
 		ciphertext, err := siv.Encrypt(nil, key, plaintext, [][]byte{nil, nonce})
@@ -42,7 +42,7 @@ func TestK32(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	a := New(key)
+	a := new2(key)
 	aResult := a.Seal(nonce, nonce, plaintext, aData)
 	if !bytes.Equal(sResult, aResult) {
 		t.Errorf("siv and siv_aead produce different results")

internal/siv_aead/siv_aead.go

@@ -15,8 +15,22 @@ type sivAead struct {
 
 var _ cipher.AEAD = &sivAead{}
 
+const (
+	KeyLen = 64
+)
+
 // New returns a new cipher.AEAD implementation.
 func New(key []byte) cipher.AEAD {
+	if len(key) != KeyLen {
+		// SIV supports more 32, 48 or 64-byte keys, but in gocryptfs we
+		// exclusively use 64.
+		log.Panicf("Key must be %d byte long (you passed %d)", KeyLen, len(key))
+	}
+	return new2(key)
+}
+
+// Same as "New" without the 64-byte restriction.
+func new2(key []byte) cipher.AEAD {
 	return &sivAead{
 		key: key,
 	}