full stack: implement HKDF support

...but keep it disabled by default for new filesystems.

We are still missing an example filesystem and CLI arguments
to explicitely enable and disable it.

12 files changed, 147 insertions, 65 deletions

diff --git a/README.md b/README.md
index 242dc89..a12c08e 100644
--- a/README.md
+++ b/README.md
@@ -139,6 +139,11 @@ Changelog
 ---------
 
 v1.3 (not released yet)
+* **Use HKDF to derive separate keys for GCM and EME**
+ * New feature flag: `HKDF` (enabled by default)
+ * This is a forwards-compatible change. gocryptfs v1.3 can mount
+   filesystems created by earlier versions but not the other way round.
+* Enable Raw64 filename encoding by default (gets rid of trailing `==` characters)
 * Drop Go 1.4 compatibility. You now need Go 1.5 (released 2015-08-19)
   or higher to build gocryptfs.
 
diff --git a/internal/configfile/config_file.go b/internal/configfile/config_file.go
index 5bb021c..7565c5e 100644
--- a/internal/configfile/config_file.go
+++ b/internal/configfile/config_file.go
@@ -56,13 +56,6 @@ func CreateConfFile(filename string, password string, plaintextNames bool, logN
 	cf.Creator = creator
 	cf.Version = contentenc.CurrentVersion
 
-	// Generate new random master key
-	key := cryptocore.RandBytes(cryptocore.KeyLen)
-
-	// Encrypt it using the password
-	// This sets ScryptObject and EncryptedKey
-	cf.EncryptKey(key, password, logN)
-
 	// Set feature flags
 	cf.FeatureFlags = append(cf.FeatureFlags, knownFlags[FlagGCMIV128])
 	if plaintextNames {
@@ -72,11 +65,22 @@ func CreateConfFile(filename string, password string, plaintextNames bool, logN
 		cf.FeatureFlags = append(cf.FeatureFlags, knownFlags[FlagEMENames])
 		cf.FeatureFlags = append(cf.FeatureFlags, knownFlags[FlagLongNames])
 		cf.FeatureFlags = append(cf.FeatureFlags, knownFlags[FlagRaw64])
+		// TODO enable this and release as v1.3-beta1 once we have enough test
+		// coverage
+		//cf.FeatureFlags = append(cf.FeatureFlags, knownFlags[FlagHKDF])
 	}
 	if aessiv {
 		cf.FeatureFlags = append(cf.FeatureFlags, knownFlags[FlagAESSIV])
 	}
 
+	// Generate new random master key
+	key := cryptocore.RandBytes(cryptocore.KeyLen)
+
+	// Encrypt it using the password
+	// This sets ScryptObject and EncryptedKey
+	// Note: this looks at the FeatureFlags, so call it AFTER setting them.
+	cf.EncryptKey(key, password, logN)
+
 	// Write file to disk
 	return cf.WriteFile()
 }
@@ -148,20 +152,13 @@ func LoadConfFile(filename string, password string) ([]byte, *ConfFile, error) {
 		// decrypt the master key. Return only the parsed config.
 		return nil, &cf, nil
 	}
+
 	// Generate derived key from password
 	scryptHash := cf.ScryptObject.DeriveKey(password)
 
 	// Unlock master key using password-based key
-	// gocryptfs v1.2 and older used 96-bit IVs for master key encryption.
-	// v1.3 and up use 128 bits, which makes EncryptedKey longer (64 bytes).
-	IVLen := contentenc.DefaultIVBits
-	if len(cf.EncryptedKey) == 60 {
-		IVLen = 96
-	}
-	// We use stock Go GCM instead of OpenSSL as speed is not
-	// important and we get better error messages
-	cc := cryptocore.New(scryptHash, cryptocore.BackendGoGCM, IVLen)
-	ce := contentenc.New(cc, 4096)
+	useHKDF := cf.IsFeatureFlagSet(FlagHKDF)
+	ce := getKeyEncrypter(scryptHash, useHKDF)
 
 	tlog.Warn.Enabled = false // Silence DecryptBlock() error messages on incorrect password
 	key, err := ce.DecryptBlock(cf.EncryptedKey, 0, nil)
@@ -184,8 +181,8 @@ func (cf *ConfFile) EncryptKey(key []byte, password string, logN int) {
 	scryptHash := cf.ScryptObject.DeriveKey(password)
 
 	// Lock master key using password-based key
-	cc := cryptocore.New(scryptHash, cryptocore.BackendGoGCM, contentenc.DefaultIVBits)
-	ce := contentenc.New(cc, 4096)
+	useHKDF := cf.IsFeatureFlagSet(FlagHKDF)
+	ce := getKeyEncrypter(scryptHash, useHKDF)
 	cf.EncryptedKey = ce.EncryptBlock(key, 0, nil)
 }
 
@@ -220,3 +217,17 @@ func (cf *ConfFile) WriteFile() error {
 	err = os.Rename(tmp, cf.filename)
 	return err
 }
+
+// getKeyEncrypter is a helper function that returns the right ContentEnc
+// instance for the "useHKDF" setting.
+func getKeyEncrypter(scryptHash []byte, useHKDF bool) *contentenc.ContentEnc {
+	IVLen := 96
+	// gocryptfs v1.2 and older used 96-bit IVs for master key encryption.
+	// v1.3 adds the "HKDF" feature flag, which also enables 128-bit nonces.
+	if useHKDF {
+		IVLen = contentenc.DefaultIVBits
+	}
+	cc := cryptocore.New(scryptHash, cryptocore.BackendGoGCM, IVLen, useHKDF)
+	ce := contentenc.New(cc, 4096)
+	return ce
+}
diff --git a/internal/configfile/feature_flags.go b/internal/configfile/feature_flags.go
index deb3a0e..2d609f2 100644
--- a/internal/configfile/feature_flags.go
+++ b/internal/configfile/feature_flags.go
@@ -36,7 +36,7 @@ var knownFlags = map[flagIota]string{
 	FlagLongNames:      "LongNames",
 	FlagAESSIV:         "AESSIV",
 	FlagRaw64:          "Raw64",
-	//FlagHKDF:           "HKDF",
+	FlagHKDF:           "HKDF",
 }
 
 // Filesystems that do not have these feature flags set are deprecated.
diff --git a/internal/contentenc/content_test.go b/internal/contentenc/content_test.go
index e6c610c..8ce496d 100644
--- a/internal/contentenc/content_test.go
+++ b/internal/contentenc/content_test.go
@@ -23,7 +23,7 @@ func TestSplitRange(t *testing.T) {
 		testRange{6654, 8945})
 
 	key := make([]byte, cryptocore.KeyLen)
-	cc := cryptocore.New(key, cryptocore.BackendOpenSSL, DefaultIVBits)
+	cc := cryptocore.New(key, cryptocore.BackendOpenSSL, DefaultIVBits, true)
 	f := New(cc, DefaultBS)
 
 	for _, r := range ranges {
@@ -51,7 +51,7 @@ func TestCiphertextRange(t *testing.T) {
 		testRange{6654, 8945})
 
 	key := make([]byte, cryptocore.KeyLen)
-	cc := cryptocore.New(key, cryptocore.BackendOpenSSL, DefaultIVBits)
+	cc := cryptocore.New(key, cryptocore.BackendOpenSSL, DefaultIVBits, true)
 	f := New(cc, DefaultBS)
 
 	for _, r := range ranges {
@@ -74,7 +74,7 @@ func TestCiphertextRange(t *testing.T) {
 
 func TestBlockNo(t *testing.T) {
 	key := make([]byte, cryptocore.KeyLen)
-	cc := cryptocore.New(key, cryptocore.BackendOpenSSL, DefaultIVBits)
+	cc := cryptocore.New(key, cryptocore.BackendOpenSSL, DefaultIVBits, true)
 	f := New(cc, DefaultBS)
 
 	b := f.CipherOffToBlockNo(788)
diff --git a/internal/cryptocore/cryptocore.go b/internal/cryptocore/cryptocore.go
index 7e1d238..5244104 100644
--- a/internal/cryptocore/cryptocore.go
+++ b/internal/cryptocore/cryptocore.go
@@ -51,46 +51,72 @@ type CryptoCore struct {
 // Even though the "GCMIV128" feature flag is now mandatory, we must still
 // support 96-bit IVs here because they were used for encrypting the master
 // key in gocryptfs.conf up to gocryptfs v1.2. v1.3 switched to 128 bits.
-func New(key []byte, aeadType AEADTypeEnum, IVBitLen int) *CryptoCore {
+func New(key []byte, aeadType AEADTypeEnum, IVBitLen int, useHKDF bool) *CryptoCore {
 	if len(key) != KeyLen {
 		log.Panic(fmt.Sprintf("Unsupported key length %d", len(key)))
 	}
 	// We want the IV size in bytes
 	IVLen := IVBitLen / 8
 
-	// Name encryption always uses built-in Go AES through blockCipher.
-	// Content encryption uses BlockCipher only if useOpenssl=false.
-	blockCipher, err := aes.NewCipher(key)
-	if err != nil {
-		log.Panic(err)
+	// Initialize EME for filename encryption.
+	var emeCipher *eme.EMECipher
+	{
+		emeKey := key
+		if useHKDF {
+			info := "EME filename encryption"
+			emeKey = hkdfDerive(key, info, KeyLen)
+		}
+		emeBlockCipher, err := aes.NewCipher(emeKey)
+		if err != nil {
+			log.Panic(err)
+		}
+		emeCipher = eme.New(emeBlockCipher)
 	}
-	emeCipher := eme.New(blockCipher)
 
+	// Initilize an AEAD cipher for file content encryption.
 	var aeadCipher cipher.AEAD
-	switch aeadType {
-	case BackendOpenSSL:
-		if IVLen != 16 {
-			log.Panic("stupidgcm only supports 128-bit IVs")
+	if aeadType == BackendOpenSSL || aeadType == BackendGoGCM {
+		gcmKey := key
+		if useHKDF {
+			info := "AES-GCM file content encryption"
+			gcmKey = hkdfDerive(key, info, KeyLen)
 		}
-		aeadCipher = stupidgcm.New(key)
-	case BackendGoGCM:
-		aeadCipher, err = cipher.NewGCMWithNonceSize(blockCipher, IVLen)
-	case BackendAESSIV:
+		switch aeadType {
+		case BackendOpenSSL:
+			if IVLen != 16 {
+				log.Panic("stupidgcm only supports 128-bit IVs")
+			}
+			aeadCipher = stupidgcm.New(gcmKey)
+		case BackendGoGCM:
+			goGcmBlockCipher, err := aes.NewCipher(gcmKey)
+			if err != nil {
+				log.Panic(err)
+			}
+			aeadCipher, err = cipher.NewGCMWithNonceSize(goGcmBlockCipher, IVLen)
+			if err != nil {
+				log.Panic(err)
+			}
+		}
+	} else if aeadType == BackendAESSIV {
 		if IVLen != 16 {
 			// SIV supports any nonce size, but we only use 16.
 			log.Panic("AES-SIV must use 16-byte nonces")
 		}
-		// AES-SIV uses 1/2 of the key for authentication, 1/2 for
-		// encryption, so we need a 64-bytes key for AES-256. Derive it from
-		// the master key by hashing it with SHA-512.
-		key64 := sha512.Sum512(key)
-		aeadCipher = siv_aead.New(key64[:])
-	default:
+		var key64 []byte
+		if useHKDF {
+			info := "AES-SIV file content encryption"
+			key64 = hkdfDerive(key, info, siv_aead.KeyLen)
+		} else {
+			// AES-SIV uses 1/2 of the key for authentication, 1/2 for
+			// encryption, so we need a 64-bytes key for AES-256. Derive it from
+			// the master key by hashing it with SHA-512.
+			s := sha512.Sum512(key)
+			key64 = s[:]
+		}
+		aeadCipher = siv_aead.New(key64)
+	} else {
 		log.Panic("unknown backend cipher")
 	}
-	if err != nil {
-		log.Panic(err)
-	}
 
 	return &CryptoCore{
 		EMECipher:   emeCipher,
diff --git a/internal/cryptocore/cryptocore_test.go b/internal/cryptocore/cryptocore_test.go
index 252c311..25f6572 100644
--- a/internal/cryptocore/cryptocore_test.go
+++ b/internal/cryptocore/cryptocore_test.go
@@ -7,18 +7,19 @@ import (
 // "New" should accept at least these param combinations
 func TestCryptoCoreNew(t *testing.T) {
 	key := make([]byte, 32)
-
-	c := New(key, BackendOpenSSL, 128)
-	if c.IVLen != 16 {
-		t.Fail()
-	}
-	c = New(key, BackendGoGCM, 96)
-	if c.IVLen != 12 {
-		t.Fail()
-	}
-	c = New(key, BackendGoGCM, 128)
-	if c.IVLen != 16 {
-		t.Fail()
+	for _, useHKDF := range []bool{true, false} {
+		c := New(key, BackendOpenSSL, 128, useHKDF)
+		if c.IVLen != 16 {
+			t.Fail()
+		}
+		c = New(key, BackendGoGCM, 96, useHKDF)
+		if c.IVLen != 12 {
+			t.Fail()
+		}
+		c = New(key, BackendGoGCM, 128, useHKDF)
+		if c.IVLen != 16 {
+			t.Fail()
+		}
 	}
 }
 
@@ -31,5 +32,5 @@ func TestNewPanic(t *testing.T) {
 	}()
 
 	key := make([]byte, 16)
-	New(key, BackendOpenSSL, 128)
+	New(key, BackendOpenSSL, 128, true)
 }
diff --git a/internal/cryptocore/hkdf.go b/internal/cryptocore/hkdf.go
new file mode 100644
index 0000000..6944825
--- /dev/null
+++ b/internal/cryptocore/hkdf.go
@@ -0,0 +1,21 @@
+package cryptocore
+
+import (
+	"crypto/sha256"
+	"log"
+
+	"golang.org/x/crypto/hkdf"
+)
+
+// hkdfDerive derives "outLen" bytes from "masterkey" and "info" using
+// HKDF-SHA256.
+// It returns the derived bytes or panics.
+func hkdfDerive(masterkey []byte, info string, outLen int) (out []byte) {
+	h := hkdf.New(sha256.New, masterkey, nil, []byte(info))
+	out = make([]byte, outLen)
+	n, err := h.Read(out)
+	if n != outLen || err != nil {
+		log.Panicf("hkdfDerive: hkdf read failed, got %d bytes, error: %v", n, err)
+	}
+	return out
+}
diff --git a/internal/fusefrontend/args.go b/internal/fusefrontend/args.go
index c111dbf..f76848d 100644
--- a/internal/fusefrontend/args.go
+++ b/internal/fusefrontend/args.go
@@ -19,8 +19,12 @@ type Args struct {
 	// to "gocryptfs.conf" in the plaintext dir.
 	ConfigCustom bool
 	// Raw64 is true when RawURLEncoding (without padding) should be used for
-	// file names
+	// file names.
+	// Corresponds to the Raw64 feature flag introduced in gocryptfs v1.2.
 	Raw64 bool
 	// NoPrealloc disables automatic preallocation before writing
 	NoPrealloc bool
+	// Use HKDF key derivation.
+	// Corresponds to the HKDF feature flag introduced in gocryptfs v1.3.
+	HKDF bool
 }
diff --git a/internal/fusefrontend/fs.go b/internal/fusefrontend/fs.go
index e0fdc48..020032b 100644
--- a/internal/fusefrontend/fs.go
+++ b/internal/fusefrontend/fs.go
@@ -40,7 +40,7 @@ var _ pathfs.FileSystem = &FS{} // Verify that interface is implemented.
 
 // NewFS returns a new encrypted FUSE overlay filesystem.
 func NewFS(args Args) *FS {
-	cryptoCore := cryptocore.New(args.Masterkey, args.CryptoBackend, contentenc.DefaultIVBits)
+	cryptoCore := cryptocore.New(args.Masterkey, args.CryptoBackend, contentenc.DefaultIVBits, args.HKDF)
 	contentEnc := contentenc.New(cryptoCore, contentenc.DefaultBS)
 	nameTransform := nametransform.New(cryptoCore.EMECipher, args.LongNames, args.Raw64)
 
diff --git a/internal/fusefrontend_reverse/rfs.go b/internal/fusefrontend_reverse/rfs.go
index 55431b6..1bcbe45 100644
--- a/internal/fusefrontend_reverse/rfs.go
+++ b/internal/fusefrontend_reverse/rfs.go
@@ -57,7 +57,7 @@ func NewFS(args fusefrontend.Args) *ReverseFS {
 		log.Panic("reverse mode must use AES-SIV, everything else is insecure")
 	}
 	initLongnameCache()
-	cryptoCore := cryptocore.New(args.Masterkey, args.CryptoBackend, contentenc.DefaultIVBits)
+	cryptoCore := cryptocore.New(args.Masterkey, args.CryptoBackend, contentenc.DefaultIVBits, args.HKDF)
 	contentEnc := contentenc.New(cryptoCore, contentenc.DefaultBS)
 	nameTransform := nametransform.New(cryptoCore.EMECipher, args.LongNames, args.Raw64)
 
diff --git a/internal/siv_aead/correctness_test.go b/internal/siv_aead/correctness_test.go
index a9885e7..b52774b 100644
--- a/internal/siv_aead/correctness_test.go
+++ b/internal/siv_aead/correctness_test.go
@@ -15,7 +15,7 @@ func TestKeyLens(t *testing.T) {
 	plaintext := []byte("foobar")
 	for _, keyLen := range keyLens {
 		key := make([]byte, keyLen)
-		a := New(key)
+		a := new2(key)
 		ciphertext2 := a.Seal(nil, nonce, plaintext, nil)
 
 		ciphertext, err := siv.Encrypt(nil, key, plaintext, [][]byte{nil, nonce})
@@ -42,7 +42,7 @@ func TestK32(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	a := New(key)
+	a := new2(key)
 	aResult := a.Seal(nonce, nonce, plaintext, aData)
 	if !bytes.Equal(sResult, aResult) {
 		t.Errorf("siv and siv_aead produce different results")
diff --git a/internal/siv_aead/siv_aead.go b/internal/siv_aead/siv_aead.go
index 6cfa937..d5df4ac 100644
--- a/internal/siv_aead/siv_aead.go
+++ b/internal/siv_aead/siv_aead.go
@@ -15,8 +15,22 @@ type sivAead struct {
 
 var _ cipher.AEAD = &sivAead{}
 
+const (
+	KeyLen = 64
+)
+
 // New returns a new cipher.AEAD implementation.
 func New(key []byte) cipher.AEAD {
+	if len(key) != KeyLen {
+		// SIV supports more 32, 48 or 64-byte keys, but in gocryptfs we
+		// exclusively use 64.
+		log.Panicf("Key must be %d byte long (you passed %d)", KeyLen, len(key))
+	}
+	return new2(key)
+}
+
+// Same as "New" without the 64-byte restriction.
+func new2(key []byte) cipher.AEAD {
 	return &sivAead{
 		key: key,
 	}