Major refactoring: Split up "cryptfs" into several internal packages

"git status" for reference:

deleted:    cryptfs/cryptfs.go
deleted:    cryptfs/names_core.go
modified:   integration_tests/cli_test.go
modified:   integration_tests/helpers.go
renamed:    cryptfs/config_file.go -> internal/configfile/config_file.go
renamed:    cryptfs/config_test.go -> internal/configfile/config_test.go
renamed:    cryptfs/config_test/.gitignore -> internal/configfile/config_test/.gitignore
renamed:    cryptfs/config_test/PlaintextNames.conf -> internal/configfile/config_test/PlaintextNames.conf
renamed:    cryptfs/config_test/StrangeFeature.conf -> internal/configfile/config_test/StrangeFeature.conf
renamed:    cryptfs/config_test/v1.conf -> internal/configfile/config_test/v1.conf
renamed:    cryptfs/config_test/v2.conf -> internal/configfile/config_test/v2.conf
renamed:    cryptfs/kdf.go -> internal/configfile/kdf.go
renamed:    cryptfs/kdf_test.go -> internal/configfile/kdf_test.go
renamed:    cryptfs/cryptfs_content.go -> internal/contentenc/content.go
new file:   internal/contentenc/content_api.go
renamed:    cryptfs/content_test.go -> internal/contentenc/content_test.go
renamed:    cryptfs/file_header.go -> internal/contentenc/file_header.go
renamed:    cryptfs/intrablock.go -> internal/contentenc/intrablock.go
renamed:    cryptfs/address_translation.go -> internal/contentenc/offsets.go
new file:   internal/cryptocore/crypto_api.go
renamed:    cryptfs/gcm_go1.4.go -> internal/cryptocore/gcm_go1.4.go
renamed:    cryptfs/gcm_go1.5.go -> internal/cryptocore/gcm_go1.5.go
renamed:    cryptfs/nonce.go -> internal/cryptocore/nonce.go
renamed:    cryptfs/openssl_aead.go -> internal/cryptocore/openssl_aead.go
renamed:    cryptfs/openssl_benchmark.bash -> internal/cryptocore/openssl_benchmark.bash
renamed:    cryptfs/openssl_test.go -> internal/cryptocore/openssl_test.go
new file:   internal/nametransform/name_api.go
new file:   internal/nametransform/names_core.go
renamed:    cryptfs/names_diriv.go -> internal/nametransform/names_diriv.go
renamed:    cryptfs/names_noiv.go -> internal/nametransform/names_noiv.go
renamed:    cryptfs/names_test.go -> internal/nametransform/names_test.go
new file:   internal/nametransform/pad16.go
renamed:    cryptfs/log.go -> internal/toggledlog/log.go
renamed:    cryptfs/log_go1.4.go -> internal/toggledlog/log_go1.4.go
renamed:    cryptfs/log_go1.5.go -> internal/toggledlog/log_go1.5.go
modified:   main.go
modified:   masterkey.go
modified:   pathfs_frontend/file.go
modified:   pathfs_frontend/file_holes.go
modified:   pathfs_frontend/fs.go
modified:   pathfs_frontend/fs_dir.go
modified:   pathfs_frontend/names.go
modified:   test.bash

7 files changed, 316 insertions, 0 deletions

diff --git a/internal/cryptocore/crypto_api.go b/internal/cryptocore/crypto_api.go
new file mode 100644
index 0000000..c6b6869
--- /dev/null
+++ b/internal/cryptocore/crypto_api.go
@@ -0,0 +1,56 @@
+package cryptocore
+
+import (
+	"crypto/cipher"
+	"crypto/aes"
+	"fmt"
+)
+
+const (
+	KeyLen          = 32 // AES-256
+	AuthTagLen      = 16
+)
+
+type CryptoCore struct {
+	BlockCipher cipher.Block
+	Gcm         cipher.AEAD
+	GcmIVGen	*nonceGenerator
+	IVLen		int
+}
+
+func New(key []byte, useOpenssl bool, GCMIV128 bool) *CryptoCore {
+
+	if len(key) != KeyLen {
+		panic(fmt.Sprintf("Unsupported key length %d", len(key)))
+	}
+
+	// We want the IV size in bytes
+	IVLen := 96 / 8
+	if GCMIV128 {
+		IVLen = 128 / 8
+	}
+
+	// We always use built-in Go crypto for blockCipher because it is not
+	// performance-critical.
+	blockCipher, err := aes.NewCipher(key)
+	if err != nil {
+		panic(err)
+	}
+
+	var gcm cipher.AEAD
+	if useOpenssl {
+		gcm = opensslGCM{key}
+	} else {
+		gcm, err = goGCMWrapper(blockCipher, IVLen)
+		if err != nil {
+			panic(err)
+		}
+	}
+
+	return &CryptoCore{
+		BlockCipher: blockCipher,
+		Gcm: gcm,
+		GcmIVGen:  &nonceGenerator{nonceLen: IVLen},
+		IVLen: IVLen,
+	}
+}
diff --git a/internal/cryptocore/gcm_go1.4.go b/internal/cryptocore/gcm_go1.4.go
new file mode 100644
index 0000000..dba222c
--- /dev/null
+++ b/internal/cryptocore/gcm_go1.4.go
@@ -0,0 +1,22 @@
+// +build !go1.5
+// = go 1.4 or lower
+
+package cryptocore
+
+import (
+	"crypto/cipher"
+	"fmt"
+)
+
+// goGCMWrapper - This wrapper makes sure gocryptfs can be compiled on Go
+// versions 1.4 and lower that lack NewGCMWithNonceSize().
+// 128 bit GCM IVs will not work when using built-in Go crypto, obviously, when
+// compiled on 1.4.
+func goGCMWrapper(bc cipher.Block, nonceSize int) (cipher.AEAD, error) {
+	if nonceSize != 12 {
+		Warn.Printf("128 bit GCM IVs are not supported by Go 1.4 and lower.")
+		Warn.Printf("Please use openssl crypto or recompile using a newer Go runtime.")
+		return nil, fmt.Errorf("128 bit GCM IVs are not supported by Go 1.4 and lower")
+	}
+	return cipher.NewGCM(bc)
+}
diff --git a/internal/cryptocore/gcm_go1.5.go b/internal/cryptocore/gcm_go1.5.go
new file mode 100644
index 0000000..0c9b1a5
--- /dev/null
+++ b/internal/cryptocore/gcm_go1.5.go
@@ -0,0 +1,16 @@
+// +build go1.5
+// = go 1.5 or higher
+
+package cryptocore
+
+import (
+	"crypto/cipher"
+)
+
+// goGCMWrapper - This wrapper makes sure gocryptfs can be compiled on Go
+// versions 1.4 and lower that lack NewGCMWithNonceSize().
+// 128 bit GCM IVs will not work when using built-in Go crypto, obviously, when
+// compiled on 1.4.
+func goGCMWrapper(bc cipher.Block, nonceSize int) (cipher.AEAD, error) {
+	return cipher.NewGCMWithNonceSize(bc, nonceSize)
+}
diff --git a/internal/cryptocore/nonce.go b/internal/cryptocore/nonce.go
new file mode 100644
index 0000000..72d8588
--- /dev/null
+++ b/internal/cryptocore/nonce.go
@@ -0,0 +1,44 @@
+package cryptocore
+
+import (
+	"bytes"
+	"crypto/rand"
+	"encoding/binary"
+	"encoding/hex"
+	"fmt"
+
+	"github.com/rfjakob/gocryptfs/internal/toggledlog"
+)
+
+// Get "n" random bytes from /dev/urandom or panic
+func RandBytes(n int) []byte {
+	b := make([]byte, n)
+	_, err := rand.Read(b)
+	if err != nil {
+		panic("Failed to read random bytes: " + err.Error())
+	}
+	return b
+}
+
+// Return a secure random uint64
+func RandUint64() uint64 {
+	b := RandBytes(8)
+	return binary.BigEndian.Uint64(b)
+}
+
+type nonceGenerator struct {
+	lastNonce []byte
+	nonceLen  int // bytes
+}
+
+// Get a random "nonceLen"-byte nonce
+func (n *nonceGenerator) Get() []byte {
+	nonce := RandBytes(n.nonceLen)
+	toggledlog.Debug.Printf("nonceGenerator.Get(): %s\n", hex.EncodeToString(nonce))
+	if bytes.Equal(nonce, n.lastNonce) {
+		m := fmt.Sprintf("Got the same nonce twice: %s. This should never happen!", hex.EncodeToString(nonce))
+		panic(m)
+	}
+	n.lastNonce = nonce
+	return nonce
+}
diff --git a/internal/cryptocore/openssl_aead.go b/internal/cryptocore/openssl_aead.go
new file mode 100644
index 0000000..d4ed64b
--- /dev/null
+++ b/internal/cryptocore/openssl_aead.go
@@ -0,0 +1,100 @@
+package cryptocore
+
+// Implements cipher.AEAD with OpenSSL backend
+
+import (
+	"bytes"
+	"github.com/spacemonkeygo/openssl"
+)
+
+// Supports all nonce sizes
+type opensslGCM struct {
+	key []byte
+}
+
+func (be opensslGCM) Overhead() int {
+	return AuthTagLen
+}
+
+func (be opensslGCM) NonceSize() int {
+	// We support any nonce size
+	return -1
+}
+
+// Seal encrypts and authenticates plaintext, authenticates the
+// additional data and appends the result to dst, returning the updated
+// slice. opensslGCM supports any nonce size.
+func (be opensslGCM) Seal(dst, nonce, plaintext, data []byte) []byte {
+
+	// Preallocate output buffer
+	var cipherBuf bytes.Buffer
+	cipherBuf.Grow(len(dst) + len(plaintext) + AuthTagLen)
+	// Output will be appended to dst
+	cipherBuf.Write(dst)
+
+	ectx, err := openssl.NewGCMEncryptionCipherCtx(KeyLen*8, nil, be.key, nonce)
+	if err != nil {
+		panic(err)
+	}
+	err = ectx.ExtraData(data)
+	if err != nil {
+		panic(err)
+	}
+	part, err := ectx.EncryptUpdate(plaintext)
+	if err != nil {
+		panic(err)
+	}
+	cipherBuf.Write(part)
+	part, err = ectx.EncryptFinal()
+	if err != nil {
+		panic(err)
+	}
+	cipherBuf.Write(part)
+	part, err = ectx.GetTag()
+	if err != nil {
+		panic(err)
+	}
+	cipherBuf.Write(part)
+
+	return cipherBuf.Bytes()
+}
+
+// Open decrypts and authenticates ciphertext, authenticates the
+// additional data and, if successful, appends the resulting plaintext
+// to dst, returning the updated slice. The nonce must be NonceSize()
+// bytes long and both it and the additional data must match the
+// value passed to Seal.
+//
+// The ciphertext and dst may alias exactly or not at all.
+func (be opensslGCM) Open(dst, nonce, ciphertext, data []byte) ([]byte, error) {
+
+	l := len(ciphertext)
+	tag := ciphertext[l-AuthTagLen : l]
+	ciphertext = ciphertext[0 : l-AuthTagLen]
+	plainBuf := bytes.NewBuffer(dst)
+
+	dctx, err := openssl.NewGCMDecryptionCipherCtx(KeyLen*8, nil, be.key, nonce)
+	if err != nil {
+		return nil, err
+	}
+	err = dctx.ExtraData(data)
+	if err != nil {
+		return nil, err
+	}
+	part, err := dctx.DecryptUpdate(ciphertext)
+	if err != nil {
+		return nil, err
+	}
+	plainBuf.Write(part)
+	err = dctx.SetTag(tag)
+	if err != nil {
+		return nil, err
+	}
+	part, err = dctx.DecryptFinal()
+	if err != nil {
+		return nil, err
+	}
+	plainBuf.Write(part)
+
+	return plainBuf.Bytes(), nil
+}
diff --git a/internal/cryptocore/openssl_benchmark.bash b/internal/cryptocore/openssl_benchmark.bash
new file mode 100755
index 0000000..df29628
--- /dev/null
+++ b/internal/cryptocore/openssl_benchmark.bash
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+go test -run NONE -bench BenchmarkEnc
diff --git a/internal/cryptocore/openssl_test.go b/internal/cryptocore/openssl_test.go
new file mode 100644
index 0000000..94b696a
--- /dev/null
+++ b/internal/cryptocore/openssl_test.go
@@ -0,0 +1,75 @@
+package cryptocore
+
+// Benchmark go built-int GCM against spacemonkey openssl bindings
+//
+// Note: The benchmarks in this file supersede the ones in the openssl_benchmark
+//       directory as they use the same code paths that gocryptfs actually uses.
+//
+// Run benchmark:
+// go test -bench Enc
+
+import (
+	"crypto/aes"
+	"testing"
+)
+
+func benchmarkGoEnc(b *testing.B, plaintext []byte, key []byte, nonce []byte) (ciphertext []byte) {
+	b.SetBytes(int64(len(plaintext)))
+	aes, err := aes.NewCipher(key[:])
+	if err != nil {
+		b.Fatal(err)
+	}
+	aesgcm, err := goGCMWrapper(aes, len(nonce))
+	if err != nil {
+		b.Fatal(err)
+	}
+	// This would be fileID + blockNo
+	aData := make([]byte, 24)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		// Encrypt plaintext and append to nonce
+		ciphertext = aesgcm.Seal(nonce, nonce, plaintext, aData)
+	}
+	return ciphertext
+}
+
+func benchmarkOpensslEnc(b *testing.B, plaintext []byte, key []byte, nonce []byte) (ciphertext []byte) {
+	b.SetBytes(int64(len(plaintext)))
+	var aesgcm opensslGCM
+	aesgcm.key = key
+	// This would be fileID + blockNo
+	aData := make([]byte, 24)
+	for i := 0; i < b.N; i++ {
+		// Encrypt plaintext and append to nonce
+		ciphertext = aesgcm.Seal(nonce, nonce, plaintext, aData)
+	}
+	return ciphertext
+}
+
+func BenchmarkEnc_Go_4k_AES256_nonce96(b *testing.B) {
+	plaintext := make([]byte, 4048)
+	key := make([]byte, 256/8)
+	nonce := make([]byte, 96/8)
+	benchmarkGoEnc(b, plaintext, key, nonce)
+}
+
+func BenchmarkEnc_Go_4k_AES256_nonce128(b *testing.B) {
+	plaintext := make([]byte, 4048)
+	key := make([]byte, 256/8)
+	nonce := make([]byte, 128/8)
+	benchmarkGoEnc(b, plaintext, key, nonce)
+}
+
+func BenchmarkEnc_OpenSSL_4k_AES256_nonce96(b *testing.B) {
+	plaintext := make([]byte, 4048)
+	key := make([]byte, 256/8)
+	nonce := make([]byte, 96/8)
+	benchmarkOpensslEnc(b, plaintext, key, nonce)
+}
+
+func BenchmarkEnc_OpenSSL_4k_AES256_nonce128(b *testing.B) {
+	plaintext := make([]byte, 4048)
+	key := make([]byte, 256/8)
+	nonce := make([]byte, 96/8)
+	benchmarkOpensslEnc(b, plaintext, key, nonce)
+}