Major refactoring: Split up "cryptfs" into several internal packages

"git status" for reference:

deleted:    cryptfs/cryptfs.go
deleted:    cryptfs/names_core.go
modified:   integration_tests/cli_test.go
modified:   integration_tests/helpers.go
renamed:    cryptfs/config_file.go -> internal/configfile/config_file.go
renamed:    cryptfs/config_test.go -> internal/configfile/config_test.go
renamed:    cryptfs/config_test/.gitignore -> internal/configfile/config_test/.gitignore
renamed:    cryptfs/config_test/PlaintextNames.conf -> internal/configfile/config_test/PlaintextNames.conf
renamed:    cryptfs/config_test/StrangeFeature.conf -> internal/configfile/config_test/StrangeFeature.conf
renamed:    cryptfs/config_test/v1.conf -> internal/configfile/config_test/v1.conf
renamed:    cryptfs/config_test/v2.conf -> internal/configfile/config_test/v2.conf
renamed:    cryptfs/kdf.go -> internal/configfile/kdf.go
renamed:    cryptfs/kdf_test.go -> internal/configfile/kdf_test.go
renamed:    cryptfs/cryptfs_content.go -> internal/contentenc/content.go
new file:   internal/contentenc/content_api.go
renamed:    cryptfs/content_test.go -> internal/contentenc/content_test.go
renamed:    cryptfs/file_header.go -> internal/contentenc/file_header.go
renamed:    cryptfs/intrablock.go -> internal/contentenc/intrablock.go
renamed:    cryptfs/address_translation.go -> internal/contentenc/offsets.go
new file:   internal/cryptocore/crypto_api.go
renamed:    cryptfs/gcm_go1.4.go -> internal/cryptocore/gcm_go1.4.go
renamed:    cryptfs/gcm_go1.5.go -> internal/cryptocore/gcm_go1.5.go
renamed:    cryptfs/nonce.go -> internal/cryptocore/nonce.go
renamed:    cryptfs/openssl_aead.go -> internal/cryptocore/openssl_aead.go
renamed:    cryptfs/openssl_benchmark.bash -> internal/cryptocore/openssl_benchmark.bash
renamed:    cryptfs/openssl_test.go -> internal/cryptocore/openssl_test.go
new file:   internal/nametransform/name_api.go
new file:   internal/nametransform/names_core.go
renamed:    cryptfs/names_diriv.go -> internal/nametransform/names_diriv.go
renamed:    cryptfs/names_noiv.go -> internal/nametransform/names_noiv.go
renamed:    cryptfs/names_test.go -> internal/nametransform/names_test.go
new file:   internal/nametransform/pad16.go
renamed:    cryptfs/log.go -> internal/toggledlog/log.go
renamed:    cryptfs/log_go1.4.go -> internal/toggledlog/log_go1.4.go
renamed:    cryptfs/log_go1.5.go -> internal/toggledlog/log_go1.5.go
modified:   main.go
modified:   masterkey.go
modified:   pathfs_frontend/file.go
modified:   pathfs_frontend/file_holes.go
modified:   pathfs_frontend/fs.go
modified:   pathfs_frontend/fs_dir.go
modified:   pathfs_frontend/names.go
modified:   test.bash

6 files changed, 411 insertions, 0 deletions

diff --git a/internal/nametransform/name_api.go b/internal/nametransform/name_api.go
new file mode 100644
index 0000000..462e99c
--- /dev/null
+++ b/internal/nametransform/name_api.go
@@ -0,0 +1,16 @@
+package nametransform
+
+import "github.com/rfjakob/gocryptfs/internal/cryptocore"
+
+type NameTransform struct {
+	cryptoCore *cryptocore.CryptoCore
+	useEME bool
+	DirIVCache dirIVCache
+}
+
+func New(c *cryptocore.CryptoCore, useEME bool) *NameTransform {
+	return &NameTransform{
+		cryptoCore: c,
+		useEME: useEME,
+	}
+}
diff --git a/internal/nametransform/names_core.go b/internal/nametransform/names_core.go
new file mode 100644
index 0000000..452ab45
--- /dev/null
+++ b/internal/nametransform/names_core.go
@@ -0,0 +1,63 @@
+package nametransform
+
+// Filename encryption / decryption functions
+
+import (
+	"crypto/aes"
+	"crypto/cipher"
+	"encoding/base64"
+	"fmt"
+
+	"github.com/rfjakob/eme"
+)
+
+// DecryptName - decrypt base64-encoded encrypted filename "cipherName"
+// The used encryption is either CBC or EME, depending on "useEME".
+//
+// This function is exported because it allows for a very efficient readdir
+// implementation (read IV once, decrypt all names using this function).
+func (n *NameTransform) DecryptName(cipherName string, iv []byte) (string, error) {
+
+	bin, err := base64.URLEncoding.DecodeString(cipherName)
+	if err != nil {
+		return "", err
+	}
+
+	if len(bin)%aes.BlockSize != 0 {
+		return "", fmt.Errorf("Decoded length %d is not a multiple of the AES block size", len(bin))
+	}
+
+	if n.useEME {
+		bin = eme.Transform(n.cryptoCore.BlockCipher, iv, bin, eme.DirectionDecrypt)
+	} else {
+		cbc := cipher.NewCBCDecrypter(n.cryptoCore.BlockCipher, iv)
+		cbc.CryptBlocks(bin, bin)
+	}
+
+	bin, err = unPad16(bin)
+	if err != nil {
+		return "", err
+	}
+
+	plain := string(bin)
+	return plain, err
+}
+
+// encryptName - encrypt "plainName", return base64-encoded "cipherName64"
+// The used encryption is either CBC or EME, depending on "useEME".
+func (n *NameTransform) encryptName(plainName string, iv []byte) (cipherName64 string) {
+
+	bin := []byte(plainName)
+	bin = pad16(bin)
+
+	if n.useEME {
+		bin = eme.Transform(n.cryptoCore.BlockCipher, iv, bin, eme.DirectionEncrypt)
+	} else {
+		cbc := cipher.NewCBCEncrypter(n.cryptoCore.BlockCipher, iv)
+		cbc.CryptBlocks(bin, bin)
+	}
+
+	cipherName64 = base64.URLEncoding.EncodeToString(bin)
+	return cipherName64
+}
+
diff --git a/internal/nametransform/names_diriv.go b/internal/nametransform/names_diriv.go
new file mode 100644
index 0000000..d31a066
--- /dev/null
+++ b/internal/nametransform/names_diriv.go
@@ -0,0 +1,151 @@
+package nametransform
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	"github.com/rfjakob/gocryptfs/internal/toggledlog"
+	"github.com/rfjakob/gocryptfs/internal/cryptocore"
+)
+
+const (
+	// identical to AES block size
+	dirIVLen       = 16
+	// dirIV is stored in this file. Exported because we have to ignore this
+	// name in directory listing.
+	DirIVFilename  = "gocryptfs.diriv"
+)
+
+// A simple one-entry DirIV cache
+type dirIVCache struct {
+	// Invalidated?
+	cleared bool
+	// The DirIV
+	iv []byte
+	// Directory the DirIV belongs to
+	dir string
+	// Ecrypted version of "dir"
+	translatedDir string
+	// Synchronisation
+	lock sync.RWMutex
+}
+
+// lookup - fetch entry for "dir" from the cache
+func (c *dirIVCache) lookup(dir string) (bool, []byte, string) {
+	c.lock.RLock()
+	defer c.lock.RUnlock()
+	if !c.cleared && c.dir == dir {
+		return true, c.iv, c.translatedDir
+	}
+	return false, nil, ""
+}
+
+// store - write entry for "dir" into the caches
+func (c *dirIVCache) store(dir string, iv []byte, translatedDir string) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	c.cleared = false
+	c.iv = iv
+	c.dir = dir
+	c.translatedDir = translatedDir
+}
+
+func (c *dirIVCache) Clear() {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	c.cleared = true
+}
+
+// readDirIV - read the "gocryptfs.diriv" file from "dir" (absolute ciphertext path)
+func (be *NameTransform) ReadDirIV(dir string) (iv []byte, readErr error) {
+	ivfile := filepath.Join(dir, DirIVFilename)
+	toggledlog.Debug.Printf("ReadDirIV: reading %s\n", ivfile)
+	iv, readErr = ioutil.ReadFile(ivfile)
+	if readErr != nil {
+		// The directory may have been concurrently deleted or moved. Failure to
+		// read the diriv is not an error in that case.
+		_, statErr := os.Stat(dir)
+		if os.IsNotExist(statErr) {
+			toggledlog.Debug.Printf("ReadDirIV: Dir %s was deleted under our feet", dir)
+		} else {
+			// This should not happen
+			toggledlog.Warn.Printf("ReadDirIV: Dir exists but diriv does not: %v\n", readErr)
+		}
+		return nil, readErr
+	}
+	if len(iv) != dirIVLen {
+		return nil, fmt.Errorf("ReadDirIV: Invalid length %d\n", len(iv))
+	}
+	return iv, nil
+}
+
+// WriteDirIV - create diriv file inside "dir" (absolute ciphertext path)
+// This function is exported because it is used from pathfs_frontend, main,
+// and also the automated tests.
+func WriteDirIV(dir string) error {
+	iv := cryptocore.RandBytes(dirIVLen)
+	file := filepath.Join(dir, DirIVFilename)
+	// 0444 permissions: the file is not secret but should not be written to
+	return ioutil.WriteFile(file, iv, 0444)
+}
+
+// EncryptPathDirIV - encrypt path using EME with DirIV
+func (be *NameTransform) EncryptPathDirIV(plainPath string, rootDir string) (cipherPath string, err error) {
+	// Empty string means root directory
+	if plainPath == "" {
+		return plainPath, nil
+	}
+	// Check if the DirIV is cached
+	parentDir := filepath.Dir(plainPath)
+	found, iv, cParentDir := be.DirIVCache.lookup(parentDir)
+	if found {
+		//fmt.Print("h")
+		baseName := filepath.Base(plainPath)
+		cBaseName := be.encryptName(baseName, iv)
+		cipherPath = cParentDir + "/" + cBaseName
+		return cipherPath, nil
+	}
+	// Walk the directory tree
+	var wd = rootDir
+	var encryptedNames []string
+	plainNames := strings.Split(plainPath, "/")
+	for _, plainName := range plainNames {
+		iv, err = be.ReadDirIV(wd)
+		if err != nil {
+			return "", err
+		}
+		encryptedName := be.encryptName(plainName, iv)
+		encryptedNames = append(encryptedNames, encryptedName)
+		wd = filepath.Join(wd, encryptedName)
+	}
+	// Cache the final DirIV
+	cipherPath = strings.Join(encryptedNames, "/")
+	cParentDir = filepath.Dir(cipherPath)
+	be.DirIVCache.store(parentDir, iv, cParentDir)
+	return cipherPath, nil
+}
+
+// DecryptPathDirIV - decrypt path using EME with DirIV
+func (be *NameTransform) DecryptPathDirIV(encryptedPath string, rootDir string, eme bool) (string, error) {
+	var wd = rootDir
+	var plainNames []string
+	encryptedNames := strings.Split(encryptedPath, "/")
+	toggledlog.Debug.Printf("DecryptPathDirIV: decrypting %v\n", encryptedNames)
+	for _, encryptedName := range encryptedNames {
+		iv, err := be.ReadDirIV(wd)
+		if err != nil {
+			return "", err
+		}
+		plainName, err := be.DecryptName(encryptedName, iv)
+		if err != nil {
+			return "", err
+		}
+		plainNames = append(plainNames, plainName)
+		wd = filepath.Join(wd, encryptedName)
+	}
+	return filepath.Join(plainNames...), nil
+}
diff --git a/internal/nametransform/names_noiv.go b/internal/nametransform/names_noiv.go
new file mode 100644
index 0000000..f301e52
--- /dev/null
+++ b/internal/nametransform/names_noiv.go
@@ -0,0 +1,63 @@
+package nametransform
+
+import (
+	"strings"
+)
+
+const (
+	OpEncrypt = iota
+	OpDecrypt
+)
+
+// DecryptPathNoIV - decrypt path using CBC without any IV.
+// This function is deprecated by the the more secure DirIV variant and only retained
+// for compatability with old filesystems.
+func (be *NameTransform) DecryptPathNoIV(cipherPath string) (plainPath string, err error) {
+	plainPath, err = be.translatePathNoIV(cipherPath, OpDecrypt)
+	return plainPath, err
+}
+
+// EncryptPathNoIV - decrypt path using CBC without any IV.
+// This function is deprecated by the the more secure DirIV variant and only retained
+// for compatability with old filesystems.
+func (be *NameTransform) EncryptPathNoIV(plainPath string) (cipherPath string) {
+	cipherPath, _ = be.translatePathNoIV(plainPath, OpEncrypt)
+	return cipherPath
+}
+
+// translatePathZeroIV - encrypt or decrypt path using CBC with an all-zero IV.
+// Just splits the string on "/" and hands the parts to encryptName() / decryptName()
+func (be *NameTransform) translatePathNoIV(path string, op int) (string, error) {
+	var err error
+
+	// Empty string means root directory
+	if path == "" {
+		return path, err
+	}
+
+	zeroIV := make([]byte, dirIVLen)
+
+	// Run operation on each path component
+	var translatedParts []string
+	parts := strings.Split(path, "/")
+	for _, part := range parts {
+		if part == "" {
+			// This happens on "/foo/bar/" on the front and on the end.
+			// Don't panic.
+			translatedParts = append(translatedParts, "")
+			continue
+		}
+		var newPart string
+		if op == OpEncrypt {
+			newPart = be.encryptName(part, zeroIV)
+		} else {
+			newPart, err = be.DecryptName(part, zeroIV)
+			if err != nil {
+				return "", err
+			}
+		}
+		translatedParts = append(translatedParts, newPart)
+	}
+
+	return strings.Join(translatedParts, "/"), err
+}
diff --git a/internal/nametransform/names_test.go b/internal/nametransform/names_test.go
new file mode 100644
index 0000000..4a901be
--- /dev/null
+++ b/internal/nametransform/names_test.go
@@ -0,0 +1,58 @@
+package nametransform
+
+import (
+	"bytes"
+	"testing"
+)
+
+func TestEncryptPathNoIV(t *testing.T) {
+	var s []string
+	s = append(s, "foo")
+	s = append(s, "foo12312312312312312313123123123")
+	s = append(s, "123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890")
+
+	key := make([]byte, KEY_LEN)
+	fs := NewCryptFS(key, true, false, true)
+
+	for _, n := range s {
+		c := fs.EncryptPathNoIV(n)
+		d, err := fs.DecryptPathNoIV(c)
+		if err != nil {
+			t.Errorf("Got error from DecryptPathNoIV: %s", err)
+		}
+		if d != n {
+			t.Errorf("Content mismatch, n != d: n=%s c=%s d=%s", n, c, d)
+		}
+	}
+}
+
+func TestPad16(t *testing.T) {
+	var s [][]byte
+	s = append(s, []byte("foo"))
+	s = append(s, []byte("12345678901234567"))
+	s = append(s, []byte("12345678901234567abcdefg"))
+
+	key := make([]byte, KEY_LEN)
+	fs := NewCryptFS(key, true, false, true)
+
+	for i := range s {
+		orig := s[i]
+		padded := fs.pad16(orig)
+		if len(padded) <= len(orig) {
+			t.Errorf("Padded length not bigger than orig: %d", len(padded))
+		}
+		if len(padded)%16 != 0 {
+			t.Errorf("Length is not aligend: %d", len(padded))
+		}
+		unpadded, err := fs.unPad16(padded)
+		if err != nil {
+			t.Error("unPad16 returned error:", err)
+		}
+		if len(unpadded) != len(orig) {
+			t.Errorf("Size mismatch: orig=%d unpadded=%d", len(s[i]), len(unpadded))
+		}
+		if !bytes.Equal(orig, unpadded) {
+			t.Error("Content mismatch orig vs unpadded")
+		}
+	}
+}
diff --git a/internal/nametransform/pad16.go b/internal/nametransform/pad16.go
new file mode 100644
index 0000000..c15160e
--- /dev/null
+++ b/internal/nametransform/pad16.go
@@ -0,0 +1,60 @@
+package nametransform
+
+import (
+	"fmt"
+	"crypto/aes"
+	"errors"
+)
+
+// pad16 - pad data to AES block size (=16 byte) using standard PKCS#7 padding
+// https://tools.ietf.org/html/rfc5652#section-6.3
+func pad16(orig []byte) (padded []byte) {
+	oldLen := len(orig)
+	if oldLen == 0 {
+		panic("Padding zero-length string makes no sense")
+	}
+	padLen := aes.BlockSize - oldLen%aes.BlockSize
+	if padLen == 0 {
+		padLen = aes.BlockSize
+	}
+	newLen := oldLen + padLen
+	padded = make([]byte, newLen)
+	copy(padded, orig)
+	padByte := byte(padLen)
+	for i := oldLen; i < newLen; i++ {
+		padded[i] = padByte
+	}
+	return padded
+}
+
+// unPad16 - remove padding
+func unPad16(padded []byte) ([]byte, error) {
+	oldLen := len(padded)
+	if oldLen%aes.BlockSize != 0 {
+		return nil, errors.New("Unaligned size")
+	}
+	// The last byte is always a padding byte
+	padByte := padded[oldLen-1]
+	// The padding byte's value is the padding length
+	padLen := int(padByte)
+	// Padding must be at least 1 byte
+	if padLen <= 0 {
+		return nil, errors.New("Padding cannot be zero-length")
+	}
+	// Larger paddings make no sense
+	if padLen > aes.BlockSize {
+		return nil, fmt.Errorf("Padding too long, padLen = %d > 16", padLen)
+	}
+	// All padding bytes must be identical
+	for i := oldLen - padLen; i < oldLen; i++ {
+		if padded[i] != padByte {
+			return nil, fmt.Errorf("Padding byte at i=%d is invalid", i)
+		}
+	}
+	newLen := oldLen - padLen
+	// Padding an empty string makes no sense
+	if newLen == 0 {
+		return nil, errors.New("Unpadded length is zero")
+	}
+	return padded[0:newLen], nil
+}