Major refactoring: Split up "cryptfs" into several internal packages

"git status" for reference:

deleted:    cryptfs/cryptfs.go
deleted:    cryptfs/names_core.go
modified:   integration_tests/cli_test.go
modified:   integration_tests/helpers.go
renamed:    cryptfs/config_file.go -> internal/configfile/config_file.go
renamed:    cryptfs/config_test.go -> internal/configfile/config_test.go
renamed:    cryptfs/config_test/.gitignore -> internal/configfile/config_test/.gitignore
renamed:    cryptfs/config_test/PlaintextNames.conf -> internal/configfile/config_test/PlaintextNames.conf
renamed:    cryptfs/config_test/StrangeFeature.conf -> internal/configfile/config_test/StrangeFeature.conf
renamed:    cryptfs/config_test/v1.conf -> internal/configfile/config_test/v1.conf
renamed:    cryptfs/config_test/v2.conf -> internal/configfile/config_test/v2.conf
renamed:    cryptfs/kdf.go -> internal/configfile/kdf.go
renamed:    cryptfs/kdf_test.go -> internal/configfile/kdf_test.go
renamed:    cryptfs/cryptfs_content.go -> internal/contentenc/content.go
new file:   internal/contentenc/content_api.go
renamed:    cryptfs/content_test.go -> internal/contentenc/content_test.go
renamed:    cryptfs/file_header.go -> internal/contentenc/file_header.go
renamed:    cryptfs/intrablock.go -> internal/contentenc/intrablock.go
renamed:    cryptfs/address_translation.go -> internal/contentenc/offsets.go
new file:   internal/cryptocore/crypto_api.go
renamed:    cryptfs/gcm_go1.4.go -> internal/cryptocore/gcm_go1.4.go
renamed:    cryptfs/gcm_go1.5.go -> internal/cryptocore/gcm_go1.5.go
renamed:    cryptfs/nonce.go -> internal/cryptocore/nonce.go
renamed:    cryptfs/openssl_aead.go -> internal/cryptocore/openssl_aead.go
renamed:    cryptfs/openssl_benchmark.bash -> internal/cryptocore/openssl_benchmark.bash
renamed:    cryptfs/openssl_test.go -> internal/cryptocore/openssl_test.go
new file:   internal/nametransform/name_api.go
new file:   internal/nametransform/names_core.go
renamed:    cryptfs/names_diriv.go -> internal/nametransform/names_diriv.go
renamed:    cryptfs/names_noiv.go -> internal/nametransform/names_noiv.go
renamed:    cryptfs/names_test.go -> internal/nametransform/names_test.go
new file:   internal/nametransform/pad16.go
renamed:    cryptfs/log.go -> internal/toggledlog/log.go
renamed:    cryptfs/log_go1.4.go -> internal/toggledlog/log_go1.4.go
renamed:    cryptfs/log_go1.5.go -> internal/toggledlog/log_go1.5.go
modified:   main.go
modified:   masterkey.go
modified:   pathfs_frontend/file.go
modified:   pathfs_frontend/file_holes.go
modified:   pathfs_frontend/fs.go
modified:   pathfs_frontend/fs_dir.go
modified:   pathfs_frontend/names.go
modified:   test.bash

9 files changed, 446 insertions, 0 deletions

diff --git a/internal/configfile/config_file.go b/internal/configfile/config_file.go
new file mode 100644
index 0000000..0128acc
--- /dev/null
+++ b/internal/configfile/config_file.go
@@ -0,0 +1,195 @@
+package configfile
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
+
+	"github.com/rfjakob/gocryptfs/internal/cryptocore"
+	"github.com/rfjakob/gocryptfs/internal/contentenc"
+	"github.com/rfjakob/gocryptfs/internal/toggledlog"
+)
+import "os"
+
+const (
+	// The dot "." is not used in base64url (RFC4648), hence
+	// we can never clash with an encrypted file.
+	ConfDefaultName = "gocryptfs.conf"
+)
+
+type ConfFile struct {
+	// File the config is saved to. Not exported to JSON.
+	filename string
+	// Encrypted AES key, unlocked using a password hashed with scrypt
+	EncryptedKey []byte
+	// Stores parameters for scrypt hashing (key derivation)
+	ScryptObject scryptKdf
+	// The On-Disk-Format version this filesystem uses
+	Version uint16
+	// List of feature flags this filesystem has enabled.
+	// If gocryptfs encounters a feature flag it does not support, it will refuse
+	// mounting. This mechanism is analogous to the ext4 feature flags that are
+	// stored in the superblock.
+	FeatureFlags []string
+}
+
+// CreateConfFile - create a new config with a random key encrypted with
+// "password" and write it to "filename".
+// Uses scrypt with cost parameter logN.
+func CreateConfFile(filename string, password string, plaintextNames bool, logN int) error {
+	var cf ConfFile
+	cf.filename = filename
+	cf.Version = contentenc.CurrentVersion
+
+	// Generate new random master key
+	key := cryptocore.RandBytes(cryptocore.KeyLen)
+
+	// Encrypt it using the password
+	// This sets ScryptObject and EncryptedKey
+	cf.EncryptKey(key, password, logN)
+
+	// Set feature flags
+	cf.FeatureFlags = append(cf.FeatureFlags, FlagGCMIV128)
+	if plaintextNames {
+		cf.FeatureFlags = append(cf.FeatureFlags, FlagPlaintextNames)
+	} else {
+		cf.FeatureFlags = append(cf.FeatureFlags, FlagDirIV)
+		cf.FeatureFlags = append(cf.FeatureFlags, FlagEMENames)
+	}
+
+	// Write file to disk
+	return cf.WriteFile()
+}
+
+// LoadConfFile - read config file from disk and decrypt the
+// contained key using password.
+//
+// Returns the decrypted key and the ConfFile object
+func LoadConfFile(filename string, password string) ([]byte, *ConfFile, error) {
+	var cf ConfFile
+	cf.filename = filename
+
+	// Read from disk
+	js, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// Unmarshal
+	err = json.Unmarshal(js, &cf)
+	if err != nil {
+		toggledlog.Warn.Printf("Failed to unmarshal config file")
+		return nil, nil, err
+	}
+
+	if cf.Version != contentenc.CurrentVersion {
+		return nil, nil, fmt.Errorf("Unsupported on-disk format %d", cf.Version)
+	}
+
+	for _, flag := range cf.FeatureFlags {
+		if cf.isFeatureFlagKnown(flag) == false {
+			return nil, nil, fmt.Errorf("Unsupported feature flag %s", flag)
+		}
+	}
+
+	// Generate derived key from password
+	scryptHash := cf.ScryptObject.DeriveKey(password)
+
+	// Unlock master key using password-based key
+	// We use stock go GCM instead of OpenSSL here as speed is not important
+	// and we get better error messages
+	cc := cryptocore.New(scryptHash, false, false)
+	ce := contentenc.New(cc, 4096)
+
+	key, err := ce.DecryptBlock(cf.EncryptedKey, 0, nil)
+	if err != nil {
+		toggledlog.Warn.Printf("failed to unlock master key: %s", err.Error())
+		toggledlog.Warn.Printf("Password incorrect.")
+		return nil, nil, err
+	}
+
+	return key, &cf, nil
+}
+
+// EncryptKey - encrypt "key" using an scrypt hash generated from "password"
+// and store it in cf.EncryptedKey.
+// Uses scrypt with cost parameter logN and stores the scrypt parameters in
+// cf.ScryptObject.
+func (cf *ConfFile) EncryptKey(key []byte, password string, logN int) {
+	// Generate derived key from password
+	cf.ScryptObject = NewScryptKdf(logN)
+	scryptHash := cf.ScryptObject.DeriveKey(password)
+
+	// Lock master key using password-based key
+	cc := cryptocore.New(scryptHash, false, false)
+	ce := contentenc.New(cc, 4096)
+	cf.EncryptedKey = ce.EncryptBlock(key, 0, nil)
+}
+
+// WriteFile - write out config in JSON format to file "filename.tmp"
+// then rename over "filename".
+// This way a password change atomically replaces the file.
+func (cf *ConfFile) WriteFile() error {
+	tmp := cf.filename + ".tmp"
+	// 0400 permissions: gocryptfs.conf should be kept secret and never be written to.
+	fd, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0400)
+	if err != nil {
+		return err
+	}
+	js, err := json.MarshalIndent(cf, "", "\t")
+	if err != nil {
+		return err
+	}
+	_, err = fd.Write(js)
+	if err != nil {
+		return err
+	}
+	err = fd.Sync()
+	if err != nil {
+		return err
+	}
+	err = fd.Close()
+	if err != nil {
+		return err
+	}
+	err = os.Rename(tmp, cf.filename)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+const (
+	// Understood Feature Flags.
+	// Also teach isFeatureFlagKnown() about any additions and
+	// add it to CreateConfFile() if you want to have it enabled by default.
+	FlagPlaintextNames = "PlaintextNames"
+	FlagDirIV          = "DirIV"
+	FlagEMENames       = "EMENames"
+	FlagGCMIV128       = "GCMIV128"
+)
+
+// Verify that we understand a feature flag
+func (cf *ConfFile) isFeatureFlagKnown(flag string) bool {
+	switch flag {
+	case FlagPlaintextNames, FlagDirIV, FlagEMENames, FlagGCMIV128:
+		return true
+	default:
+		return false
+	}
+}
+
+// isFeatureFlagSet - is the feature flag "flagWant" enabled?
+func (cf *ConfFile) IsFeatureFlagSet(flagWant string) bool {
+	if !cf.isFeatureFlagKnown(flagWant) {
+		log.Panicf("BUG: Tried to use unsupported feature flag %s", flagWant)
+	}
+	for _, flag := range cf.FeatureFlags {
+		if flag == flagWant {
+			return true
+		}
+	}
+	return false
+}
diff --git a/internal/configfile/config_test.go b/internal/configfile/config_test.go
new file mode 100644
index 0000000..6606d22
--- /dev/null
+++ b/internal/configfile/config_test.go
@@ -0,0 +1,83 @@
+package configfile
+
+import (
+	"fmt"
+	"testing"
+	"time"
+)
+
+func TestLoadV1(t *testing.T) {
+	_, _, err := LoadConfFile("config_test/v1.conf", "test")
+	if err == nil {
+		t.Errorf("Outdated v1 config file must fail to load but it didn't")
+	} else if testing.Verbose() {
+		fmt.Print(err)
+	}
+}
+
+// Load a known-good config file and verify that it takes at least 100ms
+// (brute-force protection)
+func TestLoadV2(t *testing.T) {
+	t1 := time.Now()
+
+	_, _, err := LoadConfFile("config_test/v2.conf", "foo")
+	if err != nil {
+		t.Errorf("Could not load v2 config file: %v", err)
+	}
+
+	elapsed := time.Since(t1)
+	if elapsed < 100*time.Millisecond {
+		t.Errorf("scrypt calculation runs too fast: %d ms", elapsed/time.Millisecond)
+	}
+}
+
+func TestLoadV2PwdError(t *testing.T) {
+	if !testing.Verbose() {
+		Warn.Enabled = false
+	}
+	_, _, err := LoadConfFile("config_test/v2.conf", "wrongpassword")
+	if err == nil {
+		t.Errorf("Loading with wrong password must fail but it didn't")
+	}
+}
+
+func TestLoadV2Feature(t *testing.T) {
+	_, _, err := LoadConfFile("config_test/PlaintextNames.conf", "test")
+	if err != nil {
+		t.Errorf("Could not load v2 PlaintextNames config file: %v", err)
+	}
+}
+
+func TestLoadV2StrangeFeature(t *testing.T) {
+	_, _, err := LoadConfFile("config_test/StrangeFeature.conf", "test")
+	if err == nil {
+		t.Errorf("Loading unknown feature must fail but it didn't")
+	} else if testing.Verbose() {
+		fmt.Print(err)
+	}
+}
+
+func TestCreateConfFile(t *testing.T) {
+	err := CreateConfFile("config_test/tmp.conf", "test", false, 10)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, _, err = LoadConfFile("config_test/tmp.conf", "test")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+}
+
+func TestIsFeatureFlagKnown(t *testing.T) {
+	var cf ConfFile
+	if !cf.isFeatureFlagKnown(FlagDirIV) {
+		t.Errorf("This flag should be known")
+	}
+	if !cf.isFeatureFlagKnown(FlagPlaintextNames) {
+		t.Errorf("This flag should be known")
+	}
+	if cf.isFeatureFlagKnown("StrangeFeatureFlag") {
+		t.Errorf("This flag should be NOT known")
+	}
+}
diff --git a/internal/configfile/config_test/.gitignore b/internal/configfile/config_test/.gitignore
new file mode 100644
index 0000000..0720169
--- /dev/null
+++ b/internal/configfile/config_test/.gitignore
@@ -0,0 +1 @@
+tmp.conf
diff --git a/internal/configfile/config_test/PlaintextNames.conf b/internal/configfile/config_test/PlaintextNames.conf
new file mode 100644
index 0000000..c1ff8cc
--- /dev/null
+++ b/internal/configfile/config_test/PlaintextNames.conf
@@ -0,0 +1,14 @@
+{
+	"EncryptedKey": "rG4u0argMq02V5G9Fa+gAaaHtNrj3wn7OZjP44hWOzO4yBFtn+Qn3PW4V6LMuKmGLEhyktCyWOI3K8lj",
+	"ScryptObject": {
+		"Salt": "bRjq1V63u5ML3FoTWx/GBXUhUVpTunOX3DPxS+yPjg0=",
+		"N": 65536,
+		"R": 8,
+		"P": 1,
+		"KeyLen": 32
+	},
+	"Version": 2,
+	"FeatureFlags": [
+		"PlaintextNames"
+	]
+}
diff --git a/internal/configfile/config_test/StrangeFeature.conf b/internal/configfile/config_test/StrangeFeature.conf
new file mode 100644
index 0000000..6a97781
--- /dev/null
+++ b/internal/configfile/config_test/StrangeFeature.conf
@@ -0,0 +1,14 @@
+{
+	"EncryptedKey": "rG4u0argMq02V5G9Fa+gAaaHtNrj3wn7OZjP44hWOzO4yBFtn+Qn3PW4V6LMuKmGLEhyktCyWOI3K8lj",
+	"ScryptObject": {
+		"Salt": "bRjq1V63u5ML3FoTWx/GBXUhUVpTunOX3DPxS+yPjg0=",
+		"N": 65536,
+		"R": 8,
+		"P": 1,
+		"KeyLen": 32
+	},
+	"Version": 2,
+	"FeatureFlags": [
+		"StrangeFeatureFlag"
+	]
+}
diff --git a/internal/configfile/config_test/v1.conf b/internal/configfile/config_test/v1.conf
new file mode 100644
index 0000000..588a25a
--- /dev/null
+++ b/internal/configfile/config_test/v1.conf
@@ -0,0 +1,11 @@
+{
+	"EncryptedKey": "t6YAvFQJvbv46c93bHQ5IZnvNz80DA9cohGoSPL/2M257LuIigow6jbr8b9HhnbDqHTCcz7aKkMDzneF",
+	"ScryptObject": {
+		"Salt": "yT4yQmmRmVNx2P0tJrUswk5SQzZaL6Z8kUteAoNJkXM=",
+		"N": 65536,
+		"R": 8,
+		"P": 1,
+		"KeyLen": 32
+	},
+	"Version": 1
+}
diff --git a/internal/configfile/config_test/v2.conf b/internal/configfile/config_test/v2.conf
new file mode 100644
index 0000000..8ef3dcf
--- /dev/null
+++ b/internal/configfile/config_test/v2.conf
@@ -0,0 +1,11 @@
+{
+	"EncryptedKey": "RvxJnZWKTBSU21+7xbl08xlZyNyUCkpIqlK8Z51TUrRiBhqqNPxbdk1WXMvmOf/YzZ85Xbyz+DGM+SDf",
+	"ScryptObject": {
+		"Salt": "2OrFRfdW/5SanbMXM3TMINmfMO6oYU9awG+NZ77V8E8=",
+		"N": 65536,
+		"R": 8,
+		"P": 1,
+		"KeyLen": 32
+	},
+	"Version": 2
+}
diff --git a/internal/configfile/kdf.go b/internal/configfile/kdf.go
new file mode 100644
index 0000000..f1a7a40
--- /dev/null
+++ b/internal/configfile/kdf.go
@@ -0,0 +1,57 @@
+package configfile
+
+import (
+	"fmt"
+	"math"
+	"os"
+
+	"golang.org/x/crypto/scrypt"
+
+	"github.com/rfjakob/gocryptfs/internal/cryptocore"
+)
+
+const (
+	// 1 << 16 uses 64MB of memory,
+	// takes 4 seconds on my Atom Z3735F netbook
+	ScryptDefaultLogN = 16
+)
+
+type scryptKdf struct {
+	Salt   []byte
+	N      int
+	R      int
+	P      int
+	KeyLen int
+}
+
+func NewScryptKdf(logN int) scryptKdf {
+	var s scryptKdf
+	s.Salt = cryptocore.RandBytes(cryptocore.KeyLen)
+	if logN <= 0 {
+		s.N = 1 << ScryptDefaultLogN
+	} else {
+		if logN < 10 {
+			fmt.Println("Error: scryptn below 10 is too low to make sense. Aborting.")
+			os.Exit(1)
+		}
+		s.N = 1 << uint32(logN)
+	}
+	s.R = 8 // Always 8
+	s.P = 1 // Always 1
+	s.KeyLen = cryptocore.KeyLen
+	return s
+}
+
+func (s *scryptKdf) DeriveKey(pw string) []byte {
+	k, err := scrypt.Key([]byte(pw), s.Salt, s.N, s.R, s.P, s.KeyLen)
+	if err != nil {
+		panic(fmt.Sprintf("DeriveKey failed: %s", err.Error()))
+	}
+	return k
+}
+
+// LogN - N is saved as 2^LogN, but LogN is much easier to work with.
+// This function gives you LogN = Log2(N).
+func (s *scryptKdf) LogN() int {
+	return int(math.Log2(float64(s.N)) + 0.5)
+}
diff --git a/internal/configfile/kdf_test.go b/internal/configfile/kdf_test.go
new file mode 100644
index 0000000..bc095ab
--- /dev/null
+++ b/internal/configfile/kdf_test.go
@@ -0,0 +1,60 @@
+package configfile
+
+import (
+	"testing"
+)
+
+/*
+Results on a 2.7GHz Pentium G630:
+
+gocryptfs/cryptfs$ go test -bench=.
+PASS
+BenchmarkScrypt10-2	     300	   6021435 ns/op ... 6ms
+BenchmarkScrypt11-2	     100	  11861460 ns/op
+BenchmarkScrypt12-2	     100	  23420822 ns/op
+BenchmarkScrypt13-2	      30	  47666518 ns/op
+BenchmarkScrypt14-2	      20	  92561590 ns/op ... 92ms
+BenchmarkScrypt15-2	      10	 183971593 ns/op
+BenchmarkScrypt16-2	       3	 368506365 ns/op
+BenchmarkScrypt17-2	       2	 755502608 ns/op ... 755ms
+ok  	github.com/rfjakob/gocryptfs/cryptfs	18.772s
+*/
+
+func benchmarkScryptN(n int, b *testing.B) {
+	kdf := NewScryptKdf(n)
+	for i := 0; i < b.N; i++ {
+		kdf.DeriveKey("test")
+	}
+}
+
+func BenchmarkScrypt10(b *testing.B) {
+	benchmarkScryptN(10, b)
+}
+
+func BenchmarkScrypt11(b *testing.B) {
+	benchmarkScryptN(11, b)
+}
+
+func BenchmarkScrypt12(b *testing.B) {
+	benchmarkScryptN(12, b)
+}
+
+func BenchmarkScrypt13(b *testing.B) {
+	benchmarkScryptN(13, b)
+}
+
+func BenchmarkScrypt14(b *testing.B) {
+	benchmarkScryptN(14, b)
+}
+
+func BenchmarkScrypt15(b *testing.B) {
+	benchmarkScryptN(15, b)
+}
+
+func BenchmarkScrypt16(b *testing.B) {
+	benchmarkScryptN(16, b)
+}
+
+func BenchmarkScrypt17(b *testing.B) {
+	benchmarkScryptN(17, b)
+}