Major refactoring: Split up "cryptfs" into several internal packages

"git status" for reference:

deleted:    cryptfs/cryptfs.go
deleted:    cryptfs/names_core.go
modified:   integration_tests/cli_test.go
modified:   integration_tests/helpers.go
renamed:    cryptfs/config_file.go -> internal/configfile/config_file.go
renamed:    cryptfs/config_test.go -> internal/configfile/config_test.go
renamed:    cryptfs/config_test/.gitignore -> internal/configfile/config_test/.gitignore
renamed:    cryptfs/config_test/PlaintextNames.conf -> internal/configfile/config_test/PlaintextNames.conf
renamed:    cryptfs/config_test/StrangeFeature.conf -> internal/configfile/config_test/StrangeFeature.conf
renamed:    cryptfs/config_test/v1.conf -> internal/configfile/config_test/v1.conf
renamed:    cryptfs/config_test/v2.conf -> internal/configfile/config_test/v2.conf
renamed:    cryptfs/kdf.go -> internal/configfile/kdf.go
renamed:    cryptfs/kdf_test.go -> internal/configfile/kdf_test.go
renamed:    cryptfs/cryptfs_content.go -> internal/contentenc/content.go
new file:   internal/contentenc/content_api.go
renamed:    cryptfs/content_test.go -> internal/contentenc/content_test.go
renamed:    cryptfs/file_header.go -> internal/contentenc/file_header.go
renamed:    cryptfs/intrablock.go -> internal/contentenc/intrablock.go
renamed:    cryptfs/address_translation.go -> internal/contentenc/offsets.go
new file:   internal/cryptocore/crypto_api.go
renamed:    cryptfs/gcm_go1.4.go -> internal/cryptocore/gcm_go1.4.go
renamed:    cryptfs/gcm_go1.5.go -> internal/cryptocore/gcm_go1.5.go
renamed:    cryptfs/nonce.go -> internal/cryptocore/nonce.go
renamed:    cryptfs/openssl_aead.go -> internal/cryptocore/openssl_aead.go
renamed:    cryptfs/openssl_benchmark.bash -> internal/cryptocore/openssl_benchmark.bash
renamed:    cryptfs/openssl_test.go -> internal/cryptocore/openssl_test.go
new file:   internal/nametransform/name_api.go
new file:   internal/nametransform/names_core.go
renamed:    cryptfs/names_diriv.go -> internal/nametransform/names_diriv.go
renamed:    cryptfs/names_noiv.go -> internal/nametransform/names_noiv.go
renamed:    cryptfs/names_test.go -> internal/nametransform/names_test.go
new file:   internal/nametransform/pad16.go
renamed:    cryptfs/log.go -> internal/toggledlog/log.go
renamed:    cryptfs/log_go1.4.go -> internal/toggledlog/log_go1.4.go
renamed:    cryptfs/log_go1.5.go -> internal/toggledlog/log_go1.5.go
modified:   main.go
modified:   masterkey.go
modified:   pathfs_frontend/file.go
modified:   pathfs_frontend/file_holes.go
modified:   pathfs_frontend/fs.go
modified:   pathfs_frontend/fs_dir.go
modified:   pathfs_frontend/names.go
modified:   test.bash

6 files changed, 446 insertions, 0 deletions

diff --git a/internal/contentenc/content.go b/internal/contentenc/content.go
new file mode 100644
index 0000000..14135a2
--- /dev/null
+++ b/internal/contentenc/content.go
@@ -0,0 +1,116 @@
+package contentenc
+
+// File content encryption / decryption
+
+import (
+	"encoding/binary"
+	"bytes"
+	"encoding/hex"
+	"errors"
+
+	"github.com/rfjakob/gocryptfs/internal/toggledlog"
+)
+
+// DecryptBlocks - Decrypt a number of blocks
+func (be *ContentEnc) DecryptBlocks(ciphertext []byte, firstBlockNo uint64, fileId []byte) ([]byte, error) {
+	cBuf := bytes.NewBuffer(ciphertext)
+	var err error
+	var pBuf bytes.Buffer
+	for cBuf.Len() > 0 {
+		cBlock := cBuf.Next(int(be.cipherBS))
+		var pBlock []byte
+		pBlock, err = be.DecryptBlock(cBlock, firstBlockNo, fileId)
+		if err != nil {
+			break
+		}
+		pBuf.Write(pBlock)
+		firstBlockNo++
+	}
+	return pBuf.Bytes(), err
+}
+
+// DecryptBlock - Verify and decrypt GCM block
+//
+// Corner case: A full-sized block of all-zero ciphertext bytes is translated
+// to an all-zero plaintext block, i.e. file hole passtrough.
+func (be *ContentEnc) DecryptBlock(ciphertext []byte, blockNo uint64, fileId []byte) ([]byte, error) {
+
+	// Empty block?
+	if len(ciphertext) == 0 {
+		return ciphertext, nil
+	}
+
+	// All-zero block?
+	if bytes.Equal(ciphertext, be.allZeroBlock) {
+		toggledlog.Debug.Printf("DecryptBlock: file hole encountered")
+		return make([]byte, be.plainBS), nil
+	}
+
+	if len(ciphertext) < be.cryptoCore.IVLen {
+		toggledlog.Warn.Printf("DecryptBlock: Block is too short: %d bytes", len(ciphertext))
+		return nil, errors.New("Block is too short")
+	}
+
+	// Extract nonce
+	nonce := ciphertext[:be.cryptoCore.IVLen]
+	ciphertextOrig := ciphertext
+	ciphertext = ciphertext[be.cryptoCore.IVLen:]
+
+	// Decrypt
+	var plaintext []byte
+	aData := make([]byte, 8)
+	aData = append(aData, fileId...)
+	binary.BigEndian.PutUint64(aData, blockNo)
+	plaintext, err := be.cryptoCore.Gcm.Open(plaintext, nonce, ciphertext, aData)
+
+	if err != nil {
+		toggledlog.Warn.Printf("DecryptBlock: %s, len=%d", err.Error(), len(ciphertextOrig))
+		toggledlog.Debug.Println(hex.Dump(ciphertextOrig))
+		return nil, err
+	}
+
+	return plaintext, nil
+}
+
+// encryptBlock - Encrypt and add IV and MAC
+func (be *ContentEnc) EncryptBlock(plaintext []byte, blockNo uint64, fileID []byte) []byte {
+
+	// Empty block?
+	if len(plaintext) == 0 {
+		return plaintext
+	}
+
+	// Get fresh nonce
+	nonce := be.cryptoCore.GcmIVGen.Get()
+
+	// Authenticate block with block number and file ID
+	aData := make([]byte, 8)
+	binary.BigEndian.PutUint64(aData, blockNo)
+	aData = append(aData, fileID...)
+
+	// Encrypt plaintext and append to nonce
+	ciphertext := be.cryptoCore.Gcm.Seal(nonce, nonce, plaintext, aData)
+
+	return ciphertext
+}
+
+// MergeBlocks - Merge newData into oldData at offset
+// New block may be bigger than both newData and oldData
+func (be *ContentEnc) MergeBlocks(oldData []byte, newData []byte, offset int) []byte {
+
+	// Make block of maximum size
+	out := make([]byte, be.plainBS)
+
+	// Copy old and new data into it
+	copy(out, oldData)
+	l := len(newData)
+	copy(out[offset:offset+l], newData)
+
+	// Crop to length
+	outLen := len(oldData)
+	newLen := offset + len(newData)
+	if outLen < newLen {
+		outLen = newLen
+	}
+	return out[0:outLen]
+}
diff --git a/internal/contentenc/content_api.go b/internal/contentenc/content_api.go
new file mode 100644
index 0000000..1700d35
--- /dev/null
+++ b/internal/contentenc/content_api.go
@@ -0,0 +1,31 @@
+package contentenc
+
+import "github.com/rfjakob/gocryptfs/internal/cryptocore"
+
+type ContentEnc struct {
+	// Cryptographic primitives
+	cryptoCore *cryptocore.CryptoCore
+	// Plaintext block size
+	plainBS     uint64
+	// Ciphertext block size
+	cipherBS    uint64
+	// All-zero block of size cipherBS, for fast compares
+	allZeroBlock []byte
+}
+
+func New(cc *cryptocore.CryptoCore, plainBS uint64) *ContentEnc {
+
+	cipherBS := plainBS + uint64(cc.IVLen) + cryptocore.AuthTagLen
+
+	return &ContentEnc{
+		cryptoCore: cc,
+		plainBS: plainBS,
+		cipherBS: cipherBS,
+		allZeroBlock: make([]byte, cipherBS),
+	}
+}
+
+
+func (be *ContentEnc) PlainBS() uint64 {
+	return be.plainBS
+}
diff --git a/internal/contentenc/content_test.go b/internal/contentenc/content_test.go
new file mode 100644
index 0000000..70ad58d
--- /dev/null
+++ b/internal/contentenc/content_test.go
@@ -0,0 +1,91 @@
+package contentenc
+
+import (
+	"testing"
+)
+
+type testRange struct {
+	offset uint64
+	length uint64
+}
+
+func TestSplitRange(t *testing.T) {
+	var ranges []testRange
+
+	ranges = append(ranges, testRange{0, 70000},
+		testRange{0, 10},
+		testRange{234, 6511},
+		testRange{65444, 54},
+		testRange{0, 1024 * 1024},
+		testRange{0, 65536},
+		testRange{6654, 8945})
+
+	key := make([]byte, KEY_LEN)
+	f := NewCryptFS(key, true, false, true)
+
+	for _, r := range ranges {
+		parts := f.ExplodePlainRange(r.offset, r.length)
+		var lastBlockNo uint64 = 1 << 63
+		for _, p := range parts {
+			if p.BlockNo == lastBlockNo {
+				t.Errorf("Duplicate block number %d", p.BlockNo)
+			}
+			lastBlockNo = p.BlockNo
+			if p.Length > DEFAULT_PLAINBS || p.Skip >= DEFAULT_PLAINBS {
+				t.Errorf("Test fail: n=%d, length=%d, offset=%d\n", p.BlockNo, p.Length, p.Skip)
+			}
+		}
+	}
+}
+
+func TestCiphertextRange(t *testing.T) {
+	var ranges []testRange
+
+	ranges = append(ranges, testRange{0, 70000},
+		testRange{0, 10},
+		testRange{234, 6511},
+		testRange{65444, 54},
+		testRange{6654, 8945})
+
+	key := make([]byte, KEY_LEN)
+	f := NewCryptFS(key, true, false, true)
+
+	for _, r := range ranges {
+
+		blocks := f.ExplodePlainRange(r.offset, r.length)
+		alignedOffset, alignedLength := blocks[0].JointCiphertextRange(blocks)
+		skipBytes := blocks[0].Skip
+
+		if alignedLength < r.length {
+			t.Errorf("alignedLength=%d is smaller than length=%d", alignedLength, r.length)
+		}
+		if (alignedOffset-HEADER_LEN)%f.cipherBS != 0 {
+			t.Errorf("alignedOffset=%d is not aligned", alignedOffset)
+		}
+		if r.offset%f.plainBS != 0 && skipBytes == 0 {
+			t.Errorf("skipBytes=0")
+		}
+	}
+}
+
+func TestBlockNo(t *testing.T) {
+	key := make([]byte, KEY_LEN)
+	f := NewCryptFS(key, true, false, true)
+
+	b := f.CipherOffToBlockNo(788)
+	if b != 0 {
+		t.Errorf("actual: %d", b)
+	}
+	b = f.CipherOffToBlockNo(HEADER_LEN + f.cipherBS)
+	if b != 1 {
+		t.Errorf("actual: %d", b)
+	}
+	b = f.PlainOffToBlockNo(788)
+	if b != 0 {
+		t.Errorf("actual: %d", b)
+	}
+	b = f.PlainOffToBlockNo(f.plainBS)
+	if b != 1 {
+		t.Errorf("actual: %d", b)
+	}
+}
diff --git a/internal/contentenc/file_header.go b/internal/contentenc/file_header.go
new file mode 100644
index 0000000..8a9dd2c
--- /dev/null
+++ b/internal/contentenc/file_header.go
@@ -0,0 +1,60 @@
+package contentenc
+
+// Per-file header
+//
+// Format: [ "Version" uint16 big endian ] [ "Id" 16 random bytes ]
+
+import (
+	"encoding/binary"
+	"fmt"
+
+	"github.com/rfjakob/gocryptfs/internal/cryptocore"
+)
+
+const (
+	// Current On-Disk-Format version
+	CurrentVersion = 2
+
+	HEADER_VERSION_LEN     = 2                                  // uint16
+	HEADER_ID_LEN          = 16                                 // 128 bit random file id
+	HEADER_LEN             = HEADER_VERSION_LEN + HEADER_ID_LEN // Total header length
+)
+
+type FileHeader struct {
+	Version uint16
+	Id      []byte
+}
+
+// Pack - serialize fileHeader object
+func (h *FileHeader) Pack() []byte {
+	if len(h.Id) != HEADER_ID_LEN || h.Version != CurrentVersion {
+		panic("FileHeader object not properly initialized")
+	}
+	buf := make([]byte, HEADER_LEN)
+	binary.BigEndian.PutUint16(buf[0:HEADER_VERSION_LEN], h.Version)
+	copy(buf[HEADER_VERSION_LEN:], h.Id)
+	return buf
+
+}
+
+// ParseHeader - parse "buf" into fileHeader object
+func ParseHeader(buf []byte) (*FileHeader, error) {
+	if len(buf) != HEADER_LEN {
+		return nil, fmt.Errorf("ParseHeader: invalid length: got %d, want %d", len(buf), HEADER_LEN)
+	}
+	var h FileHeader
+	h.Version = binary.BigEndian.Uint16(buf[0:HEADER_VERSION_LEN])
+	if h.Version != CurrentVersion {
+		return nil, fmt.Errorf("ParseHeader: invalid version: got %d, want %d", h.Version, CurrentVersion)
+	}
+	h.Id = buf[HEADER_VERSION_LEN:]
+	return &h, nil
+}
+
+// RandomHeader - create new fileHeader object with random Id
+func RandomHeader() *FileHeader {
+	var h FileHeader
+	h.Version = CurrentVersion
+	h.Id = cryptocore.RandBytes(HEADER_ID_LEN)
+	return &h
+}
diff --git a/internal/contentenc/intrablock.go b/internal/contentenc/intrablock.go
new file mode 100644
index 0000000..330b980
--- /dev/null
+++ b/internal/contentenc/intrablock.go
@@ -0,0 +1,51 @@
+package contentenc
+
+// intraBlock identifies a part of a file block
+type intraBlock struct {
+	BlockNo uint64 // Block number in file
+	Skip    uint64 // Offset into block plaintext
+	Length  uint64 // Length of data from this block
+	fs      *ContentEnc
+}
+
+// isPartial - is the block partial? This means we have to do read-modify-write.
+func (ib *intraBlock) IsPartial() bool {
+	if ib.Skip > 0 || ib.Length < ib.fs.plainBS {
+		return true
+	}
+	return false
+}
+
+// CiphertextRange - get byte range in ciphertext file corresponding to BlockNo
+// (complete block)
+func (ib *intraBlock) CiphertextRange() (offset uint64, length uint64) {
+	return ib.fs.BlockNoToCipherOff(ib.BlockNo), ib.fs.cipherBS
+}
+
+// PlaintextRange - get byte range in plaintext corresponding to BlockNo
+// (complete block)
+func (ib *intraBlock) PlaintextRange() (offset uint64, length uint64) {
+	return ib.fs.BlockNoToPlainOff(ib.BlockNo), ib.fs.plainBS
+}
+
+// CropBlock - crop a potentially larger plaintext block down to the relevant part
+func (ib *intraBlock) CropBlock(d []byte) []byte {
+	lenHave := len(d)
+	lenWant := int(ib.Skip + ib.Length)
+	if lenHave < lenWant {
+		return d[ib.Skip:lenHave]
+	}
+	return d[ib.Skip:lenWant]
+}
+
+// Ciphertext range corresponding to the sum of all "blocks" (complete blocks)
+func (ib *intraBlock) JointCiphertextRange(blocks []intraBlock) (offset uint64, length uint64) {
+	firstBlock := blocks[0]
+	lastBlock := blocks[len(blocks)-1]
+
+	offset = ib.fs.BlockNoToCipherOff(firstBlock.BlockNo)
+	offsetLast := ib.fs.BlockNoToCipherOff(lastBlock.BlockNo)
+	length = offsetLast + ib.fs.cipherBS - offset
+
+	return offset, length
+}
diff --git a/internal/contentenc/offsets.go b/internal/contentenc/offsets.go
new file mode 100644
index 0000000..1b5952f
--- /dev/null
+++ b/internal/contentenc/offsets.go
@@ -0,0 +1,97 @@
+package contentenc
+
+import (
+	"github.com/rfjakob/gocryptfs/internal/toggledlog"
+)
+
+// Contentenc methods that translate offsets between ciphertext and plaintext
+
+// get the block number at plain-text offset
+func (be *ContentEnc) PlainOffToBlockNo(plainOffset uint64) uint64 {
+	return plainOffset / be.plainBS
+}
+
+// get the block number at ciphter-text offset
+func (be *ContentEnc) CipherOffToBlockNo(cipherOffset uint64) uint64 {
+	return (cipherOffset - HEADER_LEN) / be.cipherBS
+}
+
+// get ciphertext offset of block "blockNo"
+func (be *ContentEnc) BlockNoToCipherOff(blockNo uint64) uint64 {
+	return HEADER_LEN + blockNo*be.cipherBS
+}
+
+// get plaintext offset of block "blockNo"
+func (be *ContentEnc) BlockNoToPlainOff(blockNo uint64) uint64 {
+	return blockNo * be.plainBS
+}
+
+// PlainSize - calculate plaintext size from ciphertext size
+func (be *ContentEnc) CipherSizeToPlainSize(cipherSize uint64) uint64 {
+
+	// Zero sized files stay zero-sized
+	if cipherSize == 0 {
+		return 0
+	}
+
+	if cipherSize == HEADER_LEN {
+		toggledlog.Warn.Printf("cipherSize %d == header size: interrupted write?\n", cipherSize)
+		return 0
+	}
+
+	if cipherSize < HEADER_LEN {
+		toggledlog.Warn.Printf("cipherSize %d < header size: corrupt file\n", cipherSize)
+		return 0
+	}
+
+	// Block number at last byte
+	blockNo := be.CipherOffToBlockNo(cipherSize - 1)
+	blockCount := blockNo + 1
+
+	overhead := be.BlockOverhead()*blockCount + HEADER_LEN
+
+	return cipherSize - overhead
+}
+
+// CipherSize - calculate ciphertext size from plaintext size
+func (be *ContentEnc) PlainSizeToCipherSize(plainSize uint64) uint64 {
+
+	// Block number at last byte
+	blockNo := be.PlainOffToBlockNo(plainSize - 1)
+	blockCount := blockNo + 1
+
+	overhead := be.BlockOverhead()*blockCount + HEADER_LEN
+
+	return plainSize + overhead
+}
+
+// Split a plaintext byte range into (possibly partial) blocks
+func (be *ContentEnc) ExplodePlainRange(offset uint64, length uint64) []intraBlock {
+	var blocks []intraBlock
+	var nextBlock intraBlock
+	nextBlock.fs = be
+
+	for length > 0 {
+		nextBlock.BlockNo = be.PlainOffToBlockNo(offset)
+		nextBlock.Skip = offset - be.BlockNoToPlainOff(nextBlock.BlockNo)
+
+		// Minimum of remaining data and remaining space in the block
+		nextBlock.Length = MinUint64(length, be.plainBS-nextBlock.Skip)
+
+		blocks = append(blocks, nextBlock)
+		offset += nextBlock.Length
+		length -= nextBlock.Length
+	}
+	return blocks
+}
+
+func (be *ContentEnc) BlockOverhead() uint64 {
+	return be.cipherBS - be.plainBS
+}
+
+func MinUint64(x uint64, y uint64) uint64 {
+	if x < y {
+		return x
+	}
+	return y
+}