aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJakob Unterwurzacher2017-08-13 21:13:44 +0200
committerJakob Unterwurzacher2017-08-15 19:03:57 +0200
commite50a6a57e57bc3cc925ba9a6e7f4dc1da4da3c84 (patch)
tree79df16efd4d2474502a0e5116a4ee9599a33daee
parentaffb1c2f6617d66bdc9fda41b017e0de000c3691 (diff)
syscallcompat: implement Getdents()
The Readdir function provided by os is inherently slow because it calls Lstat on all files. Getdents gives us all the information we need, but does not have a proper wrapper in the stdlib. Implement the "Getdents()" wrapper function that calls syscall.Getdents() and parses the returned byte blob to a fuse.DirEntry slice.
-rw-r--r--internal/syscallcompat/getdents_linux.go138
-rw-r--r--internal/syscallcompat/getdents_other.go17
-rw-r--r--internal/syscallcompat/getdents_test.go77
3 files changed, 232 insertions, 0 deletions
diff --git a/internal/syscallcompat/getdents_linux.go b/internal/syscallcompat/getdents_linux.go
new file mode 100644
index 0000000..97cd75f
--- /dev/null
+++ b/internal/syscallcompat/getdents_linux.go
@@ -0,0 +1,138 @@
+// +build linux
+
+package syscallcompat
+
+// Other implementations of getdents in Go:
+// https://github.com/ericlagergren/go-gnulib/blob/cb7a6e136427e242099b2c29d661016c19458801/dirent/getdents_unix.go
+// https://github.com/golang/tools/blob/5831d16d18029819d39f99bdc2060b8eff410b6b/imports/fastwalk_unix.go
+
+import (
+ "bytes"
+ "syscall"
+ "unsafe"
+
+ "github.com/hanwen/go-fuse/fuse"
+
+ "github.com/rfjakob/gocryptfs/internal/tlog"
+)
+
+// HaveGetdents is true if we have a working implementation of Getdents
+const HaveGetdents = true
+
+const sizeofDirent = int(unsafe.Sizeof(syscall.Dirent{}))
+
+// Getdents wraps syscall.Getdents and converts the result to []fuse.DirEntry.
+// The function takes a path instead of an fd because we need to be able to
+// call Lstat on files. Fstatat is not yet available in Go as of v1.9:
+// https://github.com/golang/go/issues/14216
+func Getdents(dir string) ([]fuse.DirEntry, error) {
+ fd, err := syscall.Open(dir, syscall.O_RDONLY, 0)
+ if err != nil {
+ return nil, err
+ }
+ defer syscall.Close(fd)
+ // Collect syscall result in smartBuf.
+ // "bytes.Buffer" is smart about expanding the capacity and avoids the
+ // exponential runtime of simple append().
+ var smartBuf bytes.Buffer
+ tmp := make([]byte, 10000)
+ for {
+ n, err := syscall.Getdents(fd, tmp)
+ if err != nil {
+ return nil, err
+ }
+ if n == 0 {
+ break
+ }
+ smartBuf.Write(tmp[:n])
+ }
+ // Make sure we have at least Sizeof(Dirent) of zeros after the last
+ // entry. This prevents a cast to Dirent from reading past the buffer.
+ smartBuf.Grow(sizeofDirent)
+ buf := smartBuf.Bytes()
+ // Count the number of directory entries in the buffer so we can allocate
+ // a fuse.DirEntry slice of the correct size at once.
+ var numEntries, offset int
+ for offset < len(buf) {
+ s := *(*syscall.Dirent)(unsafe.Pointer(&buf[offset]))
+ if s.Reclen == 0 {
+ tlog.Warn.Printf("Getdents: corrupt entry #%d: Reclen=0 at offset=%d. Returning EBADR",
+ numEntries, offset)
+ // EBADR = Invalid request descriptor
+ return nil, syscall.EBADR
+ }
+ if int(s.Reclen) > sizeofDirent {
+ tlog.Warn.Printf("Getdents: corrupt entry #%d: Reclen=%d > %d. Returning EBADR",
+ numEntries, sizeofDirent, s.Reclen)
+ return nil, syscall.EBADR
+ }
+ offset += int(s.Reclen)
+ numEntries++
+ }
+ // Parse the buffer into entries
+ entries := make([]fuse.DirEntry, 0, numEntries)
+ offset = 0
+ for offset < len(buf) {
+ s := *(*syscall.Dirent)(unsafe.Pointer(&buf[offset]))
+ name, err := getdentsName(s)
+ if err != nil {
+ return nil, err
+ }
+ offset += int(s.Reclen)
+ if name == "." || name == ".." {
+ // os.File.Readdir() drops "." and "..". Let's be compatible.
+ continue
+ }
+ mode, err := convertDType(s.Type, dir+"/"+name)
+ if err != nil {
+ // The file may have been deleted in the meantime. Just skip it
+ // and go on.
+ continue
+ }
+ entries = append(entries, fuse.DirEntry{
+ Ino: s.Ino,
+ Mode: mode,
+ Name: name,
+ })
+ }
+ return entries, nil
+}
+
+// getdentsName extracts the filename from a Dirent struct and returns it as
+// a Go string.
+func getdentsName(s syscall.Dirent) (string, error) {
+ // After the loop, l contains the index of the first '\0'.
+ l := 0
+ for l = range s.Name {
+ if s.Name[l] == 0 {
+ break
+ }
+ }
+ if l < 1 {
+ tlog.Warn.Printf("Getdents: invalid name length l=%d. Returning EBADR", l)
+ // EBADR = Invalid request descriptor
+ return "", syscall.EBADR
+ }
+ // Copy to byte slice.
+ name := make([]byte, l)
+ for i := range name {
+ name[i] = byte(s.Name[i])
+ }
+ return string(name), nil
+}
+
+// convertDType converts a Dirent.Type to at Stat_t.Mode value.
+func convertDType(dtype uint8, file string) (uint32, error) {
+ if dtype != syscall.DT_UNKNOWN {
+ // Shift up by four octal digits = 12 bits
+ return uint32(dtype) << 12, nil
+ }
+ // DT_UNKNOWN: we have to call Lstat()
+ var st syscall.Stat_t
+ err := syscall.Lstat(file, &st)
+ if err != nil {
+ return 0, err
+ }
+ // The S_IFMT bit mask extracts the file type from the mode.
+ return st.Mode & syscall.S_IFMT, nil
+}
diff --git a/internal/syscallcompat/getdents_other.go b/internal/syscallcompat/getdents_other.go
new file mode 100644
index 0000000..4ef5b8f
--- /dev/null
+++ b/internal/syscallcompat/getdents_other.go
@@ -0,0 +1,17 @@
+// +build !linux
+
+package syscallcompat
+
+import (
+ "log"
+
+ "github.com/hanwen/go-fuse/fuse"
+)
+
+// HaveGetdents is true if we have a working implementation of Getdents
+const HaveGetdents = false
+
+func Getdents(dir string) ([]fuse.DirEntry, error) {
+ log.Panic("only implemented on Linux")
+ return nil, nil
+}
diff --git a/internal/syscallcompat/getdents_test.go b/internal/syscallcompat/getdents_test.go
new file mode 100644
index 0000000..f0b1e60
--- /dev/null
+++ b/internal/syscallcompat/getdents_test.go
@@ -0,0 +1,77 @@
+// +build linux
+
+package syscallcompat
+
+import (
+ "io/ioutil"
+ "os"
+ "strings"
+ "syscall"
+ "testing"
+
+ "github.com/hanwen/go-fuse/fuse"
+)
+
+func TestGetdents(t *testing.T) {
+ // Fill a directory with filenames of length 1 ... 255
+ testDir, err := ioutil.TempDir("", "TestGetdents")
+ if err != nil {
+ t.Fatal(err)
+ }
+ for i := 1; i <= syscall.NAME_MAX; i++ {
+ n := strings.Repeat("x", i)
+ err = ioutil.WriteFile(testDir+"/"+n, nil, 0600)
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+ // "/", "/dev" and "/proc" are good test cases because they contain many
+ // different file types (block and char devices, symlinks, mountpoints)
+ dirs := []string{testDir, "/", "/dev", "/proc"}
+ for _, dir := range dirs {
+ // Read directory using stdlib Readdir()
+ fd, err := os.Open(dir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ readdirEntries, err := fd.Readdir(0)
+ if err != nil {
+ t.Fatal(err)
+ }
+ fd.Close()
+ readdirMap := make(map[string]*syscall.Stat_t)
+ for _, v := range readdirEntries {
+ readdirMap[v.Name()] = fuse.ToStatT(v)
+ }
+ // Read using our Getdents()
+ getdentsEntries, err := Getdents(dir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ getdentsMap := make(map[string]fuse.DirEntry)
+ for _, v := range getdentsEntries {
+ getdentsMap[v.Name] = v
+ }
+ // Compare results
+ if len(getdentsEntries) != len(readdirEntries) {
+ t.Fatalf("len(getdentsEntries)=%d, len(readdirEntries)=%d",
+ len(getdentsEntries), len(readdirEntries))
+ }
+ for name := range readdirMap {
+ g := getdentsMap[name]
+ r := readdirMap[name]
+ rTyp := r.Mode & syscall.S_IFMT
+ if g.Mode != rTyp {
+ t.Errorf("%q: g.Mode=%#o, r.Mode=%#o", name, g.Mode, rTyp)
+ }
+ if g.Ino != r.Ino {
+ // The inode number of a directory that is reported by stat
+ // and getdents is different when it is a mountpoint. Only
+ // throw an error when we are NOT looking at a directory.
+ if g.Mode != syscall.S_IFDIR {
+ t.Errorf("%s: g.Ino=%d, r.Ino=%d", name, g.Ino, r.Ino)
+ }
+ }
+ }
+ }
+}