general cleanup

cmd/hdfs/bash_autocomplete

@@ -9,7 +9,7 @@ _hdfs_complete()
     opts=$(${COMP_WORDS[0]} complete "$words")
     COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
 
-    if [[ $COMPREPLY != */ && $COMPREPLY != "" ]]
+    if [[ $COMPREPLY != "" && $COMPREPLY != */ ]]
     then
       COMPREPLY="$COMPREPLY "
     fi

cmd/hdfs/cache.go

@@ -0,0 +1,82 @@
+package main
+
+import (
+	"errors"
+	"github.com/colinmarc/hdfs"
+	"os"
+	"path"
+)
+
+var (
+	cachedClient *hdfs.Client
+
+	statCache    = make(map[string]os.FileInfo)
+	readDirCache = make(map[string][]os.FileInfo)
+)
+
+func getClient(namenode string) (*hdfs.Client, error) {
+	if cachedClient != nil {
+		return cachedClient, nil
+	}
+
+	if namenode == "" {
+		namenode = os.Getenv("HADOOP_NAMENODE")
+	}
+
+	if namenode == "" {
+		return nil, errors.New("Couldn't find a namenode to connect to. You should specify hdfs://<namenode>:<port> in your paths, or set HADOOP_NAMENODE in your environment.")
+	}
+
+	c, err := hdfs.New(namenode)
+	if err != nil {
+		return nil, err
+	}
+
+	cachedClient = c
+	return cachedClient, nil
+}
+
+func stat(client *hdfs.Client, fullPath string) (os.FileInfo, error) {
+	if cachedRes, exists := statCache[fullPath]; exists {
+		return cachedRes, nil
+	}
+
+	res, err := client.Stat(fullPath)
+	if err != nil {
+		return nil, err
+	}
+
+	statCache[fullPath] = res
+	return res, nil
+}
+
+func readDir(client *hdfs.Client, dir string, glob string) ([]os.FileInfo, error) {
+	if cachedRes, exists := readDirCache[dir]; exists {
+		return cachedRes, nil
+	}
+
+	res, err := client.ReadDir(dir)
+	if err != nil {
+		return nil, err
+	}
+
+	readDirCache[dir] = res
+	for _, fi := range res {
+		childPath := path.Join(dir, fi.Name())
+		statCache[childPath] = fi
+	}
+
+	if glob != "" {
+		matched := make([]os.FileInfo, 0, len(res))
+		for _, fi := range res {
+			match, _ := path.Match(glob, fi.Name())
+			if match {
+				matched = append(matched, fi)
+			}
+		}
+
+		return matched, nil
+	} else {
+		return res, nil
+	}
+}

cmd/hdfs/complete.go

@@ -5,7 +5,7 @@ import (
 	"strings"
 )
 
-var	knownCommands = []string{"ls", "rm", "mv"}
+var knownCommands = []string{"ls", "rm", "mv"}
 
 func complete(args []string) []string {
 	if len(args) <= 1 {

cmd/hdfs/main.go

@@ -2,16 +2,12 @@ package main
 
 import (
 	"code.google.com/p/getopt"
-	"errors"
 	"fmt"
-	"github.com/colinmarc/hdfs"
 	"os"
 	"strings"
 )
 
 var (
-	cachedClient *hdfs.Client
-
 	lsOpts = getopt.New()
 	lsa    = lsOpts.Bool('a')
 	lsl    = lsOpts.Bool('l')
@@ -55,28 +51,6 @@ func main() {
 	os.Exit(0)
 }
 
-func getClient(namenode string) (*hdfs.Client, error) {
-	if cachedClient != nil {
-		return cachedClient, nil
-	}
-
-	if namenode == "" {
-		namenode = os.Getenv("HADOOP_NAMENODE")
-	}
-
-	if namenode == "" {
-		return nil, errors.New("Couldn't find a namenode to connect to. You should specify hdfs://<namenode>:<port> in your paths, or set HADOOP_NAMENODE in your environment.")
-	}
-
-	c, err := hdfs.New(namenode)
-	if err != nil {
-		return nil, err
-	}
-
-	cachedClient = c
-	return cachedClient, nil
-}
-
 func printHelp(exit int) {
 	fmt.Fprintln(os.Stderr, usage)
 	os.Exit(exit)

cmd/hdfs/paths.go

@@ -4,7 +4,6 @@ import (
 	"errors"
 	"github.com/colinmarc/hdfs"
 	"net/url"
-	"os"
 	"os/user"
 	"path"
 	"regexp"
@@ -14,9 +13,6 @@ import (
 var (
 	multipleNamenodeUrls = errors.New("Multiple namenode URLs specified")
 	rootPath             = userDir()
-
-	statCache    = make(map[string]os.FileInfo)
-	readDirCache = make(map[string][]os.FileInfo)
 )
 
 func userDir() string {
@@ -28,7 +24,9 @@ func userDir() string {
 	}
 }
 
-// returns absolute paths, and an namenode host, if a single one was found
+// normalizePaths parses the hosts out of HDFS URLs, and turns relative paths
+// into absolute ones (by appending /user/<user>). If multiple HDFS urls with
+// differing hosts are passed in, it returns an error.
 func normalizePaths(paths []string) ([]string, string, error) {
 	namenode := ""
 	cleanPaths := make([]string, 0, len(paths))
@@ -58,25 +56,15 @@ func normalizePaths(paths []string) ([]string, string, error) {
 	return cleanPaths, namenode, nil
 }
 
+// TODO: not really sure checking for a leading \ is the way to test for
+// escapedness.
 func hasGlob(fragment string) bool {
 	match, _ := regexp.MatchString(`[^\\][[*?]`, fragment)
 	return match
 }
 
-func filterByGlob(paths []string, fragment string) []string {
-	res := make([]string, 0, len(paths))
-	for _, p := range paths {
-		_, name := path.Split(p)
-		match, _ := path.Match(fragment, name)
-		if match {
-			res = append(res, p)
-		}
-	}
-
-	return res
-}
-
-// recursively expands globs. this assumes the path is already absolute
+// expandGlobs recursively expands globs in a filepath. It assumes the paths
+// are already cleaned and normalize (ie, absolute).
 func expandGlobs(client *hdfs.Client, p string) ([]string, error) {
 	if !hasGlob(p) {
 		return []string{p}, nil
@@ -126,48 +114,3 @@ func expandPaths(client *hdfs.Client, paths []string) ([]string, error) {
 
 	return res, nil
 }
-
-func stat(client *hdfs.Client, fullPath string) (os.FileInfo, error) {
-	if cachedRes, exists := statCache[fullPath]; exists {
-		return cachedRes, nil
-	}
-
-	res, err := client.Stat(fullPath)
-	if err != nil {
-		return nil, err
-	}
-
-	statCache[fullPath] = res
-	return res, nil
-}
-
-func readDir(client *hdfs.Client, dir string, glob string) ([]os.FileInfo, error) {
-	if cachedRes, exists := readDirCache[dir]; exists {
-		return cachedRes, nil
-	}
-
-	res, err := client.ReadDir(dir)
-	if err != nil {
-		return nil, err
-	}
-
-	readDirCache[dir] = res
-	for _, fi := range res {
-		childPath := path.Join(dir, fi.Name())
-		statCache[childPath] = fi
-	}
-
-	if glob != "" {
-		matched := make([]os.FileInfo, 0, len(res))
-		for _, fi := range res {
-			match, _ := path.Match(glob, fi.Name())
-			if match {
-				matched = append(matched, fi)
-			}
-		}
-
-		return matched, nil
-	} else {
-		return res, nil
-	}
-}