From 4c2c0c274dcb56b1cfe1435d27c8ef708de4518b Mon Sep 17 00:00:00 2001
From: Avinash Sridhar <sridharavinash@github.com>
Date: Wed, 16 May 2018 09:14:22 -0400
Subject: [PATCH 01/37] Add pip support for licensed

This is an initial take on adding pip package support for `licensed`.
---
 lib/licensed/configuration.rb |  3 +-
 lib/licensed/source/python.rb | 63 +++++++++++++++++++++++++++++++++++
 script/source-setup/python    |  8 +++++
 3 files changed, 73 insertions(+), 1 deletion(-)
 create mode 100644 lib/licensed/source/python.rb
 create mode 100755 script/source-setup/python

diff --git a/lib/licensed/configuration.rb b/lib/licensed/configuration.rb
index 202685f..35c4611 100644
--- a/lib/licensed/configuration.rb
+++ b/lib/licensed/configuration.rb
@@ -52,7 +52,8 @@ module Licensed
         Source::Cabal.new(self),
         Source::Go.new(self),
         Source::Manifest.new(self),
-        Source::NPM.new(self)
+        Source::NPM.new(self),
+        Source::Python.new(self)
       ].select(&:enabled?)
     end
 
diff --git a/lib/licensed/source/python.rb b/lib/licensed/source/python.rb
new file mode 100644
index 0000000..58aca15
--- /dev/null
+++ b/lib/licensed/source/python.rb
@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+require "json"
+require "English"
+
+module Licensed
+  module Source
+    class Python
+      def initialize(config)
+        @config = config
+      end
+
+      def type
+        "python"
+      end
+
+      def enabled?
+        @config.enabled?(type) && File.exist?(@config.pwd.join("requirements,txt"))
+      end
+
+      def dependencies
+        packages = parse_requirements_txt
+
+        @dependencies = packages.map do |package_name|
+            package = package_info(package_name)
+            location = File.join(package["Location"],"-" + package["Version"] +".dist-info")
+            Dependency.new(package_dir, {
+              "type"        => type,
+              "name"        => package["Name"],
+              "summary"     => package["Summary"],
+              "homepage"    => package["Home-page"],
+              "search_root" => location,
+              "version"     => package["Version"]
+            })
+        end.compact
+      end
+
+      # Build the list of packages from a ''requirements.txt'
+      # Assumes that the requirements.txt follow the format pkg=1.0.0 or pkg==1.0.0
+      def parse_requirements_txt
+        packages = []
+        File.open(@config.pwd.join("requirements,txt")).each do |line|
+          p = line.split("=")
+          package.push(p[0])
+        end
+        packages
+      end
+
+      def package_info(package_name)
+        info = {}
+        pip_command(package_name).each do |p|
+          k, v = p.split(":")
+          info[k.strip] = info[v.strip]
+        end
+        info
+      end
+
+      def pip_command(*args)
+        Licensed::Shell.execute("pip", "--disable-pip-version-check", "show", *args)
+      end
+
+    end
+  end
+end
diff --git a/script/source-setup/python b/script/source-setup/python
new file mode 100755
index 0000000..bc7acb3
--- /dev/null
+++ b/script/source-setup/python
@@ -0,0 +1,8 @@
+#!/bin/bash
+set -e
+
+if [ -z "$(which pip)" ]; then
+  echo "A local pip installation is required for python development." >&2
+  exit 127
+fi
+pip3
\ No newline at end of file

From 15be89d4bb6789317d189bfb8d1b18f355952b7d Mon Sep 17 00:00:00 2001
From: Avinash Sridhar <sridharavinash@github.com>
Date: Wed, 16 May 2018 14:44:40 -0400
Subject: [PATCH 02/37] Fixed to the get the script working

Also added some changes to get the tests working correctly
---
 .travis.yml                           |  9 +++++++++
 lib/licensed.rb                       |  1 +
 lib/licensed/source/python.rb         | 22 ++++++++++++----------
 script/source-setup/python            | 15 ++++++++++++++-
 test/fixtures/command/python.yml      |  5 +++++
 test/fixtures/python/requirements.txt |  1 +
 6 files changed, 42 insertions(+), 11 deletions(-)
 create mode 100644 test/fixtures/command/python.yml
 create mode 100644 test/fixtures/python/requirements.txt

diff --git a/.travis.yml b/.travis.yml
index 504e72c..e8efa8a 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -51,5 +51,14 @@ matrix:
       script: ./script/test manifest
       env: NAME="manifest"
 
+    # python tests
+    - language: python
+      python:
+        - "2.7"
+        - "3.6"
+      before_script: ./script/source-setup/python
+      script: ./script/test python
+      env: NAME="python"
+
 notifications:
   disable: true
diff --git a/lib/licensed.rb b/lib/licensed.rb
index 80403ed..c1c9420 100644
--- a/lib/licensed.rb
+++ b/lib/licensed.rb
@@ -11,6 +11,7 @@ require "licensed/source/manifest"
 require "licensed/source/npm"
 require "licensed/source/go"
 require "licensed/source/cabal"
+require "licensed/source/python"
 require "licensed/command/cache"
 require "licensed/command/status"
 require "licensed/command/list"
diff --git a/lib/licensed/source/python.rb b/lib/licensed/source/python.rb
index 58aca15..cca9fc7 100644
--- a/lib/licensed/source/python.rb
+++ b/lib/licensed/source/python.rb
@@ -14,7 +14,7 @@ module Licensed
       end
 
       def enabled?
-        @config.enabled?(type) && File.exist?(@config.pwd.join("requirements,txt"))
+        @config.enabled?(type) && File.exist?(@config.pwd.join("requirements.txt"))
       end
 
       def dependencies
@@ -22,8 +22,8 @@ module Licensed
 
         @dependencies = packages.map do |package_name|
             package = package_info(package_name)
-            location = File.join(package["Location"],"-" + package["Version"] +".dist-info")
-            Dependency.new(package_dir, {
+            location = File.join(package["Location"], "-" + package["Version"] + ".dist-info")
+            Dependency.new(location, {
               "type"        => type,
               "name"        => package["Name"],
               "summary"     => package["Summary"],
@@ -38,26 +38,28 @@ module Licensed
       # Assumes that the requirements.txt follow the format pkg=1.0.0 or pkg==1.0.0
       def parse_requirements_txt
         packages = []
-        File.open(@config.pwd.join("requirements,txt")).each do |line|
+        File.open(@config.pwd.join("requirements.txt")).each do |line|
           p = line.split("=")
-          package.push(p[0])
+          packages.push(p[0])
         end
         packages
       end
 
       def package_info(package_name)
         info = {}
-        pip_command(package_name).each do |p|
-          k, v = p.split(":")
-          info[k.strip] = info[v.strip]
+        p_info = pip_command(package_name).split("\n")
+        p_info.each do |p|
+          k, v = p.split(":",2)
+          info[k&.strip] = v&.strip
         end
         info
       end
 
       def pip_command(*args)
-        Licensed::Shell.execute("pip", "--disable-pip-version-check", "show", *args)
+        venv_dir = @config.dig("python", "virtual_env_dir")
+        pip = File.join(venv_dir, "bin", "pip")
+        Licensed::Shell.execute(pip, "--disable-pip-version-check", "show", *args)
       end
-
     end
   end
 end
diff --git a/script/source-setup/python b/script/source-setup/python
index bc7acb3..c331dd0 100755
--- a/script/source-setup/python
+++ b/script/source-setup/python
@@ -5,4 +5,17 @@ if [ -z "$(which pip)" ]; then
   echo "A local pip installation is required for python development." >&2
   exit 127
 fi
-pip3
\ No newline at end of file
+
+if [ -z "$(which virtualenv)" ]; then
+    echo "A local virtualenv installation is required for python development." >&2
+    exit 127
+fi
+
+
+# setup test fixtures
+BASE_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+cd $BASE_PATH/test/fixtures/python
+virtualenv venv
+. venv/bin/activate
+pip install -r requirements.txt
+deactivate
diff --git a/test/fixtures/command/python.yml b/test/fixtures/command/python.yml
new file mode 100644
index 0000000..b7e03f8
--- /dev/null
+++ b/test/fixtures/command/python.yml
@@ -0,0 +1,5 @@
+expected_dependency: Jinja2
+config:
+  source_path: test/fixtures/python
+  python:
+    virtual_env_dir: "venv"
diff --git a/test/fixtures/python/requirements.txt b/test/fixtures/python/requirements.txt
new file mode 100644
index 0000000..a7bcc47
--- /dev/null
+++ b/test/fixtures/python/requirements.txt
@@ -0,0 +1 @@
+Jinja2==2.9.6

From 9727aa118261bcd3c5510dd120740ae6d7cba26a Mon Sep 17 00:00:00 2001
From: Avinash Sridhar <sridharavinash@github.com>
Date: Wed, 16 May 2018 15:45:50 -0400
Subject: [PATCH 03/37] Minor fixes to remove unneeded vars

---
 lib/licensed/source/python.rb | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/lib/licensed/source/python.rb b/lib/licensed/source/python.rb
index cca9fc7..69ccbbc 100644
--- a/lib/licensed/source/python.rb
+++ b/lib/licensed/source/python.rb
@@ -28,10 +28,9 @@ module Licensed
               "name"        => package["Name"],
               "summary"     => package["Summary"],
               "homepage"    => package["Home-page"],
-              "search_root" => location,
               "version"     => package["Version"]
             })
-        end.compact
+        end
       end
 
       # Build the list of packages from a ''requirements.txt'
@@ -49,7 +48,7 @@ module Licensed
         info = {}
         p_info = pip_command(package_name).split("\n")
         p_info.each do |p|
-          k, v = p.split(":",2)
+          k, v = p.split(":", 2)
           info[k&.strip] = v&.strip
         end
         info

From 7867752f92ff4d1fd4b5d7a892b6b65bf317c597 Mon Sep 17 00:00:00 2001
From: Avinash Sridhar <sridharavinash@github.com>
Date: Wed, 16 May 2018 15:54:26 -0400
Subject: [PATCH 04/37] don't use overload 'p' var in ruby!

---
 lib/licensed/source/python.rb | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/licensed/source/python.rb b/lib/licensed/source/python.rb
index 69ccbbc..4751e1a 100644
--- a/lib/licensed/source/python.rb
+++ b/lib/licensed/source/python.rb
@@ -38,8 +38,8 @@ module Licensed
       def parse_requirements_txt
         packages = []
         File.open(@config.pwd.join("requirements.txt")).each do |line|
-          p = line.split("=")
-          packages.push(p[0])
+          p_split = line.split("=")
+          packages.push(p_split[0])
         end
         packages
       end
@@ -47,8 +47,8 @@ module Licensed
       def package_info(package_name)
         info = {}
         p_info = pip_command(package_name).split("\n")
-        p_info.each do |p|
-          k, v = p.split(":", 2)
+        p_info.each do |pkg|
+          k, v = pkg.split(":", 2)
           info[k&.strip] = v&.strip
         end
         info

From 1483fcc4df05950f434d6327a8d2fa0a025cf371 Mon Sep 17 00:00:00 2001
From: Avinash Sridhar <sridharavinash@github.com>
Date: Wed, 16 May 2018 16:10:58 -0400
Subject: [PATCH 05/37] rename to pip

---
 lib/licensed/source/{python.rb => pip.rb}      | 4 ++--
 script/source-setup/{python => pip}            | 0
 test/fixtures/command/{python.yml => pip.yml}  | 0
 test/fixtures/{python => pip}/requirements.txt | 0
 4 files changed, 2 insertions(+), 2 deletions(-)
 rename lib/licensed/source/{python.rb => pip.rb} (95%)
 rename script/source-setup/{python => pip} (100%)
 rename test/fixtures/command/{python.yml => pip.yml} (100%)
 rename test/fixtures/{python => pip}/requirements.txt (100%)

diff --git a/lib/licensed/source/python.rb b/lib/licensed/source/pip.rb
similarity index 95%
rename from lib/licensed/source/python.rb
rename to lib/licensed/source/pip.rb
index 4751e1a..4fc10d8 100644
--- a/lib/licensed/source/python.rb
+++ b/lib/licensed/source/pip.rb
@@ -4,7 +4,7 @@ require "English"
 
 module Licensed
   module Source
-    class Python
+    class Pip
       def initialize(config)
         @config = config
       end
@@ -33,7 +33,7 @@ module Licensed
         end
       end
 
-      # Build the list of packages from a ''requirements.txt'
+      # Build the list of packages from a 'requirements.txt'
       # Assumes that the requirements.txt follow the format pkg=1.0.0 or pkg==1.0.0
       def parse_requirements_txt
         packages = []
diff --git a/script/source-setup/python b/script/source-setup/pip
similarity index 100%
rename from script/source-setup/python
rename to script/source-setup/pip
diff --git a/test/fixtures/command/python.yml b/test/fixtures/command/pip.yml
similarity index 100%
rename from test/fixtures/command/python.yml
rename to test/fixtures/command/pip.yml
diff --git a/test/fixtures/python/requirements.txt b/test/fixtures/pip/requirements.txt
similarity index 100%
rename from test/fixtures/python/requirements.txt
rename to test/fixtures/pip/requirements.txt

From 16e9ab1c9758bd9a0a2739fddf6ec6455025bad6 Mon Sep 17 00:00:00 2001
From: Avinash Sridhar <sridharavinash@github.com>
Date: Wed, 16 May 2018 16:13:11 -0400
Subject: [PATCH 06/37] rename to pip

---
 lib/licensed/configuration.rb | 2 +-
 script/source-setup/pip       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/licensed/configuration.rb b/lib/licensed/configuration.rb
index 35c4611..3d75dd2 100644
--- a/lib/licensed/configuration.rb
+++ b/lib/licensed/configuration.rb
@@ -53,7 +53,7 @@ module Licensed
         Source::Go.new(self),
         Source::Manifest.new(self),
         Source::NPM.new(self),
-        Source::Python.new(self)
+        Source::Pip.new(self)
       ].select(&:enabled?)
     end
 
diff --git a/script/source-setup/pip b/script/source-setup/pip
index c331dd0..c1f4616 100755
--- a/script/source-setup/pip
+++ b/script/source-setup/pip
@@ -14,7 +14,7 @@ fi
 
 # setup test fixtures
 BASE_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
-cd $BASE_PATH/test/fixtures/python
+cd $BASE_PATH/test/fixtures/pip
 virtualenv venv
 . venv/bin/activate
 pip install -r requirements.txt

From 79cb1b13b2bdfb8238b2a79053423f8a91228dee Mon Sep 17 00:00:00 2001
From: Avinash Sridhar <sridharavinash@github.com>
Date: Wed, 16 May 2018 16:14:11 -0400
Subject: [PATCH 07/37] fix import after rename

---
 lib/licensed.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/licensed.rb b/lib/licensed.rb
index c1c9420..55f1140 100644
--- a/lib/licensed.rb
+++ b/lib/licensed.rb
@@ -11,7 +11,7 @@ require "licensed/source/manifest"
 require "licensed/source/npm"
 require "licensed/source/go"
 require "licensed/source/cabal"
-require "licensed/source/python"
+require "licensed/source/pip"
 require "licensed/command/cache"
 require "licensed/command/status"
 require "licensed/command/list"

From f722cbb9a37bda06ed88cdf886947727458b1408 Mon Sep 17 00:00:00 2001
From: Avinash Sridhar <sridharavinash@github.com>
Date: Wed, 16 May 2018 16:16:42 -0400
Subject: [PATCH 08/37] fix source_path in tests after rename

---
 test/fixtures/command/pip.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/fixtures/command/pip.yml b/test/fixtures/command/pip.yml
index b7e03f8..6068055 100644
--- a/test/fixtures/command/pip.yml
+++ b/test/fixtures/command/pip.yml
@@ -1,5 +1,5 @@
 expected_dependency: Jinja2
 config:
-  source_path: test/fixtures/python
+  source_path: test/fixtures/pip
   python:
     virtual_env_dir: "venv"

From c48f5acd369fd85c2eed43be3a370598cc72bc83 Mon Sep 17 00:00:00 2001
From: Avinash Sridhar <sridharavinash@github.com>
Date: Wed, 16 May 2018 16:41:06 -0400
Subject: [PATCH 09/37] Set the right type after the rename

---
 lib/licensed/source/pip.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/licensed/source/pip.rb b/lib/licensed/source/pip.rb
index 4fc10d8..e12ee74 100644
--- a/lib/licensed/source/pip.rb
+++ b/lib/licensed/source/pip.rb
@@ -10,7 +10,7 @@ module Licensed
       end
 
       def type
-        "python"
+        "pip"
       end
 
       def enabled?

From 9026f7cf4528444a428c0ded4484be9b2bd928e5 Mon Sep 17 00:00:00 2001
From: Avinash Sridhar <sridharavinash@github.com>
Date: Wed, 16 May 2018 16:41:53 -0400
Subject: [PATCH 10/37] memoize

---
 lib/licensed/source/pip.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/licensed/source/pip.rb b/lib/licensed/source/pip.rb
index e12ee74..6759981 100644
--- a/lib/licensed/source/pip.rb
+++ b/lib/licensed/source/pip.rb
@@ -18,7 +18,7 @@ module Licensed
       end
 
       def dependencies
-        packages = parse_requirements_txt
+        packages ||= parse_requirements_txt
 
         @dependencies = packages.map do |package_name|
             package = package_info(package_name)

From f9113f06045d40c5e84dc21ccf1b014a6a66f5b6 Mon Sep 17 00:00:00 2001
From: Avinash Sridhar <sridharavinash@github.com>
Date: Wed, 16 May 2018 16:46:06 -0400
Subject: [PATCH 11/37] memoize part duex

---
 lib/licensed/source/pip.rb | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/lib/licensed/source/pip.rb b/lib/licensed/source/pip.rb
index 6759981..1b103eb 100644
--- a/lib/licensed/source/pip.rb
+++ b/lib/licensed/source/pip.rb
@@ -18,9 +18,7 @@ module Licensed
       end
 
       def dependencies
-        packages ||= parse_requirements_txt
-
-        @dependencies = packages.map do |package_name|
+        @dependencies ||= parse_requirements_txt.map do |package_name|
             package = package_info(package_name)
             location = File.join(package["Location"], "-" + package["Version"] + ".dist-info")
             Dependency.new(location, {

From eafd320001b338c7825edf086f1d1b1732fd96be Mon Sep 17 00:00:00 2001
From: Avinash Sridhar <sridharavinash@github.com>
Date: Thu, 17 May 2018 09:17:52 -0400
Subject: [PATCH 12/37] Add all dependencies to test. Use map to generate
 packages.

---
 lib/licensed/source/pip.rb         | 6 ++----
 test/fixtures/command/pip.yml      | 2 +-
 test/fixtures/pip/requirements.txt | 1 +
 3 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/lib/licensed/source/pip.rb b/lib/licensed/source/pip.rb
index 1b103eb..6360ba9 100644
--- a/lib/licensed/source/pip.rb
+++ b/lib/licensed/source/pip.rb
@@ -34,12 +34,10 @@ module Licensed
       # Build the list of packages from a 'requirements.txt'
       # Assumes that the requirements.txt follow the format pkg=1.0.0 or pkg==1.0.0
       def parse_requirements_txt
-        packages = []
-        File.open(@config.pwd.join("requirements.txt")).each do |line|
+        File.open(@config.pwd.join("requirements.txt")).map do |line|
           p_split = line.split("=")
-          packages.push(p_split[0])
+          p_split[0]
         end
-        packages
       end
 
       def package_info(package_name)
diff --git a/test/fixtures/command/pip.yml b/test/fixtures/command/pip.yml
index 6068055..ea7d1e4 100644
--- a/test/fixtures/command/pip.yml
+++ b/test/fixtures/command/pip.yml
@@ -1,4 +1,4 @@
-expected_dependency: Jinja2
+expected_dependency: MarkupSafe
 config:
   source_path: test/fixtures/pip
   python:
diff --git a/test/fixtures/pip/requirements.txt b/test/fixtures/pip/requirements.txt
index a7bcc47..474215f 100644
--- a/test/fixtures/pip/requirements.txt
+++ b/test/fixtures/pip/requirements.txt
@@ -1 +1,2 @@
 Jinja2==2.9.6
+MarkupSafe==1.0

From cc30c6feb598e2c32a91e2fdb00bd83a1684305a Mon Sep 17 00:00:00 2001
From: Avinash Sridhar <sridharavinash@github.com>
Date: Thu, 17 May 2018 09:34:17 -0400
Subject: [PATCH 13/37] Location was missing package name. use each_with_object
 and drop var

---
 lib/licensed/source/pip.rb | 26 ++++++++++++--------------
 1 file changed, 12 insertions(+), 14 deletions(-)

diff --git a/lib/licensed/source/pip.rb b/lib/licensed/source/pip.rb
index 6360ba9..cb611fc 100644
--- a/lib/licensed/source/pip.rb
+++ b/lib/licensed/source/pip.rb
@@ -19,15 +19,15 @@ module Licensed
 
       def dependencies
         @dependencies ||= parse_requirements_txt.map do |package_name|
-            package = package_info(package_name)
-            location = File.join(package["Location"], "-" + package["Version"] + ".dist-info")
-            Dependency.new(location, {
-              "type"        => type,
-              "name"        => package["Name"],
-              "summary"     => package["Summary"],
-              "homepage"    => package["Home-page"],
-              "version"     => package["Version"]
-            })
+          package = package_info(package_name)
+          location = File.join(package["Location"], package["Name"] +  "-" + package["Version"] + ".dist-info")
+          Dependency.new(location, {
+                           "type"        => type,
+                           "name"        => package["Name"],
+                           "summary"     => package["Summary"],
+                           "homepage"    => package["Home-page"],
+                           "version"     => package["Version"]
+                         })
         end
       end
 
@@ -41,13 +41,11 @@ module Licensed
       end
 
       def package_info(package_name)
-        info = {}
         p_info = pip_command(package_name).split("\n")
-        p_info.each do |pkg|
+        p_info.each_with_object(Hash.new(0)) { |pkg, a|
           k, v = pkg.split(":", 2)
-          info[k&.strip] = v&.strip
-        end
-        info
+          a[k&.strip] = v&.strip
+        }
       end
 
       def pip_command(*args)

From fe27df667e637ab8ae69802beaa898840412fd71 Mon Sep 17 00:00:00 2001
From: Avinash Sridhar <sridharavinash@github.com>
Date: Thu, 17 May 2018 09:37:20 -0400
Subject: [PATCH 14/37] skip if key is nil

---
 lib/licensed/source/pip.rb | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/licensed/source/pip.rb b/lib/licensed/source/pip.rb
index cb611fc..452826f 100644
--- a/lib/licensed/source/pip.rb
+++ b/lib/licensed/source/pip.rb
@@ -44,7 +44,8 @@ module Licensed
         p_info = pip_command(package_name).split("\n")
         p_info.each_with_object(Hash.new(0)) { |pkg, a|
           k, v = pkg.split(":", 2)
-          a[k&.strip] = v&.strip
+          next if k.nil? || k.empty?
+          a[k.strip] = v&.strip
         }
       end
 

From 0b01e5187ca1e5058a68665324ae8cc2b97c9088 Mon Sep 17 00:00:00 2001
From: Avinash Sridhar <sridharavinash@github.com>
Date: Thu, 17 May 2018 09:40:04 -0400
Subject: [PATCH 15/37] use lines moar ruby-er

---
 lib/licensed/source/pip.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/licensed/source/pip.rb b/lib/licensed/source/pip.rb
index 452826f..67cfdb7 100644
--- a/lib/licensed/source/pip.rb
+++ b/lib/licensed/source/pip.rb
@@ -41,7 +41,7 @@ module Licensed
       end
 
       def package_info(package_name)
-        p_info = pip_command(package_name).split("\n")
+        p_info = pip_command(package_name).lines
         p_info.each_with_object(Hash.new(0)) { |pkg, a|
           k, v = pkg.split(":", 2)
           next if k.nil? || k.empty?

From 50908563be20035f9a31fe1d5bb999877f815cd4 Mon Sep 17 00:00:00 2001
From: Avinash Sridhar <sridharavinash@github.com>
Date: Thu, 17 May 2018 10:03:07 -0400
Subject: [PATCH 16/37] fix travis builds

---
 .travis.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index e8efa8a..2b8de65 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -56,9 +56,9 @@ matrix:
       python:
         - "2.7"
         - "3.6"
-      before_script: ./script/source-setup/python
-      script: ./script/test python
-      env: NAME="python"
+      before_script: ./script/source-setup/pip
+      script: ./script/test pip
+      env: NAME="pip"
 
 notifications:
   disable: true

From 8cb29e4b2b301b654587fbdb28e50ba80ef40a3d Mon Sep 17 00:00:00 2001
From: Avinash Sridhar <sridharavinash@github.com>
Date: Sat, 19 May 2018 15:34:33 -0400
Subject: [PATCH 17/37] fix typo

---
 lib/licensed/command/cache.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/licensed/command/cache.rb b/lib/licensed/command/cache.rb
index 00e489d..30e8667 100644
--- a/lib/licensed/command/cache.rb
+++ b/lib/licensed/command/cache.rb
@@ -11,7 +11,7 @@ module Licensed
       def run(force: false)
         summary = @config.apps.flat_map do |app|
           app_name = app["name"]
-          @config.ui.info "Caching licenes for #{app_name}:"
+          @config.ui.info "Caching licenses for #{app_name}:"
 
           # load the app environment
           Dir.chdir app.source_path do

From 4fef6a73dc95ac7067d512a494becd10221b71b2 Mon Sep 17 00:00:00 2001
From: Avinash Sridhar <sridharavinash@github.com>
Date: Tue, 22 May 2018 20:58:34 -0400
Subject: [PATCH 18/37] Fix merge conflicts and update class

---
 lib/licensed/source/pip.rb | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/lib/licensed/source/pip.rb b/lib/licensed/source/pip.rb
index 67cfdb7..97b3233 100644
--- a/lib/licensed/source/pip.rb
+++ b/lib/licensed/source/pip.rb
@@ -5,16 +5,16 @@ require "English"
 module Licensed
   module Source
     class Pip
+      def self.type
+        "pip"
+      end
+
       def initialize(config)
         @config = config
       end
 
-      def type
-        "pip"
-      end
-
       def enabled?
-        @config.enabled?(type) && File.exist?(@config.pwd.join("requirements.txt"))
+        File.exist?(@config.pwd.join("requirements.txt"))
       end
 
       def dependencies
@@ -22,7 +22,7 @@ module Licensed
           package = package_info(package_name)
           location = File.join(package["Location"], package["Name"] +  "-" + package["Version"] + ".dist-info")
           Dependency.new(location, {
-                           "type"        => type,
+                           "type"        => Pip.type,
                            "name"        => package["Name"],
                            "summary"     => package["Summary"],
                            "homepage"    => package["Home-page"],

From 114ceaacb809a5d6b5d3dd5da6febf23d2c369b3 Mon Sep 17 00:00:00 2001
From: Avinash Sridhar <sridharavinash@github.com>
Date: Wed, 23 May 2018 08:46:06 -0400
Subject: [PATCH 19/37] Init tests for pip

---
 test/source/pip_test.rb | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100644 test/source/pip_test.rb

diff --git a/test/source/pip_test.rb b/test/source/pip_test.rb
new file mode 100644
index 0000000..2530248
--- /dev/null
+++ b/test/source/pip_test.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+require "test_helper"
+require "tmpdir"
+
+if Licensed::Shell.tool_available?("pip")
+  describe Licensed::Source::Pip do
+    let (:fixtures)  { File.expand_path("../../fixtures/pip", __FILE__) }
+    let (:config)   { Licensed::Configuration.new("python" => {"virtual_env_dir" => "venv"}) }
+    let (:source)   { Licensed::Source::Pip.new(config) }
+
+    describe "enabled?" do
+      it "is true if pip source is available" do
+        Dir.chdir(fixtures) do
+        assert source.enabled?
+        end
+      end
+    end
+  end
+end

From cd621ddaf894bcc52cb7e4077d6e389f8f500d54 Mon Sep 17 00:00:00 2001
From: Avinash Sridhar <sridharavinash@github.com>
Date: Wed, 23 May 2018 16:39:48 -0400
Subject: [PATCH 20/37] Add a few more tests

---
 lib/licensed/source/pip.rb |  3 +++
 test/source/pip_test.rb    | 19 +++++++++++++++++++
 2 files changed, 22 insertions(+)

diff --git a/lib/licensed/source/pip.rb b/lib/licensed/source/pip.rb
index 97b3233..3d6881e 100644
--- a/lib/licensed/source/pip.rb
+++ b/lib/licensed/source/pip.rb
@@ -51,6 +51,9 @@ module Licensed
 
       def pip_command(*args)
         venv_dir = @config.dig("python", "virtual_env_dir")
+        if venv_dir.nil?
+          raise "Virtual env directory not set."
+        end
         pip = File.join(venv_dir, "bin", "pip")
         Licensed::Shell.execute(pip, "--disable-pip-version-check", "show", *args)
       end
diff --git a/test/source/pip_test.rb b/test/source/pip_test.rb
index 2530248..3fe5236 100644
--- a/test/source/pip_test.rb
+++ b/test/source/pip_test.rb
@@ -14,6 +14,25 @@ if Licensed::Shell.tool_available?("pip")
         assert source.enabled?
         end
       end
+
+      it "is false if go source is not available" do
+        Dir.mktmpdir do |dir|
+          Dir.chdir(dir) do
+            refute source.enabled?
+          end
+        end
+      end
+    end
+
+    describe "config file params check" do
+      it "fails if virtual_env_dir is not set" do
+        config.delete("python")
+        assert_raises RuntimeError  do
+          Dir.chdir(fixtures) do
+            source.pip_command
+          end
+        end
+      end
     end
   end
 end

From 2b1377bac08f1d60fed999ae5c69283ee88384fd Mon Sep 17 00:00:00 2001
From: Avinash Sridhar <sridharavinash@github.com>
Date: Wed, 23 May 2018 16:41:44 -0400
Subject: [PATCH 21/37] copy pasta error

---
 test/source/pip_test.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/source/pip_test.rb b/test/source/pip_test.rb
index 3fe5236..b5c97c7 100644
--- a/test/source/pip_test.rb
+++ b/test/source/pip_test.rb
@@ -15,7 +15,7 @@ if Licensed::Shell.tool_available?("pip")
         end
       end
 
-      it "is false if go source is not available" do
+      it "is false if pip source is not available" do
         Dir.mktmpdir do |dir|
           Dir.chdir(dir) do
             refute source.enabled?

From 223bce865ff49b737f7ec36714293b96bbeb41c9 Mon Sep 17 00:00:00 2001
From: Jon Ruskin <jon.ruskin@gmail.com>
Date: Thu, 24 May 2018 19:00:13 -0700
Subject: [PATCH 22/37] add tests

---
 .gitignore                      |  2 +-
 script/source-setup/cabal       |  2 +-
 test/fixtures/cabal/app.cabal   |  6 ++++++
 test/fixtures/cabal/app/Main.hs |  1 +
 test/source/cabal_test.rb       | 18 ++++++++++++++++++
 5 files changed, 27 insertions(+), 2 deletions(-)
 create mode 100644 test/fixtures/cabal/app/Main.hs

diff --git a/.gitignore b/.gitignore
index fc41947..ae9bfab 100644
--- a/.gitignore
+++ b/.gitignore
@@ -20,7 +20,7 @@ test/fixtures/go/src/*
 test/fixtures/go/pkg
 !test/fixtures/go/src/test
 test/fixtures/cabal/*
-!test/fixtures/cabal/app.cabal
+!test/fixtures/cabal/app*
 
 vendor/licenses
 .licenses
diff --git a/script/source-setup/cabal b/script/source-setup/cabal
index c5d21f9..0a98b3c 100755
--- a/script/source-setup/cabal
+++ b/script/source-setup/cabal
@@ -11,7 +11,7 @@ BASE_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
 cd $BASE_PATH/test/fixtures/cabal
 
 if [ "$1" == "-f" ]; then
-  find . -not -regex "\.*" -and -not -name "app\.cabal" -print0 | xargs -0 rm -rf
+  find . -not -regex "\.*" -and -not -path "*app*" -print0 | xargs -0 rm -rf
 fi
 
 cabal new-build
diff --git a/test/fixtures/cabal/app.cabal b/test/fixtures/cabal/app.cabal
index 4f602f2..89bdc22 100644
--- a/test/fixtures/cabal/app.cabal
+++ b/test/fixtures/cabal/app.cabal
@@ -11,3 +11,9 @@ library
   hs-source-dirs:      .
   build-depends:       zlib == 0.6.2
   default-language:    Haskell2010
+
+executable app
+  hs-source-dirs:      app
+  main-is:             Main.hs
+  build-depends:       base, Glob == 0.9.2
+  default-language:    Haskell2010
diff --git a/test/fixtures/cabal/app/Main.hs b/test/fixtures/cabal/app/Main.hs
new file mode 100644
index 0000000..91c9ba4
--- /dev/null
+++ b/test/fixtures/cabal/app/Main.hs
@@ -0,0 +1 @@
+main = putStrLn "Hello world"
diff --git a/test/source/cabal_test.rb b/test/source/cabal_test.rb
index 4e56d68..b7da92e 100644
--- a/test/source/cabal_test.rb
+++ b/test/source/cabal_test.rb
@@ -59,6 +59,24 @@ if Licensed::Shell.tool_available?("ghc")
           assert dep["summary"]
         end
       end
+
+      it "finds dependencies for executables" do
+        config["cabal"] = { "ghc_package_db" => ["global", user_db, local_db] }
+        Dir.chdir(fixtures) do
+          dep = source.dependencies.detect { |d| d["name"] == "Glob" }
+          assert dep
+          assert_equal "cabal", dep["type"]
+          assert_equal "0.9.2", dep["version"]
+          assert dep["summary"]
+        end
+      end
+
+      it "does not include the target project" do
+        config["cabal"] = { "ghc_package_db" => ["global", user_db, local_db] }
+        Dir.chdir(fixtures) do
+          refute source.dependencies.detect { |d| d["name"] == "app" }
+        end
+      end
     end
 
     describe "package_db_args" do

From 541745f3fb5e5dba71a98793b0de434410b417a6 Mon Sep 17 00:00:00 2001
From: Jon Ruskin <jon.ruskin@gmail.com>
Date: Fri, 25 May 2018 11:16:07 -0700
Subject: [PATCH 23/37] check for required tools during `#enabled?`

---
 lib/licensed/source/bundler.rb | 6 ++++--
 lib/licensed/source/go.rb      | 2 +-
 lib/licensed/source/npm.rb     | 2 +-
 3 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/lib/licensed/source/bundler.rb b/lib/licensed/source/bundler.rb
index b789334..354349b 100644
--- a/lib/licensed/source/bundler.rb
+++ b/lib/licensed/source/bundler.rb
@@ -1,5 +1,7 @@
 # frozen_string_literal: true
-require "bundler"
+if Licensed::Shell.tool_available?("bundle")
+  require "bundler"
+end
 
 module Licensed
   module Source
@@ -15,7 +17,7 @@ module Licensed
       end
 
       def enabled?
-        lockfile_path && lockfile_path.exist?
+        Licensed::Shell.tool_available?("bundle") && lockfile_path && lockfile_path.exist?
       end
 
       def dependencies
diff --git a/lib/licensed/source/go.rb b/lib/licensed/source/go.rb
index d66a254..b40c44b 100644
--- a/lib/licensed/source/go.rb
+++ b/lib/licensed/source/go.rb
@@ -14,7 +14,7 @@ module Licensed
       end
 
       def enabled?
-        go_source?
+        Licensed::Shell.tool_available?("go") && go_source?
       end
 
       def dependencies
diff --git a/lib/licensed/source/npm.rb b/lib/licensed/source/npm.rb
index e680257..d02ce8c 100644
--- a/lib/licensed/source/npm.rb
+++ b/lib/licensed/source/npm.rb
@@ -13,7 +13,7 @@ module Licensed
       end
 
       def enabled?
-        File.exist?(@config.pwd.join("package.json"))
+        Licensed::Shell.tool_available?("npm") && File.exist?(@config.pwd.join("package.json"))
       end
 
       def dependencies

From 45800906d496f7a941fc90ddc39235b9866e2bd8 Mon Sep 17 00:00:00 2001
From: Jon Ruskin <jon.ruskin@gmail.com>
Date: Mon, 28 May 2018 10:54:47 -0700
Subject: [PATCH 24/37] bundler checks for loaded lib

more closely match how we use bundler in the source
---
 lib/licensed/source/bundler.rb | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/licensed/source/bundler.rb b/lib/licensed/source/bundler.rb
index 354349b..7e6e416 100644
--- a/lib/licensed/source/bundler.rb
+++ b/lib/licensed/source/bundler.rb
@@ -1,6 +1,7 @@
 # frozen_string_literal: true
-if Licensed::Shell.tool_available?("bundle")
+begin
   require "bundler"
+rescue LoadError
 end
 
 module Licensed
@@ -17,7 +18,7 @@ module Licensed
       end
 
       def enabled?
-        Licensed::Shell.tool_available?("bundle") && lockfile_path && lockfile_path.exist?
+        defined?(::Bundler) && lockfile_path && lockfile_path.exist?
       end
 
       def dependencies

From 436911d113bd514e004052b835d8a44e019747f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mislav=20Marohni=C4=87?= <mislav.marohnic@gmail.com>
Date: Tue, 29 May 2018 11:36:40 +0200
Subject: [PATCH 25/37] Raise informative error from `Licensed::Shell.execute`
 on failures

A call point can opt-in to tolerate failures by passing `allow_failure: true`.
---
 lib/licensed/git.rb          |  2 +-
 lib/licensed/shell.rb        | 45 +++++++++++++++++++++++++++++++-----
 lib/licensed/source/cabal.rb |  2 +-
 lib/licensed/source/go.rb    | 10 ++++++--
 test/source/npm_test.rb      | 13 +++++++++++
 5 files changed, 62 insertions(+), 10 deletions(-)

diff --git a/lib/licensed/git.rb b/lib/licensed/git.rb
index 9da12b3..c29012b 100644
--- a/lib/licensed/git.rb
+++ b/lib/licensed/git.rb
@@ -23,7 +23,7 @@ module Licensed
         file = File.directory?(descriptor) ? "." : File.basename(descriptor)
 
         Dir.chdir dir do
-          Licensed::Shell.execute("git", "rev-list", "-1", "HEAD", "--", file)
+          Licensed::Shell.execute("git", "rev-list", "-1", "HEAD", "--", file, allow_failure: true)
         end
       end
 
diff --git a/lib/licensed/shell.rb b/lib/licensed/shell.rb
index 8bcbb0c..c15917b 100644
--- a/lib/licensed/shell.rb
+++ b/lib/licensed/shell.rb
@@ -3,12 +3,19 @@ require "open3"
 
 module Licensed
   module Shell
-    # Executes a command, returning it's STDOUT on success.  Returns an empty
-    # string on failure
-    def self.execute(cmd, *args)
-      output, _, status = Open3.capture3(cmd, *args)
-      return "" unless status.success?
-      output.strip
+    # Executes a command, returning its standard output on success. On failure,
+    # it raises an exception that contains the error output, unless
+    # `allow_failure` is true, in which case it returns an empty string.
+    def self.execute(cmd, *args, allow_failure: false)
+      stdout, stderr, status = Open3.capture3(cmd, *args)
+
+      if status.success?
+        stdout.strip
+      elsif allow_failure
+        ""
+      else
+        raise Error.new([cmd, *args], status.exitstatus, stderr)
+      end
     end
 
     # Executes a command and returns a boolean value indicating if the command
@@ -24,5 +31,31 @@ module Licensed
       output, err, status = Open3.capture3("which", tool)
       status.success? && !output.strip.empty? && err.strip.empty?
     end
+
+    class Error < RuntimeError
+      def initialize(cmd, status, stderr)
+        super()
+        @cmd = cmd
+        @exitstatus = status
+        @output = stderr
+      end
+
+      def message
+        output = @output.to_s.strip
+        extra = output.empty?? "" : "\n#{output.gsub(/^/, "    ")}"
+        "command exited with status #{@exitstatus}\n  #{escape_cmd}#{extra}"
+      end
+
+      def escape_cmd
+        @cmd.map do |arg|
+          if arg =~ /[\s'"]/
+            escaped = arg.gsub(/([\\"])/, '\\\\\1')
+            %("#{escaped}")
+          else
+            arg
+          end
+        end.join(" ")
+      end
+    end
   end
 end
diff --git a/lib/licensed/source/cabal.rb b/lib/licensed/source/cabal.rb
index b863c7a..5badabb 100644
--- a/lib/licensed/source/cabal.rb
+++ b/lib/licensed/source/cabal.rb
@@ -125,7 +125,7 @@ module Licensed
       # Runs a `ghc-pkg field` command for a given set of fields and arguments
       # Automatically includes ghc package DB locations in the command
       def ghc_pkg_field_command(id, fields, *args)
-        Licensed::Shell.execute("ghc-pkg", "field", id, fields.join(","), *args, *package_db_args)
+        Licensed::Shell.execute("ghc-pkg", "field", id, fields.join(","), *args, *package_db_args, allow_failure: true)
       end
 
       # Returns an array of ghc package DB locations as specified in the app
diff --git a/lib/licensed/source/go.rb b/lib/licensed/source/go.rb
index d66a254..684e461 100644
--- a/lib/licensed/source/go.rb
+++ b/lib/licensed/source/go.rb
@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 require "json"
 require "English"
+require "pathname"
 
 module Licensed
   module Source
@@ -113,7 +114,7 @@ module Licensed
       # package - Go package import path
       def package_info_command(package)
         package ||= ""
-        Licensed::Shell.execute("go", "list", "-json", package)
+        Licensed::Shell.execute("go", "list", "-json", package, allow_failure: true)
       end
 
       # Returns the info for the package under test
@@ -140,7 +141,12 @@ module Licensed
         @gopath = if path.nil? || path.empty?
                     ENV["GOPATH"]
                   else
-                    File.expand_path(path, Licensed::Git.repository_root)
+                    root = begin
+                             Licensed::Git.repository_root
+                           rescue Licensed::Shell::Error
+                             Pathname.pwd
+                           end
+                    File.expand_path(path, root)
                   end
       end
 
diff --git a/test/source/npm_test.rb b/test/source/npm_test.rb
index 4f1d2d1..022b1fa 100644
--- a/test/source/npm_test.rb
+++ b/test/source/npm_test.rb
@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 require "test_helper"
 require "tmpdir"
+require "fileutils"
 
 if Licensed::Shell.tool_available?("npm")
   describe Licensed::Source::NPM do
@@ -52,6 +53,18 @@ if Licensed::Shell.tool_available?("npm")
           refute @source.dependencies.detect { |dep| dep["name"] == "string.prototype.startswith" }
         end
       end
+
+      it "raises when dependencies are missing" do
+        Dir.mktmpdir do |dir|
+          FileUtils.cp(File.join(fixtures, "package.json"), File.join(dir, "package.json"))
+          Dir.chdir(dir) do
+            error = assert_raises(Licensed::Shell::Error) { @source.dependencies }
+            assert_includes error.message, "command exited with status 1"
+            assert_includes error.message, "npm list --parseable --production --long"
+            assert_includes error.message, "npm ERR! missing: autoprefixer@"
+          end
+        end
+      end
     end
   end
 end

From 6aacbe0156dcdd7022b50aaa08cac137073ed1bf Mon Sep 17 00:00:00 2001
From: Jon Ruskin <jon.ruskin@gmail.com>
Date: Tue, 29 May 2018 12:26:18 -0700
Subject: [PATCH 26/37] source update

---
 lib/licensed/source/cabal.rb | 59 +++++++++++++++++++++++++++++++-----
 1 file changed, 52 insertions(+), 7 deletions(-)

diff --git a/lib/licensed/source/cabal.rb b/lib/licensed/source/cabal.rb
index b863c7a..24ff4c1 100644
--- a/lib/licensed/source/cabal.rb
+++ b/lib/licensed/source/cabal.rb
@@ -4,6 +4,9 @@ require "English"
 module Licensed
   module Source
     class Cabal
+      DEPENDENCY_REGEX = /\s*.+?\s*/.freeze
+      DEFAULT_TARGETS = %w{executable library}.freeze
+
       def self.type
         "cabal"
       end
@@ -63,8 +66,7 @@ module Licensed
 
       # Returns a `Set` of the package ids for all cabal dependencies
       def package_ids
-        deps = cabal_packages.flat_map { |n| package_dependencies(n, false) }
-        recursive_dependencies(deps)
+        recursive_dependencies(cabal_packages)
       end
 
       # Recursively finds the dependencies for each cabal package.
@@ -148,12 +150,55 @@ module Licensed
         path.gsub("<ghc_version>", ghc_version)
       end
 
-      # Return an array of the top-level cabal packages for the current app
       def cabal_packages
-        cabal_files.map do |f|
-          name_match = File.read(f).match(/^name:\s*(.*)$/)
-          name_match[1] if name_match
-        end.compact
+        cabal_files.each_with_object(Set.new) do |cabal_file, packages|
+          content = File.read(cabal_file)
+          next if content.nil? || content.empty?
+
+          # add any dependencies for explicitly matched content from
+          # the cabal file.  by default this will find executable's dependencies
+          content.scan(cabal_file_regex).each do |match|
+            # match[1] is a string of "," separated dependencies
+            dependencies = match[1].split(",").map(&:strip)
+            dependencies.each do |dep|
+              # the dependency might have a version specifier.
+              # remove it so we can get the full id specifier for each package
+              id = cabal_package_id(dep.split(/\s/)[0])
+              packages.add(id) if id
+            end
+          end
+        end
+      end
+
+      # Returns an installed package id for the package.
+      def cabal_package_id(package_name)
+        field = ghc_pkg_field_command(package_name, ["id"])
+        id = field.split(":", 2)[1]
+        id.strip if id
+      end
+
+      # Find `build-depends` lists from specified targets in a cabal file
+      def cabal_file_regex
+        # this will match 0 or more occurences of
+        # match[0] - specifier, e.g. executable, library, etc
+        # match[1] - full list of matched dependencies
+        # match[2] - first matched dependency (required)
+        # match[3] - remainder of matched dependencies (not required)
+        @cabal_file_regex ||= /
+          # match a specifier, e.g. library or executable
+          ^(#{cabal_file_targets.join("|")})
+            .*? # stuff
+
+            # match a list of 1 or more dependencies
+            build-depends:(#{DEPENDENCY_REGEX}(,#{DEPENDENCY_REGEX})*)\n
+        /xmi
+      end
+
+      # Returns the targets to search for `build-depends` in a cabal file
+      def cabal_file_targets
+        targets = Array(@config.dig("cabal", "cabal_file_targets"))
+        targets.push(*DEFAULT_TARGETS) if targets.empty?
+        targets
       end
 
       # Returns an array of the local directory cabal package files

From 0bbad650cd59f8139d80a8e62790229f54587113 Mon Sep 17 00:00:00 2001
From: Jon Ruskin <jon.ruskin@gmail.com>
Date: Tue, 29 May 2018 12:27:02 -0700
Subject: [PATCH 27/37] docs updates

---
 docs/sources/cabal.md | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/docs/sources/cabal.md b/docs/sources/cabal.md
index 13a4fbc..1d5a9e6 100644
--- a/docs/sources/cabal.md
+++ b/docs/sources/cabal.md
@@ -2,7 +2,25 @@
 
 The cabal source uses the `ghc-pkg` command to enumerate dependencies and provide metadata.  It is un-opinionated on GHC packagedb locations and requires some configuration to ensure that all packages are properly found.
 
-The cabal source will detect dependencies when a `.cabal` file is found at an apps `source_path`.
+The cabal source will detect dependencies when a `.cabal` file is found at an apps `source_path`.  By default, the cabal source will enumerate dependencies for all executable and library targets in a cabal file.
+
+### Specifying which cabal file targets should enumerate dependencies
+The cabal source can be configured to override which cabal file targets contain dependencies that need to be documented.
+
+The default configuration is equivalent to:
+```yml
+cabal:
+  cabal_file_targets:
+    - executable
+    - library
+```
+
+However if you only wanted to enumerate dependencies for a `my_cabal_exe` executable target, you could specify:
+```yml
+cabal:
+  cabal_file_targets:
+    - executable my_cabal_exe
+```
 
 ### Specifying GHC packagedb locations through environment
 You can configure the `cabal` source to use specific packagedb locations by setting the `GHC_PACKAGE_PATH` environment variable before running `licensed`.

From 6134251213113c185bb73d21b8dcfe5b133a8c1c Mon Sep 17 00:00:00 2001
From: Jon Ruskin <jon.ruskin@gmail.com>
Date: Tue, 29 May 2018 14:14:12 -0700
Subject: [PATCH 28/37] update comments and method name

---
 lib/licensed/source/cabal.rb | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/lib/licensed/source/cabal.rb b/lib/licensed/source/cabal.rb
index 24ff4c1..f424f93 100644
--- a/lib/licensed/source/cabal.rb
+++ b/lib/licensed/source/cabal.rb
@@ -16,7 +16,7 @@ module Licensed
       end
 
       def enabled?
-        cabal_packages.any? && ghc?
+        cabal_file_dependencies.any? && ghc?
       end
 
       def dependencies
@@ -66,7 +66,7 @@ module Licensed
 
       # Returns a `Set` of the package ids for all cabal dependencies
       def package_ids
-        recursive_dependencies(cabal_packages)
+        recursive_dependencies(cabal_file_dependencies)
       end
 
       # Recursively finds the dependencies for each cabal package.
@@ -150,13 +150,14 @@ module Licensed
         path.gsub("<ghc_version>", ghc_version)
       end
 
-      def cabal_packages
+      # Returns a set containing the top-level dependencies found in cabal files
+      def cabal_file_dependencies
         cabal_files.each_with_object(Set.new) do |cabal_file, packages|
           content = File.read(cabal_file)
           next if content.nil? || content.empty?
 
-          # add any dependencies for explicitly matched content from
-          # the cabal file.  by default this will find executable's dependencies
+          # add any dependencies for matched targets from the cabal file.
+          # by default this will find executable and library dependencies
           content.scan(cabal_file_regex).each do |match|
             # match[1] is a string of "," separated dependencies
             dependencies = match[1].split(",").map(&:strip)

From 048d824042224c76284628d333edbed8ae1d95bf Mon Sep 17 00:00:00 2001
From: Jon Ruskin <jon.ruskin@gmail.com>
Date: Wed, 30 May 2018 12:33:01 -0700
Subject: [PATCH 29/37] check that cached license version is valid

---
 lib/licensed/command/cache.rb |  5 ++--
 test/command/cache_test.rb    | 44 +++++++++++++++++++++++++++++++++++
 test/test_helper.rb           | 16 +++++++------
 3 files changed, 56 insertions(+), 9 deletions(-)

diff --git a/lib/licensed/command/cache.rb b/lib/licensed/command/cache.rb
index edafcfc..5339f58 100644
--- a/lib/licensed/command/cache.rb
+++ b/lib/licensed/command/cache.rb
@@ -40,8 +40,9 @@ module Licensed
                 # or default to a blank license
                 license = Licensed::License.read(filename) || Licensed::License.new
 
-                # Version did not change, no need to re-cache
-                if !force && version == license["version"]
+                # cached version string exists and did not change, no need to re-cache
+                has_version = !license["version"].nil? && !license["version"].empty?
+                if !force && has_version && version == license["version"]
                   @config.ui.info "    Using #{name} (#{version})"
                   next
                 end
diff --git a/test/command/cache_test.rb b/test/command/cache_test.rb
index e668f9a..648ce39 100644
--- a/test/command/cache_test.rb
+++ b/test/command/cache_test.rb
@@ -72,6 +72,50 @@ describe Licensed::Command::Cache do
     refute_equal "0.0", license["version"]
   end
 
+  it "does not reuse nil license version" do
+    generator.run
+
+    path = config.cache_path.join("test/dependency.txt")
+    license = Licensed::License.read(path)
+    license["license"] = "test"
+    license.save(path)
+
+    test_dependency = Licensed::Dependency.new(Dir.pwd, {
+      "type"     => TestSource.type,
+      "name"     => "dependency"
+    })
+    TestSource.stub(:create_dependency, test_dependency) do
+      generator.run
+    end
+
+    license = Licensed::License.read(path)
+    assert_equal "test", license["license"]
+    assert_equal "1.0", license["version"]
+  end
+
+  it "does not reuse empty license version" do
+    generator.run
+
+    path = config.cache_path.join("test/dependency.txt")
+    license = Licensed::License.read(path)
+    license["license"] = "test"
+    license["version"] = ""
+    license.save(path)
+
+    test_dependency = Licensed::Dependency.new(Dir.pwd, {
+      "type"     => TestSource.type,
+      "name"     => "dependency",
+      "version"  => ""
+    })
+    TestSource.stub(:create_dependency, test_dependency) do
+      generator.run
+    end
+
+    license = Licensed::License.read(path)
+    assert_equal "test", license["license"]
+    assert_equal "1.0", license["version"]
+  end
+
   it "does not include ignored dependencies in dependency counts" do
     config.ui.level = "info"
     out, _ = capture_io { generator.run }
diff --git a/test/test_helper.rb b/test/test_helper.rb
index d487b2c..b6b0cec 100644
--- a/test/test_helper.rb
+++ b/test/test_helper.rb
@@ -36,13 +36,15 @@ class TestSource
 
   def dependencies
     @dependencies_hook.call if @dependencies_hook.respond_to?(:call)
-    @dependencies ||= [
-      Licensed::Dependency.new(Dir.pwd, {
-        "type"     => TestSource.type,
-        "name"     => "dependency",
-        "version"  => "1.0"
-      })
-    ]
+    @dependencies ||= [TestSource.create_dependency]
+  end
+
+  def self.create_dependency
+    Licensed::Dependency.new(Dir.pwd, {
+      "type"     => TestSource.type,
+      "name"     => "dependency",
+      "version"  => "1.0"
+    })
   end
 end
 

From b1688096265bfcc1b8c245da545929d087623889 Mon Sep 17 00:00:00 2001
From: Colin Seymour <colin@github.com>
Date: Thu, 31 May 2018 09:21:54 +0100
Subject: [PATCH 30/37] Fix licenses typo

---
 lib/licensed/command/cache.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/licensed/command/cache.rb b/lib/licensed/command/cache.rb
index edafcfc..e3a0b3d 100644
--- a/lib/licensed/command/cache.rb
+++ b/lib/licensed/command/cache.rb
@@ -11,7 +11,7 @@ module Licensed
       def run(force: false)
         summary = @config.apps.flat_map do |app|
           app_name = app["name"]
-          @config.ui.info "Caching licenes for #{app_name}:"
+          @config.ui.info "Caching licenses for #{app_name}:"
 
           # load the app environment
           Dir.chdir app.source_path do

From fe0ac9262cff82e44abf5040df24139aef2a7d2d Mon Sep 17 00:00:00 2001
From: Avinash Sridhar <sridharavinash@github.com>
Date: Thu, 31 May 2018 11:02:19 -0400
Subject: [PATCH 31/37] Add more tests, update documentation for the pip
 support

---
 README.md               |  1 +
 docs/sources/pip.md     | 11 +++++++++++
 test/source/pip_test.rb | 22 ++++++++++++++++++++++
 3 files changed, 34 insertions(+)
 create mode 100644 docs/sources/pip.md

diff --git a/README.md b/README.md
index 8f00f9a..b033549 100644
--- a/README.md
+++ b/README.md
@@ -79,6 +79,7 @@ Dependencies will be automatically detected for
 4. [Go](./docs/sources/go.md)
 5. [Manifest lists](./docs/sources/manifests.md)
 6. [NPM](./docs/sources/npm.md)
+7. [Pip](./docs/source/pip.md)
 
 You can disable any of them in the configuration file:
 
diff --git a/docs/sources/pip.md b/docs/sources/pip.md
new file mode 100644
index 0000000..5605512
--- /dev/null
+++ b/docs/sources/pip.md
@@ -0,0 +1,11 @@
+# Pip
+
+The pip source uses `pip` CLI commands to enumerate dependencies and properties. It is expected that `pip` is available in the PATH before running `licensed`
+
+#### virtual_env_dir
+A `virtualenv` is assumed to be setup before running `licensed`. The `pip` command will be sourced from this directory.
+An example usage of this might look like:
+```yaml
+python:
+    virtual_env_dir:"venv"
+```
diff --git a/test/source/pip_test.rb b/test/source/pip_test.rb
index b5c97c7..00bf6b9 100644
--- a/test/source/pip_test.rb
+++ b/test/source/pip_test.rb
@@ -34,5 +34,27 @@ if Licensed::Shell.tool_available?("pip")
         end
       end
     end
+
+    describe "dependencies" do
+      it "includes direct dependencies" do
+        Dir.chdir fixtures do
+          dep = source.dependencies.detect { |d| d["name"] == "Jinja2" }
+          assert dep
+          assert_equal "pip", dep["type"]
+          assert dep["homepage"]
+          assert dep["summary"]
+        end
+      end
+
+      it "includes indirect dependencies" do
+        Dir.chdir fixtures do
+          dep = source.dependencies.detect { |d| d["name"] == "MarkupSafe" }
+          assert dep
+          assert_equal "pip", dep["type"]
+          assert dep["homepage"]
+          assert dep["summary"]
+        end
+      end
+    end
   end
 end

From 8a8e2ed6bfc3bc83e70ab5252d34a73c8ddbda66 Mon Sep 17 00:00:00 2001
From: Avinash Sridhar <sridharavinash@github.com>
Date: Thu, 31 May 2018 15:44:23 -0400
Subject: [PATCH 32/37] Use repository_root and support -f flag for fixture
 setup

---
 lib/licensed/source/pip.rb    |  1 +
 script/source-setup/pip       | 16 ++++++++++++----
 test/fixtures/command/pip.yml |  2 +-
 test/source/pip_test.rb       |  3 ++-
 4 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/lib/licensed/source/pip.rb b/lib/licensed/source/pip.rb
index 3d6881e..0ceac3d 100644
--- a/lib/licensed/source/pip.rb
+++ b/lib/licensed/source/pip.rb
@@ -54,6 +54,7 @@ module Licensed
         if venv_dir.nil?
           raise "Virtual env directory not set."
         end
+        venv_dir = File.expand_path(venv_dir, Licensed::Git.repository_root)
         pip = File.join(venv_dir, "bin", "pip")
         Licensed::Shell.execute(pip, "--disable-pip-version-check", "show", *args)
       end
diff --git a/script/source-setup/pip b/script/source-setup/pip
index c1f4616..c720d12 100755
--- a/script/source-setup/pip
+++ b/script/source-setup/pip
@@ -14,8 +14,16 @@ fi
 
 # setup test fixtures
 BASE_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
-cd $BASE_PATH/test/fixtures/pip
-virtualenv venv
-. venv/bin/activate
-pip install -r requirements.txt
+
+
+# clean up any previous fixture venv that might have been created.
+if [ "$1" == "-f" ]; then
+    echo "removing old fixture setup..."
+    rm -rf  $BASE_PATH/test/fixtures/pip/venv
+fi
+
+# set up a virtualenv and install the packages in the test requirements
+virtualenv $BASE_PATH/test/fixtures/pip/venv
+. $BASE_PATH/test/fixtures/pip/venv/bin/activate
+pip install -r $BASE_PATH/test/fixtures/pip/requirements.txt
 deactivate
diff --git a/test/fixtures/command/pip.yml b/test/fixtures/command/pip.yml
index ea7d1e4..7fa9d27 100644
--- a/test/fixtures/command/pip.yml
+++ b/test/fixtures/command/pip.yml
@@ -2,4 +2,4 @@ expected_dependency: MarkupSafe
 config:
   source_path: test/fixtures/pip
   python:
-    virtual_env_dir: "venv"
+    virtual_env_dir: "test/fixtures/pip/venv"
diff --git a/test/source/pip_test.rb b/test/source/pip_test.rb
index 00bf6b9..63a9bd0 100644
--- a/test/source/pip_test.rb
+++ b/test/source/pip_test.rb
@@ -5,7 +5,8 @@ require "tmpdir"
 if Licensed::Shell.tool_available?("pip")
   describe Licensed::Source::Pip do
     let (:fixtures)  { File.expand_path("../../fixtures/pip", __FILE__) }
-    let (:config)   { Licensed::Configuration.new("python" => {"virtual_env_dir" => "venv"}) }
+    let (:config)   { Licensed::Configuration.new(
+                        "python" => {"virtual_env_dir" => "test/fixtures/pip/venv"})}
     let (:source)   { Licensed::Source::Pip.new(config) }
 
     describe "enabled?" do

From 5ff87ec56e0eb0ae78c263b090197817ccf576a8 Mon Sep 17 00:00:00 2001
From: Avinash Sridhar <sridharavinash@github.com>
Date: Thu, 31 May 2018 15:54:46 -0400
Subject: [PATCH 33/37] Update docs

---
 docs/sources/pip.md | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/docs/sources/pip.md b/docs/sources/pip.md
index 5605512..2cf2880 100644
--- a/docs/sources/pip.md
+++ b/docs/sources/pip.md
@@ -1,11 +1,22 @@
 # Pip
 
-The pip source uses `pip` CLI commands to enumerate dependencies and properties. It is expected that `pip` is available in the PATH before running `licensed`
+The pip source uses `pip` CLI commands to enumerate dependencies and properties. It is expected that `pip` is available in the PATH before running `licensed`.
+
+Your repository root should also contain a `requirements.txt` file which contains all the packages and dependences that are needed. You can generate one with `pip` using the command:
+```
+pip freeze > requirements.txt
+```
+
+A `virtualenv` is assumed to be setup before running `licensed`. You can setup a virtualenv by running the command:
+```
+virtualenv <your_venv_dir>
+```
+_note_: `<your_venv_dir>` path should be relative to the repository root.
 
 #### virtual_env_dir
-A `virtualenv` is assumed to be setup before running `licensed`. The `pip` command will be sourced from this directory.
+ The `pip` command will be sourced from this directory.
 An example usage of this might look like:
 ```yaml
 python:
-    virtual_env_dir:"venv"
+    virtual_env_dir:"/path/to/your/venv_dir"
 ```

From 89595154b5ceda83765c80c3954466b789bdde11 Mon Sep 17 00:00:00 2001
From: Avinash Sridhar <sridharavinash@github.com>
Date: Thu, 31 May 2018 16:09:38 -0400
Subject: [PATCH 34/37] lint

---
 test/source/pip_test.rb | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/test/source/pip_test.rb b/test/source/pip_test.rb
index 63a9bd0..3962f01 100644
--- a/test/source/pip_test.rb
+++ b/test/source/pip_test.rb
@@ -5,8 +5,7 @@ require "tmpdir"
 if Licensed::Shell.tool_available?("pip")
   describe Licensed::Source::Pip do
     let (:fixtures)  { File.expand_path("../../fixtures/pip", __FILE__) }
-    let (:config)   { Licensed::Configuration.new(
-                        "python" => {"virtual_env_dir" => "test/fixtures/pip/venv"})}
+    let (:config)   { Licensed::Configuration.new("python" => {"virtual_env_dir" => "test/fixtures/pip/venv"}) }
     let (:source)   { Licensed::Source::Pip.new(config) }
 
     describe "enabled?" do

From 4a0a0cd263140a73b02945c9916aac751078a711 Mon Sep 17 00:00:00 2001
From: Avinash Sridhar <sridharavinash@github.com>
Date: Fri, 1 Jun 2018 08:56:58 -0400
Subject: [PATCH 35/37] update path requirement for pip

---
 docs/sources/pip.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/sources/pip.md b/docs/sources/pip.md
index 2cf2880..41276b8 100644
--- a/docs/sources/pip.md
+++ b/docs/sources/pip.md
@@ -1,13 +1,13 @@
 # Pip
 
-The pip source uses `pip` CLI commands to enumerate dependencies and properties. It is expected that `pip` is available in the PATH before running `licensed`.
+The pip source uses `pip` CLI commands to enumerate dependencies and properties. It is expected that `pip` is available in the `virtual_env_dir` specific directory before running `licensed`.
 
 Your repository root should also contain a `requirements.txt` file which contains all the packages and dependences that are needed. You can generate one with `pip` using the command:
 ```
 pip freeze > requirements.txt
 ```
 
-A `virtualenv` is assumed to be setup before running `licensed`. You can setup a virtualenv by running the command:
+A `virtualenv` is assumed to be setup before running `licensed`. You can setup a `virtualenv` by running the command:
 ```
 virtualenv <your_venv_dir>
 ```

From b63460dced1222f9c9ed0c4884558a25304ddc48 Mon Sep 17 00:00:00 2001
From: Avinash Sridhar <sridharavinash@github.com>
Date: Fri, 1 Jun 2018 16:36:01 -0400
Subject: [PATCH 36/37] break out python versions for the test matrix

---
 .travis.yml | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 7c4613b..8ded2b0 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -58,10 +58,17 @@ matrix:
       script: ./script/test manifest
       env: NAME="manifest"
 
-    # python tests
+    # python 2.7 tests
     - language: python
       python:
         - "2.7"
+      before_script: ./script/source-setup/pip
+      script: ./script/test pip
+      env: NAME="pip"
+
+    # python 3.6 tests
+    - language: python
+      python:
         - "3.6"
       before_script: ./script/source-setup/pip
       script: ./script/test pip

From bb76ab3ff617aebfb8102feadb6dd7b155750d43 Mon Sep 17 00:00:00 2001
From: Avinash Sridhar <sridharavinash@github.com>
Date: Fri, 1 Jun 2018 16:38:55 -0400
Subject: [PATCH 37/37] clarify and make mention virtualenv directory as
 required

---
 docs/sources/pip.md | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/docs/sources/pip.md b/docs/sources/pip.md
index 41276b8..dece16a 100644
--- a/docs/sources/pip.md
+++ b/docs/sources/pip.md
@@ -7,14 +7,15 @@ Your repository root should also contain a `requirements.txt` file which contain
 pip freeze > requirements.txt
 ```
 
-A `virtualenv` is assumed to be setup before running `licensed`. You can setup a `virtualenv` by running the command:
+A `virtualenv` directory is required before running `licensed`. You can setup a `virtualenv` by running the command:
 ```
 virtualenv <your_venv_dir>
 ```
-_note_: `<your_venv_dir>` path should be relative to the repository root.
+_note_: `<your_venv_dir>` path should be relative to the repository root or can be specified as an absolute path.
 
-#### virtual_env_dir
- The `pip` command will be sourced from this directory.
+#### virtual_env_dir (Required)
+
+The `pip` command will be sourced from this directory.
 An example usage of this might look like:
 ```yaml
 python: