Merge branch 'master' into go-dep-source

This commit is contained in:
Jon Ruskin 2018-06-01 14:49:03 -07:00
Родитель 4f293b8484 56d9e1a790
Коммит 75104d0ec5
25 изменённых файлов: 427 добавлений и 36 удалений

2
.gitignore поставляемый
Просмотреть файл

@ -19,7 +19,7 @@ test/fixtures/go/src/*
test/fixtures/go/pkg
!test/fixtures/go/src/test
test/fixtures/cabal/*
!test/fixtures/cabal/app.cabal
!test/fixtures/cabal/app*
vendor/licenses
.licenses

Просмотреть файл

@ -65,5 +65,21 @@ matrix:
script: ./script/test manifest
env: NAME="manifest"
# python 2.7 tests
- language: python
python:
- "2.7"
before_script: ./script/source-setup/pip
script: ./script/test pip
env: NAME="pip"
# python 3.6 tests
- language: python
python:
- "3.6"
before_script: ./script/source-setup/pip
script: ./script/test pip
env: NAME="pip"
notifications:
disable: true

Просмотреть файл

@ -80,6 +80,7 @@ Dependencies will be automatically detected for
5. [Go Dep](./docs/sources/dep.md)
6. [Manifest lists](./docs/sources/manifests.md)
7. [NPM](./docs/sources/npm.md)
8. [Pip](./docs/source/pip.md)
You can disable any of them in the configuration file:

Просмотреть файл

@ -2,7 +2,25 @@
The cabal source uses the `ghc-pkg` command to enumerate dependencies and provide metadata. It is un-opinionated on GHC packagedb locations and requires some configuration to ensure that all packages are properly found.
The cabal source will detect dependencies when a `.cabal` file is found at an apps `source_path`.
The cabal source will detect dependencies when a `.cabal` file is found at an apps `source_path`. By default, the cabal source will enumerate dependencies for all executable and library targets in a cabal file.
### Specifying which cabal file targets should enumerate dependencies
The cabal source can be configured to override which cabal file targets contain dependencies that need to be documented.
The default configuration is equivalent to:
```yml
cabal:
cabal_file_targets:
- executable
- library
```
However if you only wanted to enumerate dependencies for a `my_cabal_exe` executable target, you could specify:
```yml
cabal:
cabal_file_targets:
- executable my_cabal_exe
```
### Specifying GHC packagedb locations through environment
You can configure the `cabal` source to use specific packagedb locations by setting the `GHC_PACKAGE_PATH` environment variable before running `licensed`.

23
docs/sources/pip.md Normal file
Просмотреть файл

@ -0,0 +1,23 @@
# Pip
The pip source uses `pip` CLI commands to enumerate dependencies and properties. It is expected that `pip` is available in the `virtual_env_dir` specific directory before running `licensed`.
Your repository root should also contain a `requirements.txt` file which contains all the packages and dependences that are needed. You can generate one with `pip` using the command:
```
pip freeze > requirements.txt
```
A `virtualenv` directory is required before running `licensed`. You can setup a `virtualenv` by running the command:
```
virtualenv <your_venv_dir>
```
_note_: `<your_venv_dir>` path should be relative to the repository root or can be specified as an absolute path.
#### virtual_env_dir (Required)
The `pip` command will be sourced from this directory.
An example usage of this might look like:
```yaml
python:
virtual_env_dir:"/path/to/your/venv_dir"
```

Просмотреть файл

@ -11,6 +11,7 @@ require "licensed/source/npm"
require "licensed/source/go"
require "licensed/source/dep"
require "licensed/source/cabal"
require "licensed/source/pip"
require "licensed/configuration"
require "licensed/command/cache"
require "licensed/command/status"

Просмотреть файл

@ -11,7 +11,7 @@ module Licensed
def run(force: false)
summary = @config.apps.flat_map do |app|
app_name = app["name"]
@config.ui.info "Caching licenes for #{app_name}:"
@config.ui.info "Caching licenses for #{app_name}:"
# load the app environment
Dir.chdir app.source_path do
@ -40,8 +40,9 @@ module Licensed
# or default to a blank license
license = Licensed::License.read(filename) || Licensed::License.new
# Version did not change, no need to re-cache
if !force && version == license["version"]
# cached version string exists and did not change, no need to re-cache
has_version = !license["version"].nil? && !license["version"].empty?
if !force && has_version && version == license["version"]
@config.ui.info " Using #{name} (#{version})"
next
end

Просмотреть файл

@ -23,7 +23,7 @@ module Licensed
file = File.directory?(descriptor) ? "." : File.basename(descriptor)
Dir.chdir dir do
Licensed::Shell.execute("git", "rev-list", "-1", "HEAD", "--", file)
Licensed::Shell.execute("git", "rev-list", "-1", "HEAD", "--", file, allow_failure: true)
end
end

Просмотреть файл

@ -3,12 +3,19 @@ require "open3"
module Licensed
module Shell
# Executes a command, returning it's STDOUT on success. Returns an empty
# string on failure
def self.execute(cmd, *args)
output, _, status = Open3.capture3(cmd, *args)
return "" unless status.success?
output.strip
# Executes a command, returning its standard output on success. On failure,
# it raises an exception that contains the error output, unless
# `allow_failure` is true, in which case it returns an empty string.
def self.execute(cmd, *args, allow_failure: false)
stdout, stderr, status = Open3.capture3(cmd, *args)
if status.success?
stdout.strip
elsif allow_failure
""
else
raise Error.new([cmd, *args], status.exitstatus, stderr)
end
end
# Executes a command and returns a boolean value indicating if the command
@ -24,5 +31,31 @@ module Licensed
output, err, status = Open3.capture3("which", tool)
status.success? && !output.strip.empty? && err.strip.empty?
end
class Error < RuntimeError
def initialize(cmd, status, stderr)
super()
@cmd = cmd
@exitstatus = status
@output = stderr
end
def message
output = @output.to_s.strip
extra = output.empty?? "" : "\n#{output.gsub(/^/, " ")}"
"command exited with status #{@exitstatus}\n #{escape_cmd}#{extra}"
end
def escape_cmd
@cmd.map do |arg|
if arg =~ /[\s'"]/
escaped = arg.gsub(/([\\"])/, '\\\\\1')
%("#{escaped}")
else
arg
end
end.join(" ")
end
end
end
end

Просмотреть файл

@ -1,5 +1,8 @@
# frozen_string_literal: true
require "bundler"
begin
require "bundler"
rescue LoadError
end
module Licensed
module Source
@ -15,7 +18,7 @@ module Licensed
end
def enabled?
lockfile_path && lockfile_path.exist?
defined?(::Bundler) && lockfile_path && lockfile_path.exist?
end
def dependencies

Просмотреть файл

@ -4,6 +4,9 @@ require "English"
module Licensed
module Source
class Cabal
DEPENDENCY_REGEX = /\s*.+?\s*/.freeze
DEFAULT_TARGETS = %w{executable library}.freeze
def self.type
"cabal"
end
@ -13,7 +16,7 @@ module Licensed
end
def enabled?
cabal_packages.any? && ghc?
cabal_file_dependencies.any? && ghc?
end
def dependencies
@ -63,8 +66,7 @@ module Licensed
# Returns a `Set` of the package ids for all cabal dependencies
def package_ids
deps = cabal_packages.flat_map { |n| package_dependencies(n, false) }
recursive_dependencies(deps)
recursive_dependencies(cabal_file_dependencies)
end
# Recursively finds the dependencies for each cabal package.
@ -125,7 +127,7 @@ module Licensed
# Runs a `ghc-pkg field` command for a given set of fields and arguments
# Automatically includes ghc package DB locations in the command
def ghc_pkg_field_command(id, fields, *args)
Licensed::Shell.execute("ghc-pkg", "field", id, fields.join(","), *args, *package_db_args)
Licensed::Shell.execute("ghc-pkg", "field", id, fields.join(","), *args, *package_db_args, allow_failure: true)
end
# Returns an array of ghc package DB locations as specified in the app
@ -148,12 +150,56 @@ module Licensed
path.gsub("<ghc_version>", ghc_version)
end
# Return an array of the top-level cabal packages for the current app
def cabal_packages
cabal_files.map do |f|
name_match = File.read(f).match(/^name:\s*(.*)$/)
name_match[1] if name_match
end.compact
# Returns a set containing the top-level dependencies found in cabal files
def cabal_file_dependencies
cabal_files.each_with_object(Set.new) do |cabal_file, packages|
content = File.read(cabal_file)
next if content.nil? || content.empty?
# add any dependencies for matched targets from the cabal file.
# by default this will find executable and library dependencies
content.scan(cabal_file_regex).each do |match|
# match[1] is a string of "," separated dependencies
dependencies = match[1].split(",").map(&:strip)
dependencies.each do |dep|
# the dependency might have a version specifier.
# remove it so we can get the full id specifier for each package
id = cabal_package_id(dep.split(/\s/)[0])
packages.add(id) if id
end
end
end
end
# Returns an installed package id for the package.
def cabal_package_id(package_name)
field = ghc_pkg_field_command(package_name, ["id"])
id = field.split(":", 2)[1]
id.strip if id
end
# Find `build-depends` lists from specified targets in a cabal file
def cabal_file_regex
# this will match 0 or more occurences of
# match[0] - specifier, e.g. executable, library, etc
# match[1] - full list of matched dependencies
# match[2] - first matched dependency (required)
# match[3] - remainder of matched dependencies (not required)
@cabal_file_regex ||= /
# match a specifier, e.g. library or executable
^(#{cabal_file_targets.join("|")})
.*? # stuff
# match a list of 1 or more dependencies
build-depends:(#{DEPENDENCY_REGEX}(,#{DEPENDENCY_REGEX})*)\n
/xmi
end
# Returns the targets to search for `build-depends` in a cabal file
def cabal_file_targets
targets = Array(@config.dig("cabal", "cabal_file_targets"))
targets.push(*DEFAULT_TARGETS) if targets.empty?
targets
end
# Returns an array of the local directory cabal package files

Просмотреть файл

@ -1,6 +1,7 @@
# frozen_string_literal: true
require "json"
require "English"
require "pathname"
module Licensed
module Source
@ -14,7 +15,7 @@ module Licensed
end
def enabled?
go_source?
Licensed::Shell.tool_available?("go") && go_source?
end
def dependencies
@ -113,7 +114,7 @@ module Licensed
# package - Go package import path
def package_info_command(package)
package ||= ""
Licensed::Shell.execute("go", "list", "-json", package)
Licensed::Shell.execute("go", "list", "-json", package, allow_failure: true)
end
# Returns the info for the package under test
@ -140,7 +141,12 @@ module Licensed
@gopath = if path.nil? || path.empty?
ENV["GOPATH"]
else
File.expand_path(path, Licensed::Git.repository_root)
root = begin
Licensed::Git.repository_root
rescue Licensed::Shell::Error
Pathname.pwd
end
File.expand_path(path, root)
end
end

Просмотреть файл

@ -13,7 +13,7 @@ module Licensed
end
def enabled?
File.exist?(@config.pwd.join("package.json"))
Licensed::Shell.tool_available?("npm") && File.exist?(@config.pwd.join("package.json"))
end
def dependencies

Просмотреть файл

@ -0,0 +1,63 @@
# frozen_string_literal: true
require "json"
require "English"
module Licensed
module Source
class Pip
def self.type
"pip"
end
def initialize(config)
@config = config
end
def enabled?
File.exist?(@config.pwd.join("requirements.txt"))
end
def dependencies
@dependencies ||= parse_requirements_txt.map do |package_name|
package = package_info(package_name)
location = File.join(package["Location"], package["Name"] + "-" + package["Version"] + ".dist-info")
Dependency.new(location, {
"type" => Pip.type,
"name" => package["Name"],
"summary" => package["Summary"],
"homepage" => package["Home-page"],
"version" => package["Version"]
})
end
end
# Build the list of packages from a 'requirements.txt'
# Assumes that the requirements.txt follow the format pkg=1.0.0 or pkg==1.0.0
def parse_requirements_txt
File.open(@config.pwd.join("requirements.txt")).map do |line|
p_split = line.split("=")
p_split[0]
end
end
def package_info(package_name)
p_info = pip_command(package_name).lines
p_info.each_with_object(Hash.new(0)) { |pkg, a|
k, v = pkg.split(":", 2)
next if k.nil? || k.empty?
a[k.strip] = v&.strip
}
end
def pip_command(*args)
venv_dir = @config.dig("python", "virtual_env_dir")
if venv_dir.nil?
raise "Virtual env directory not set."
end
venv_dir = File.expand_path(venv_dir, Licensed::Git.repository_root)
pip = File.join(venv_dir, "bin", "pip")
Licensed::Shell.execute(pip, "--disable-pip-version-check", "show", *args)
end
end
end
end

Просмотреть файл

@ -11,7 +11,7 @@ BASE_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
cd $BASE_PATH/test/fixtures/cabal
if [ "$1" == "-f" ]; then
find . -not -regex "\.*" -and -not -name "app\.cabal" -print0 | xargs -0 rm -rf
find . -not -regex "\.*" -and -not -path "*app*" -print0 | xargs -0 rm -rf
fi
cabal new-build

29
script/source-setup/pip Executable file
Просмотреть файл

@ -0,0 +1,29 @@
#!/bin/bash
set -e
if [ -z "$(which pip)" ]; then
echo "A local pip installation is required for python development." >&2
exit 127
fi
if [ -z "$(which virtualenv)" ]; then
echo "A local virtualenv installation is required for python development." >&2
exit 127
fi
# setup test fixtures
BASE_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
# clean up any previous fixture venv that might have been created.
if [ "$1" == "-f" ]; then
echo "removing old fixture setup..."
rm -rf $BASE_PATH/test/fixtures/pip/venv
fi
# set up a virtualenv and install the packages in the test requirements
virtualenv $BASE_PATH/test/fixtures/pip/venv
. $BASE_PATH/test/fixtures/pip/venv/bin/activate
pip install -r $BASE_PATH/test/fixtures/pip/requirements.txt
deactivate

Просмотреть файл

@ -72,6 +72,50 @@ describe Licensed::Command::Cache do
refute_equal "0.0", license["version"]
end
it "does not reuse nil license version" do
generator.run
path = config.cache_path.join("test/dependency.txt")
license = Licensed::License.read(path)
license["license"] = "test"
license.save(path)
test_dependency = Licensed::Dependency.new(Dir.pwd, {
"type" => TestSource.type,
"name" => "dependency"
})
TestSource.stub(:create_dependency, test_dependency) do
generator.run
end
license = Licensed::License.read(path)
assert_equal "test", license["license"]
assert_equal "1.0", license["version"]
end
it "does not reuse empty license version" do
generator.run
path = config.cache_path.join("test/dependency.txt")
license = Licensed::License.read(path)
license["license"] = "test"
license["version"] = ""
license.save(path)
test_dependency = Licensed::Dependency.new(Dir.pwd, {
"type" => TestSource.type,
"name" => "dependency",
"version" => ""
})
TestSource.stub(:create_dependency, test_dependency) do
generator.run
end
license = Licensed::License.read(path)
assert_equal "test", license["license"]
assert_equal "1.0", license["version"]
end
it "does not include ignored dependencies in dependency counts" do
config.ui.level = "info"
out, _ = capture_io { generator.run }

6
test/fixtures/cabal/app.cabal поставляемый
Просмотреть файл

@ -11,3 +11,9 @@ library
hs-source-dirs: .
build-depends: zlib == 0.6.2
default-language: Haskell2010
executable app
hs-source-dirs: app
main-is: Main.hs
build-depends: base, Glob == 0.9.2
default-language: Haskell2010

1
test/fixtures/cabal/app/Main.hs поставляемый Normal file
Просмотреть файл

@ -0,0 +1 @@
main = putStrLn "Hello world"

5
test/fixtures/command/pip.yml поставляемый Normal file
Просмотреть файл

@ -0,0 +1,5 @@
expected_dependency: MarkupSafe
config:
source_path: test/fixtures/pip
python:
virtual_env_dir: "test/fixtures/pip/venv"

2
test/fixtures/pip/requirements.txt поставляемый Normal file
Просмотреть файл

@ -0,0 +1,2 @@
Jinja2==2.9.6
MarkupSafe==1.0

Просмотреть файл

@ -59,6 +59,24 @@ if Licensed::Shell.tool_available?("ghc")
assert dep["summary"]
end
end
it "finds dependencies for executables" do
config["cabal"] = { "ghc_package_db" => ["global", user_db, local_db] }
Dir.chdir(fixtures) do
dep = source.dependencies.detect { |d| d["name"] == "Glob" }
assert dep
assert_equal "cabal", dep["type"]
assert_equal "0.9.2", dep["version"]
assert dep["summary"]
end
end
it "does not include the target project" do
config["cabal"] = { "ghc_package_db" => ["global", user_db, local_db] }
Dir.chdir(fixtures) do
refute source.dependencies.detect { |d| d["name"] == "app" }
end
end
end
describe "package_db_args" do

Просмотреть файл

@ -1,6 +1,7 @@
# frozen_string_literal: true
require "test_helper"
require "tmpdir"
require "fileutils"
if Licensed::Shell.tool_available?("npm")
describe Licensed::Source::NPM do
@ -52,6 +53,18 @@ if Licensed::Shell.tool_available?("npm")
refute @source.dependencies.detect { |dep| dep["name"] == "string.prototype.startswith" }
end
end
it "raises when dependencies are missing" do
Dir.mktmpdir do |dir|
FileUtils.cp(File.join(fixtures, "package.json"), File.join(dir, "package.json"))
Dir.chdir(dir) do
error = assert_raises(Licensed::Shell::Error) { @source.dependencies }
assert_includes error.message, "command exited with status 1"
assert_includes error.message, "npm list --parseable --production --long"
assert_includes error.message, "npm ERR! missing: autoprefixer@"
end
end
end
end
end
end

60
test/source/pip_test.rb Normal file
Просмотреть файл

@ -0,0 +1,60 @@
# frozen_string_literal: true
require "test_helper"
require "tmpdir"
if Licensed::Shell.tool_available?("pip")
describe Licensed::Source::Pip do
let (:fixtures) { File.expand_path("../../fixtures/pip", __FILE__) }
let (:config) { Licensed::Configuration.new("python" => {"virtual_env_dir" => "test/fixtures/pip/venv"}) }
let (:source) { Licensed::Source::Pip.new(config) }
describe "enabled?" do
it "is true if pip source is available" do
Dir.chdir(fixtures) do
assert source.enabled?
end
end
it "is false if pip source is not available" do
Dir.mktmpdir do |dir|
Dir.chdir(dir) do
refute source.enabled?
end
end
end
end
describe "config file params check" do
it "fails if virtual_env_dir is not set" do
config.delete("python")
assert_raises RuntimeError do
Dir.chdir(fixtures) do
source.pip_command
end
end
end
end
describe "dependencies" do
it "includes direct dependencies" do
Dir.chdir fixtures do
dep = source.dependencies.detect { |d| d["name"] == "Jinja2" }
assert dep
assert_equal "pip", dep["type"]
assert dep["homepage"]
assert dep["summary"]
end
end
it "includes indirect dependencies" do
Dir.chdir fixtures do
dep = source.dependencies.detect { |d| d["name"] == "MarkupSafe" }
assert dep
assert_equal "pip", dep["type"]
assert dep["homepage"]
assert dep["summary"]
end
end
end
end
end

Просмотреть файл

@ -36,13 +36,15 @@ class TestSource
def dependencies
@dependencies_hook.call if @dependencies_hook.respond_to?(:call)
@dependencies ||= [
Licensed::Dependency.new(Dir.pwd, {
"type" => TestSource.type,
"name" => "dependency",
"version" => "1.0"
})
]
@dependencies ||= [TestSource.create_dependency]
end
def self.create_dependency
Licensed::Dependency.new(Dir.pwd, {
"type" => TestSource.type,
"name" => "dependency",
"version" => "1.0"
})
end
end