[dev_setup] Fix Chef regression

This change replaces the use of remote_file with cf_remote_file to
work around two limitations in Chef 0.10:
  - Chef shares its downloading logic of remote_file with its REST
    client.  The REST client decompresses archives when it sees the
    header "Content-Encoding: gzip", which is an undesirable behavior
    for remote_file.
  - Chef only uses the specified checksum to trigger update, but it does
    no validation after new remote files are downloaded.

  Our requirement of cf_remote_file:
  - Don't do content decoding magic
  - checksum required and *always* validates the checksum

  Test plan: remove locally compiled ruby and the cached tarballs, run
dev_setup.

Change-Id: Ifb91188ec7ec2f77fafc671d128625ec0f8f1c77

dev_setup/cookbooks/cloud_controller/attributes/default.rb

@@ -1,4 +1,4 @@
-include_recipe "deployment"
+include_attributes "deployment"
 include_attributes "uaa"
 default[:deployment][:welcome] = "VMware's Cloud Application Platform"
 

dev_setup/cookbooks/deployment/providers/remote_file.rb

@@ -0,0 +1,57 @@
+require 'chef/mixin/checksum'
+require 'chef/mixin/enforce_ownership_and_permissions'
+
+include Chef::Mixin::Checksum
+
+require 'net/http'
+
+def load_current_resource
+  @current_resource = @new_resource.class.new(@new_resource.name)
+  if @current_resource.path.exist?
+    @current_resource.checksum(checksum(@current_resource.path.to_s))
+  end
+end
+
+def stream_to_tempfile(uri)
+  Net::HTTP.start(uri.host, uri.port) do |http|
+    http.request_get(uri.path) do |response|
+      Tempfile.open('cf-temp') do |tf|
+        response.read_body {|chunk| tf.write(chunk) }
+        tf.close
+        yield tf.path
+      end
+    end
+  end
+end
+
+def action_create
+  unless @new_resource.path.dirname.directory?
+    raise "Cannot create file under #{@new_resource.path.dirname}, check if it refers to a real directory"
+  end
+
+  Chef::Log.debug("#{@new_resource} checking for changes")
+
+  if @current_resource.path.exist? && @current_resource.checksum == @new_resource.checksum
+    Chef::Log.debug("#{@new_resource} checksum matches target checksum (#{@new_resource.checksum}) - not updating")
+  else
+    uri = URI(@new_resource.source)
+    stream_to_tempfile(uri) do |tempfile|
+      # new download should be rejected if checksum is invalid
+      Chef::Log.debug "#{@new_resource} has checksum set to: #{@new_resource.checksum}, checking remote file checksum"
+      c = checksum(tempfile)
+      if c == @new_resource.checksum
+        Chef::Log.debug "remote file checksum: #{c}, validated!"
+        FileUtils.cp(tempfile, @new_resource.path)
+        Chef::Log.info "#{@new_resource} updated"
+        @new_resource.updated_by_last_action(true)
+      else
+        Chef::Log.debug "remote file checksum: #{c}, invalid"
+        # TODO: retry?
+        raise "Checksum mismatch for cf_remote_file #{@new_resource}"
+      end
+    end
+  end
+
+  enforce_ownership_and_permissions
+  @new_resource.updated_by_last_action?
+end

dev_setup/cookbooks/deployment/resources/remote_file.rb

@@ -0,0 +1,18 @@
+provides :cf_remote_file
+
+require 'pathname'
+
+default_action :create
+
+attribute :checksum, :regex => /^[\da-fA-F]{64}$/, :required => true
+attribute :source, :regex => /^http:/, :required => true
+attribute :owner, :kind_of => [Integer, String]
+attribute :group, :kind_of => [Integer, String]
+attribute :mode, :regex => /^0?\d{3,4}$/
+
+attr_reader :path
+
+def initialize(name, *reset)
+  super
+  @path = Pathname.new(name).expand_path
+end

dev_setup/cookbooks/erlang/recipes/default.rb

@@ -3,13 +3,13 @@
 #
 # Copyright 2012, VMware
 #
-#
+
 %w[ build-essential libncurses5-dev openssl libssl-dev ].each do |pkg|
   package pkg
 end
 
 tarball_path = File.join(node[:deployment][:setup_cache], "otp_src_#{node[:erlang][:version]}.tar.gz")
-remote_file tarball_path do
+cf_remote_file tarball_path do
   owner node[:deployment][:user]
   source node[:erlang][:source]
   checksum "849d050b59821e9f2831fee2e3267d84b410eee860a55f6fc9320cc00b5205bd"

dev_setup/cookbooks/maven/recipes/default.rb

@@ -28,6 +28,7 @@ remote_file  File.join("", "tmp", "apache-maven-#{node[:maven][:version]}.tar.gz
   owner node[:deployment][:user]
   source node[:maven][:source]
   not_if { ::File.exists?(File.join("", "tmp", "apache-maven-#{node[:maven][:version]}.tar.gz")) }
+  checksum 'd35a876034c08cb7e20ea2fbcf168bcad4dff5801abad82d48055517513faa2f'
 end
 
 directory node[:maven][:base] do

dev_setup/cookbooks/mongodb/recipes/default.rb

@@ -4,11 +4,12 @@
 #
 # Copyright 2011, VMware
 #
-#
-remote_file File.join("", "tmp", "mongodb-linux-#{node[:kernel][:machine]}-#{node[:mongodb][:version]}.tgz") do
-  owner node[:deployment][:user]
+
+cf_remote_file File.join("", "tmp", "mongodb-linux-#{node[:kernel][:machine]}-#{node[:mongodb][:version]}.tgz") do
+  # owner node[:deployment][:user]
   source node[:mongodb][:source]
-  not_if { ::File.exists?(File.join("", "tmp", "mongodb-linux-#{node[:kernel][:machine]}-#{node[:mongodb][:version]}.tgz")) }
+  # not_if { ::File.exists?(File.join("", "tmp", "mongodb-linux-#{node[:kernel][:machine]}-#{node[:mongodb][:version]}.tgz")) }
+  checksum '8f6a58293068e0fb28b463b955f3660f492094e53129fb88af4a7efcfc7995da'
 end
 
 directory File.join(node[:mongodb][:path], "bin") do

dev_setup/cookbooks/neo4j/recipes/default.rb

@@ -4,6 +4,7 @@
 #
 # Copyright 2011, VMware
 #
+
 [node[:neo4j][:service_dir], File.join(node[:neo4j][:service_dir], "instances")].each do |dir|
   directory dir do
     owner node[:deployment][:user]

dev_setup/cookbooks/redis/recipes/default.rb

@@ -1,4 +1,4 @@
-remote_file File.join(node[:deployment][:setup_cache], "redis-#{node[:redis][:version]}.tar.gz") do
+cf_remote_file File.join(node[:deployment][:setup_cache], "redis-#{node[:redis][:version]}.tar.gz") do
   owner node[:deployment][:user]
   source "http://redis.googlecode.com/files/redis-#{node[:redis][:version]}.tar.gz"
   checksum node[:redis][:checksum]

dev_setup/cookbooks/ruby/attributes/default.rb

@@ -7,3 +7,4 @@ default[:ruby][:checksums]["1.9.2-p180"] = "9027a5abaaadc2af85005ed74aeb628ce232
 default[:rubygems][:version] = "1.8.24"
 default[:rubygems][:bundler][:version] = "1.1.3"
 default[:rubygems][:rake][:version] = "0.8.7"
+default[:rubygems][:checksum] = "4b61fa51869b3027bcfe67184b42d2e8c23fa6ab17d47c5c438484b9be2821dd"

dev_setup/cookbooks/ruby/libraries/ruby_install.rb

@@ -9,7 +9,7 @@ module RubyInstall
     end
 
     tarball_path = File.join(node[:deployment][:setup_cache], "ruby-#{ruby_version}.tar.gz")
-    remote_file tarball_path do
+    cf_remote_file tarball_path do
       owner node[:deployment][:user]
       source ruby_source
       checksum node[:ruby][:checksums][ruby_version]
@@ -27,6 +27,8 @@ module RubyInstall
       cwd File.join("", "tmp")
       user node[:deployment][:user]
       code <<-EOH
+      # work around chef's decompression of source tarball before a more elegant
+      # solution is found
       tar xzf #{tarball_path}
       cd ruby-#{ruby_version}
       ./configure --disable-pthread --prefix=#{ruby_path}
@@ -38,9 +40,10 @@ module RubyInstall
       end
     end
 
-    remote_file File.join("", "tmp", "rubygems-#{rubygems_version}.tgz") do
+    cf_remote_file File.join("", "tmp", "rubygems-#{rubygems_version}.tgz") do
       owner node[:deployment][:user]
       source "http://production.cf.rubygems.org/rubygems/rubygems-#{rubygems_version}.tgz"
+      checksum node[:rubygems][:checksum]
       not_if { ::File.exists?(File.join("", "tmp", "rubygems-#{rubygems_version}.tgz")) }
     end