Merge branch 'master' into router_v2_in_gerrit

Conflicts:
	cloud_controller/Rakefile
	dev_setup/cookbooks/nginx/templates/default/ubuntu-nginx.conf.erb
	router/Gemfile.lock

Change-Id: I7cb5e2b65c21926e9875f613db103a026d20125a

cloud_controller/Gemfile

@@ -3,18 +3,18 @@ source 'http://rubygems.org'
 # Rails itself
 gem 'rails', '~> 3.0.5'
 # Message bus
-gem 'nats', '>= 0.4.10', :require => 'nats/client'
+gem 'nats', :require => 'nats/client'
 
 gem 'logging', '>= 1.5.0'
 
 # VCAP common components
-gem 'vcap_common', :require => ['vcap/common', 'vcap/component'], :path => '../common'
+gem 'vcap_common', :require => ['vcap/common', 'vcap/component']
 gem 'vcap_logging', :require => ['vcap/logging']
-gem 'vcap_staging', '= 0.1.25'
+gem 'vcap_staging'
 
 # For queuing staging tasks
 gem 'em-hiredis'
-gem 'vcap_stager', '= 0.1.3'
+gem 'vcap_stager', '~> 0.1.7'
 
 # Databases
 gem 'sqlite3'
@@ -34,20 +34,20 @@ group :production do
 end
 
 # EventMachine and async support
-gem 'eventmachine',  '~> 0.12.11.cloudfoundry.2'
-gem 'thin', '> 1.2'
+gem 'eventmachine'
+gem 'thin'
 gem 'em-http-request', '~> 1.0.0.beta.3', :require => 'em-http'
-gem 'em-redis', :require => nil
-gem 'rack-fiber_pool', :require => nil
+gem 'em-redis', '~> 0.3.0', :require => nil
+gem 'rack-fiber_pool', '~> 0.9.1', :require => nil
 
 # Support libraries
-gem 'yajl-ruby', '>= 0.7.9'
-gem 'nokogiri', '>= 1.4.4'
-gem 'bcrypt-ruby', '>= 2.1.4'
-gem 'ruby-hmac', :require => 'hmac-sha1'
-gem 'SystemTimer', :platforms => :mri_18
-gem 'uuidtools'
-gem 'rest-client', '= 1.6.7'
+gem 'yajl-ruby', '~> 0.8.3'
+gem 'nokogiri', '~> 1.4.4'
+gem 'bcrypt-ruby', '~> 2.1.4'
+gem 'ruby-hmac', '~> 0.4.0', :require => 'hmac-sha1'
+gem 'SystemTimer', '~> 1.2', :platforms => :mri_18
+gem 'uuidtools', '~> 2.1.2'
+gem 'rest-client', '~> 1.6.7'
 
 # rspec-rails is outside the 'test' group in order to consistently provide Rake tasks.
 gem 'rspec-rails', '>= 2.4.1'

cloud_controller/Gemfile.lock

@@ -1,14 +1,3 @@
-PATH
-  remote: ../common
-  specs:
-    vcap_common (0.99)
-      eventmachine (~> 0.12.11.cloudfoundry.2)
-      logging (>= 1.5.0)
-      nats
-      posix-spawn
-      thin
-      yajl-ruby
-
 GEM
   remote: http://rubygems.org/
   specs:
@@ -47,10 +36,10 @@ GEM
     chronic (0.6.4)
     ci_reporter (1.6.4)
       builder (>= 2.1.2)
-    daemons (1.1.2)
+    daemons (1.1.5)
     delorean (1.1.0)
       chronic
-    diff-lcs (1.1.2)
+    diff-lcs (1.1.3)
     em-hiredis (0.1.0)
       hiredis (~> 0.3.0)
     em-http-request (1.0.0.beta.3)
@@ -64,13 +53,13 @@ GEM
       eventmachine
     erubis (2.6.6)
       abstract (>= 1.0.0)
-    eventmachine (0.12.11.cloudfoundry.2)
+    eventmachine (0.12.11.cloudfoundry.3)
     hiredis (0.3.2)
     http_parser.rb (0.5.1)
     i18n (0.5.0)
-    json_pure (1.5.1)
-    little-plugger (1.1.2)
-    logging (1.5.0)
+    json_pure (1.6.4)
+    little-plugger (1.1.3)
+    logging (1.6.1)
       little-plugger (>= 1.1.2)
     mail (2.2.15)
       activesupport (>= 2.3.6)
@@ -80,15 +69,16 @@ GEM
     mime-types (1.16)
     mocha (0.9.12)
     mysql2 (0.2.7)
-    nats (0.4.10)
-      daemons (>= 1.1.0)
+    nats (0.4.22.beta.4)
+      daemons (>= 1.1.4)
       eventmachine (>= 0.12.10)
-      json_pure (>= 1.5.1)
-    nokogiri (1.4.4)
+      json_pure (>= 1.6.1)
+      thin (>= 1.3.1)
+    nokogiri (1.4.7)
     pg (0.10.1)
     polyglot (0.3.1)
     posix-spawn (0.3.6)
-    rack (1.2.2)
+    rack (1.2.5)
     rack-fiber_pool (0.9.1)
     rack-mount (0.6.14)
       rack (>= 1.0.0)
@@ -107,7 +97,7 @@ GEM
       activesupport (= 3.0.5)
       rake (>= 0.8.7)
       thor (~> 0.14.4)
-    rake (0.8.7)
+    rake (0.9.2.2)
     rcov (0.9.9)
     rest-client (1.6.7)
       mime-types (>= 1.16)
@@ -115,7 +105,7 @@ GEM
       rspec-core (~> 2.5.0)
       rspec-expectations (~> 2.5.0)
       rspec-mocks (~> 2.5.0)
-    rspec-core (2.5.1)
+    rspec-core (2.5.2)
     rspec-expectations (2.5.0)
       diff-lcs (~> 1.1.2)
     rspec-mocks (2.5.0)
@@ -129,7 +119,7 @@ GEM
       rack (~> 1.1)
       tilt (>= 1.2.2, < 2.0)
     sqlite3 (1.3.3)
-    thin (1.2.11)
+    thin (1.3.1)
       daemons (>= 1.0.9)
       eventmachine (>= 0.12.6)
       rack (>= 1.0.0)
@@ -139,47 +129,54 @@ GEM
       polyglot (>= 0.3.1)
     tzinfo (0.3.26)
     uuidtools (2.1.2)
+    vcap_common (1.0.3)
+      eventmachine (~> 0.12.11.cloudfoundry.3)
+      logging (>= 1.5.0)
+      nats (~> 0.4.22.beta.4)
+      posix-spawn (~> 0.3.6)
+      thin (~> 1.3.1)
+      yajl-ruby (~> 0.8.3)
     vcap_logging (0.1.3)
-    vcap_stager (0.1.3)
-    vcap_staging (0.1.25)
+    vcap_stager (0.1.7)
+    vcap_staging (0.1.30)
       nokogiri (>= 1.4.4)
       rake
       rspec
       vcap_common
       yajl-ruby (>= 0.7.9)
-    yajl-ruby (0.8.2)
+    yajl-ruby (0.8.3)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  SystemTimer
-  bcrypt-ruby (>= 2.1.4)
+  SystemTimer (~> 1.2)
+  bcrypt-ruby (~> 2.1.4)
   ci_reporter
   delorean
   em-hiredis
   em-http-request (~> 1.0.0.beta.3)
-  em-redis
-  eventmachine (~> 0.12.11.cloudfoundry.2)
+  em-redis (~> 0.3.0)
+  eventmachine
   logging (>= 1.5.0)
   mocha
   mysql2 (>= 0.2.6)
-  nats (>= 0.4.10)
-  nokogiri (>= 1.4.4)
+  nats
+  nokogiri (~> 1.4.4)
   pg
-  rack-fiber_pool
+  rack-fiber_pool (~> 0.9.1)
   rails (~> 3.0.5)
   rcov
-  rest-client (= 1.6.7)
+  rest-client (~> 1.6.7)
   rspec (>= 2.4.0)
   rspec-rails (>= 2.4.1)
-  ruby-hmac
+  ruby-hmac (~> 0.4.0)
   sinatra
   sqlite3
-  thin (> 1.2)
-  uuidtools
-  vcap_common!
+  thin
+  uuidtools (~> 2.1.2)
+  vcap_common
   vcap_logging
-  vcap_stager (= 0.1.3)
-  vcap_staging (= 0.1.25)
-  yajl-ruby (>= 0.7.9)
+  vcap_stager (~> 0.1.7)
+  vcap_staging
+  yajl-ruby (~> 0.8.3)

cloud_controller/Rakefile

@@ -1,5 +1,5 @@
 require File.expand_path('../config/application', __FILE__)
-require 'ci/reporter/rake/rspec'
+require 'ci/reporter/rake/rspec' if Rails.env != 'production'
 
 CloudController::Application.load_tasks
 

cloud_controller/app/controllers/users_controller.rb

@@ -47,7 +47,7 @@ class UsersController < ApplicationController
     target_user = ::User.find_by_email(params['email'])
     if target_user
       if target_user.email == user.email || @current_user.admin?
-        render :json => { :email => target_user.email }
+        render :json => { :email => target_user.email, :admin => target_user.admin? }
       else
         raise CloudError.new(CloudError::FORBIDDEN)
       end

cloud_controller/app/models/app_package.rb

@@ -22,6 +22,7 @@ class AppPackage
   def to_zip
     tmpdir = Dir.mktmpdir
     dir = path = nil
+    check_package_size
     timed_section(CloudController.logger, 'app_to_zip') do
       dir = unpack_upload
       synchronize_pool_with(dir)
@@ -96,8 +97,6 @@ class AppPackage
     end
   end
 
-private
-
   def package_dir
     self.class.package_dir
   end
@@ -109,6 +108,56 @@ private
     new_path
   end
 
+  # Verifies that the recreated droplet size is less than the
+  # maximum allowed by the AppConfig.
+  def check_package_size
+    unless @uploaded_file
+      # When the entire set of files that make up the application is already
+      # in the resource pool, the client may not send us any additional contents
+      # i.e. the payload is empty.
+      CloudController.logger.debug "No uploaded file for application, contents assumed to be present in resource pool"
+      return
+    end
+
+    # Avoid stat'ing files in the resource pool if possible
+    total_size = get_unzipped_size
+    unless total_size <= AppConfig[:max_droplet_size]
+      limit_str = VCAP.pp_bytesize(AppConfig[:max_droplet_size])
+      size_str = VCAP.pp_bytesize(total_size)
+      CloudController.logger.warn "Zipped app is #{size_str}, limit is #{limit_str}"
+      raise AppPackageError, "Application exceeds maximum allowed size (#{limit_str})"
+    end
+
+    # Ugh, this stat's all the files that would need to be copied
+    # from the resource pool. Consider caching sizes in resource pool?
+    sizes = CloudController.resource_pool.resource_sizes(@resource_descriptors)
+    total_size += sizes.reduce(0) {|accum, cur| accum + cur[:size] }
+    unless total_size <= AppConfig[:max_droplet_size]
+      limit_str = VCAP.pp_bytesize(AppConfig[:max_droplet_size])
+      size_str = VCAP.pp_bytesize(total_size)
+      CloudController.logger.warn "Zipped app + cached resources is #{size_str}, limit is #{limit_str}"
+      raise AppPackageError, "Application exceeds maximum allowed size (#{limit_str})"
+    end
+  end
+
+  def get_unzipped_size
+    cmd = "unzip -l #{@uploaded_file.path}"
+    f = Fiber.current
+    EM.system(cmd) {|output, status| f.resume({:status => status, :stdout => output}) }
+    result = Fiber.yield
+    unless result[:status].exitstatus == 0
+      raise AppPackageError, "Failed listing application archive"
+    end
+
+    lines = result[:stdout].split(/\n/)
+    matches = lines.last.match(/^\s*(\d+)\s+(\d+) file/)
+    unless matches
+      raise AppPackageError, "Failed parsing application archive listing"
+    end
+
+    Integer(matches[1])
+  end
+
   def unpack_upload
     working_dir = Dir.mktmpdir
     if @uploaded_file
@@ -128,6 +177,19 @@ private
     working_dir
   end
 
+  # enforce property that any file in resource list must be located in the
+  # apps directory e.g. '../../foo' or a symlink pointing outside working_dir
+  # should raise an exception.
+  def resolve_path(working_dir, tainted_path)
+    expanded_dir  = File.realdirpath(working_dir)
+    expanded_path = File.realdirpath(tainted_path, expanded_dir)
+    pattern = "#{expanded_dir}/*"
+    unless File.fnmatch?(pattern, expanded_path)
+      raise ArgumentError, "Resource path sanity check failed #{pattern}:#{expanded_path}!!!!"
+    end
+    expanded_path
+  end
+
   # Do resource pool synch, needs to be called with a Fiber context
   def synchronize_pool_with(working_dir)
     timed_section(CloudController.logger, 'process_app_resources') do
@@ -135,8 +197,8 @@ private
         pool = CloudController.resource_pool
         pool.add_directory(working_dir)
         @resource_descriptors.each do |descriptor|
-          target = File.join(working_dir, descriptor[:fn])
-          pool.copy(descriptor, target)
+          path = resolve_path(working_dir, descriptor[:fn])
+          pool.copy(descriptor, path)
         end
       end
     end

cloud_controller/config/appconfig.rb

@@ -230,3 +230,5 @@ unless AppConfig.has_key?(:allow_registration)
   $stderr.puts "Allow registration not set, defaulting to true"
   AppConfig[:allow_registration] = true
 end
+
+AppConfig[:max_droplet_size] ||= 512 * 1024 * 1024

cloud_controller/config/cloud_controller.yml

@@ -149,7 +149,7 @@ runtimes:
   ruby19:
     version: 1.9.2
   node:
-    version: 0.4.[2-9]
+    version: 0.4.12
   java:
     version: 1.6.0
     debug_modes:

cloud_controller/lib/resource_pool/filesystem_pool.rb

@@ -38,6 +38,19 @@ class FilesystemPool < ResourcePool
     true
   end
 
+  def resource_sizes(resources)
+    sizes = []
+    resources.each do |descriptor|
+      resource_path = path_from_sha1(descriptor[:sha1])
+      if File.exists?(resource_path)
+        entry = descriptor.dup
+        entry[:size] = File.size(resource_path)
+        sizes << entry
+      end
+    end
+    sizes
+  end
+
   private
 
   def overwrite_destination_with!(descriptor, destination)

cloud_controller/spec/controllers/users_controller_spec.rb

@@ -18,6 +18,7 @@ describe UsersController do
       json = Yajl::Parser.parse(response.body)
       json.should be_kind_of(Hash)
       json['email'].should == @user.email
+      json['admin'].should == @user.admin?
     end
 
     it 'should return an user info as an admin requesting for an existent user' do
@@ -29,6 +30,7 @@ describe UsersController do
       json = Yajl::Parser.parse(response.body)
       json.should be_kind_of(Hash)
       json['email'].should == @user.email
+      json['admin'].should == @user.admin?
     end
 
     it 'should return an error as an admin requesting for a non-existent user' do

cloud_controller/spec/models/app_package_spec.rb

@@ -1,10 +1,45 @@
 require 'spec_helper'
+require 'tmpdir'
 
 describe AppPackage do
   before :all do
     EM.instance_variable_set(:@next_tick_queue, [])
   end
 
+  describe '#resolve_path' do
+    before(:all) do
+      @tmpdir = Dir.mktmpdir
+      @dummy_zip = Tempfile.new('app_package_test')
+      @app_package = AppPackage.new(nil, @dummy_zip)
+    end
+
+    after(:all) do
+      FileUtils.rm_rf @tmpdir
+    end
+
+    it 'should succeed if the given path points to a file in the apps directory' do
+      testpath = File.join(@tmpdir,'testfile')
+      File.new(testpath, 'w+')
+      @app_package.resolve_path(@tmpdir, 'testfile').should == File.realdirpath(testpath)
+    end
+
+    it 'should fail if the given path does not resolve to a file in the applications directory' do
+      expect do
+       @app_package.resolve_path(@tmpdir, '../foo')
+      end.to raise_error(ArgumentError)
+    end
+
+    it 'should fail if the given path contains a symlink that points outside of the applications directory' do
+      Dir.chdir(@tmpdir) {
+        File.symlink('/etc', 'foo')
+      }
+      expect do
+       @app_package.resolve_path(@tmpdir, 'foo/bar')
+      end.to raise_error(ArgumentError)
+    end
+  end
+
+
   describe '#unpack_upload' do
     it 'should raise an instance of AppPackageError if unzip exits with a nonzero status code' do
       invalid_zip = Tempfile.new('app_package_test')
@@ -20,6 +55,96 @@ describe AppPackage do
     end
   end
 
+  describe '#get_unzipped_size' do
+    before :each do
+      @tmpdir = Dir.mktmpdir
+    end
+
+    after :each do
+      FileUtils.rm_rf(@tmpdir)
+    end
+
+    it 'should raise an instance of AppPackageError if unzip exits with a nonzero status' do
+      invalid_zip = Tempfile.new('unzipped_size_test')
+      app_package = AppPackage.new(nil, invalid_zip)
+      em do
+        Fiber.new do
+          expect do
+            app_package.send(:get_unzipped_size)
+          end.to raise_error(AppPackageError, /Failed listing/)
+          EM.stop
+        end.resume
+      end
+    end
+
+    it 'should return the total size of the unzipped droplet' do
+      [1, 5].each do |file_count|
+        zipname = File.join(@tmpdir, "test#{file_count}.zip")
+        unzipped_size = create_zip(zipname, file_count)
+        app_package = AppPackage.new(nil, File.new(zipname))
+        computed_size = nil
+        em do
+          Fiber.new do
+            computed_size = app_package.send(:get_unzipped_size)
+            EM.stop
+          end.resume
+        end
+        computed_size.should == unzipped_size
+      end
+    end
+  end
+
+  describe '#check_package_size' do
+    before :each do
+      @tmpdir = Dir.mktmpdir
+      @saved_size = AppConfig[:max_droplet_size]
+      @saved_pool = CloudController.resource_pool
+    end
+
+    after :each do
+      FileUtils.rm_rf(@tmpdir)
+      AppConfig[:max_droplet_size] = @saved_size
+      CloudController.resource_pool = @saved_pool
+    end
+
+    it 'should raise an instance of AppPackageError if the unzipped size is too large' do
+      zipname = File.join(@tmpdir, "test.zip")
+      unzipped_size = create_zip(zipname, 10, 1024)
+      app_package = AppPackage.new([], File.new(zipname))
+      AppConfig[:max_droplet_size] = unzipped_size - 1024
+      em do
+        Fiber.new do
+          expect do
+            app_package.send(:check_package_size)
+          end.to raise_error(AppPackageError, /exceeds/)
+          EM.stop
+        end.resume
+      end
+    end
+
+    it 'should raise an instance of AppPackageError if the total size is too large' do
+      CloudController.resource_pool = FilesystemPool.new(:directory => @tmpdir)
+      tf = Tempfile.new('mytemp')
+      tf.write("A" * 1024)
+      tf.close
+      CloudController.resource_pool.add_path(tf.path)
+      sha1 = Digest::SHA1.file(tf.path).hexdigest
+      zipname = File.join(@tmpdir, "test.zip")
+      unzipped_size = create_zip(zipname, 1, 1024)
+      app_package = AppPackage.new(nil, File.new(zipname), [{:sha1 => sha1, :fn => 'test/path'}])
+      AppConfig[:max_droplet_size] = unzipped_size + 512
+      em do
+        Fiber.new do
+          expect do
+            app_package.send(:check_package_size)
+          end.to raise_error(AppPackageError, /exceeds/)
+          EM.stop
+        end.resume
+      end
+
+    end
+  end
+
   describe '.blocking_defer' do
     it 'should result the result of the deferred operation' do
       deferred_result = nil
@@ -66,4 +191,17 @@ describe AppPackage do
       yield
     end
   end
+
+  def create_zip(zip_name, file_count, file_size=1024)
+    total_size = file_count * file_size
+    file_paths = []
+    file_count.times do |ii|
+      tf = Tempfile.new("ziptest_#{ii}")
+      file_paths << tf.path
+      tf.write("A" * file_size)
+      tf.close
+    end
+    system("zip #{zip_name} #{file_paths.join(' ')}").should be_true
+    total_size
+  end
 end

common/.gitignore

@@ -0,0 +1 @@
+/*.gem

common/Gemfile

@@ -4,5 +4,5 @@ gemspec
 
 group :spec do
   gem 'rspec'
-  gem 'em-http-request', '~> 1.0.0.beta3'
+  gem 'em-http-request' #, '~> 1.0.0.beta3'
 end

common/Gemfile.lock

@@ -1,49 +1,47 @@
 PATH
   remote: .
   specs:
-    vcap_common (0.99)
-      eventmachine (~> 0.12.11.cloudfoundry.2)
+    vcap_common (1.0.3)
+      eventmachine (~> 0.12.11.cloudfoundry.3)
       logging (>= 1.5.0)
-      nats
-      posix-spawn
-      thin
-      yajl-ruby
+      nats (~> 0.4.22.beta.4)
+      posix-spawn (~> 0.3.6)
+      thin (~> 1.3.1)
+      yajl-ruby (~> 0.8.3)
 
 GEM
   remote: http://rubygems.org/
   specs:
-    addressable (2.2.4)
-    daemons (1.1.4)
-    diff-lcs (1.1.2)
-    em-http-request (1.0.0.beta.3)
-      addressable (>= 2.2.3)
-      em-socksify
-      eventmachine
-      http_parser.rb (>= 0.5.1)
-    em-socksify (0.1.0)
-      eventmachine
-    eventmachine (0.12.11.cloudfoundry.2)
-    http_parser.rb (0.5.1)
-    json_pure (1.5.3)
-    little-plugger (1.1.2)
-    logging (1.6.0)
+    addressable (2.2.6)
+    daemons (1.1.5)
+    diff-lcs (1.1.3)
+    em-http-request (0.3.0)
+      addressable (>= 2.0.0)
+      escape_utils
+      eventmachine (>= 0.12.9)
+    escape_utils (0.2.4)
+    eventmachine (0.12.11.cloudfoundry.3)
+    json_pure (1.6.4)
+    little-plugger (1.1.3)
+    logging (1.6.1)
       little-plugger (>= 1.1.2)
-    nats (0.4.10)
-      daemons (>= 1.1.0)
+    nats (0.4.22.beta.4)
+      daemons (>= 1.1.4)
       eventmachine (>= 0.12.10)
-      json_pure (>= 1.5.1)
+      json_pure (>= 1.6.1)
+      thin (>= 1.3.1)
     posix-spawn (0.3.6)
-    rack (1.3.2)
+    rack (1.4.0)
     rake (0.9.2.2)
-    rspec (2.5.0)
-      rspec-core (~> 2.5.0)
-      rspec-expectations (~> 2.5.0)
-      rspec-mocks (~> 2.5.0)
-    rspec-core (2.5.1)
-    rspec-expectations (2.5.0)
+    rspec (2.7.0)
+      rspec-core (~> 2.7.0)
+      rspec-expectations (~> 2.7.0)
+      rspec-mocks (~> 2.7.0)
+    rspec-core (2.7.1)
+    rspec-expectations (2.7.0)
       diff-lcs (~> 1.1.2)
-    rspec-mocks (2.5.0)
-    thin (1.2.11)
+    rspec-mocks (2.7.0)
+    thin (1.3.1)
       daemons (>= 1.0.9)
       eventmachine (>= 0.12.6)
       rack (>= 1.0.0)
@@ -53,7 +51,7 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
-  em-http-request (~> 1.0.0.beta3)
-  rake
+  em-http-request
+  rake (~> 0.9.2)
   rspec
   vcap_common!

common/vcap_common.gemspec

@@ -1,6 +1,6 @@
 spec = Gem::Specification.new do |s|
   s.name = 'vcap_common'
-  s.version = 0.99
+  s.version = '1.0.3'
   s.date = '2011-02-09'
   s.summary = 'vcap common'
   s.homepage = "http://github.com/vmware-ac/core"
@@ -9,13 +9,13 @@ spec = Gem::Specification.new do |s|
   s.authors = ["Derek Collison"]
   s.email = ["derek.collison@gmail.com"]
 
-  s.add_dependency('eventmachine', '~> 0.12.11.cloudfoundry.2')
-  s.add_dependency('thin')
-  s.add_dependency('yajl-ruby')
-  s.add_dependency('nats')
+  s.add_dependency('eventmachine', '~> 0.12.11.cloudfoundry.3')
+  s.add_dependency('thin', '~> 1.3.1')
+  s.add_dependency('yajl-ruby', '~> 0.8.3')
+  s.add_dependency('nats', '~> 0.4.22.beta.4')
   s.add_dependency('logging', '>= 1.5.0')
-  s.add_dependency('posix-spawn')
-  s.add_development_dependency('rake')
+  s.add_dependency('posix-spawn', '~> 0.3.6')
+  s.add_development_dependency('rake', '~> 0.9.2')
 
   s.require_paths = ['lib']
 

dea/Gemfile

@@ -1,8 +1,8 @@
 source "http://rubygems.org"
 
 gem 'bundler', '>= 1.0.10'
-gem 'nats', '>= 0.4.10', :require => 'nats/client'
-gem 'eventmachine',  '~> 0.12.11.cloudfoundry.2'
+gem 'nats', :require => 'nats/client'
+gem 'eventmachine'
 gem 'em-http-request', '~> 1.0.0.beta.3', :require => 'em-http'
 
 gem 'rack', :require => ["rack/utils", "rack/mime"]
@@ -11,7 +11,7 @@ gem 'thin'
 gem 'yajl-ruby', :require => ['yajl', 'yajl/json_gem']
 gem 'logging', '>= 1.5.0'
 
-gem 'vcap_common', :path => '../common'
+gem 'vcap_common', '~> 1.0.3'
 gem 'vcap_logging', :require => ['vcap/logging']
 
 group :test do

dea/Gemfile.lock

@@ -1,14 +1,3 @@
-PATH
-  remote: ../common
-  specs:
-    vcap_common (0.99)
-      eventmachine (~> 0.12.11.cloudfoundry.2)
-      logging (>= 1.5.0)
-      nats
-      posix-spawn
-      thin
-      yajl-ruby
-
 GEM
   remote: http://rubygems.org/
   specs:
@@ -16,7 +5,7 @@ GEM
     builder (3.0.0)
     ci_reporter (1.6.4)
       builder (>= 2.1.2)
-    daemons (1.1.2)
+    daemons (1.1.5)
     diff-lcs (1.1.2)
     em-http-request (1.0.0.beta.3)
       addressable (>= 2.2.3)
@@ -25,18 +14,19 @@ GEM
       http_parser.rb (>= 0.5.1)
     em-socksify (0.1.0)
       eventmachine
-    eventmachine (0.12.11.cloudfoundry.2)
+    eventmachine (0.12.11.cloudfoundry.3)
     http_parser.rb (0.5.1)
-    json_pure (1.5.1)
-    little-plugger (1.1.2)
-    logging (1.5.0)
+    json_pure (1.6.4)
+    little-plugger (1.1.3)
+    logging (1.6.1)
       little-plugger (>= 1.1.2)
-    nats (0.4.10)
-      daemons (>= 1.1.0)
+    nats (0.4.22.beta.4)
+      daemons (>= 1.1.4)
       eventmachine (>= 0.12.10)
-      json_pure (>= 1.5.1)
+      json_pure (>= 1.6.1)
+      thin (>= 1.3.1)
     posix-spawn (0.3.6)
-    rack (1.2.2)
+    rack (1.4.0)
     rake (0.8.7)
     rcov (0.9.9)
     rspec (2.5.0)
@@ -47,12 +37,19 @@ GEM
     rspec-expectations (2.5.0)
       diff-lcs (~> 1.1.2)
     rspec-mocks (2.5.0)
-    thin (1.2.11)
+    thin (1.3.1)
       daemons (>= 1.0.9)
       eventmachine (>= 0.12.6)
       rack (>= 1.0.0)
+    vcap_common (1.0.3)
+      eventmachine (~> 0.12.11.cloudfoundry.3)
+      logging (>= 1.5.0)
+      nats (~> 0.4.22.beta.4)
+      posix-spawn (~> 0.3.6)
+      thin (~> 1.3.1)
+      yajl-ruby (~> 0.8.3)
     vcap_logging (0.1.3)
-    yajl-ruby (0.8.2)
+    yajl-ruby (0.8.3)
 
 PLATFORMS
   ruby
@@ -61,14 +58,14 @@ DEPENDENCIES
   bundler (>= 1.0.10)
   ci_reporter
   em-http-request (~> 1.0.0.beta.3)
-  eventmachine (~> 0.12.11.cloudfoundry.2)
+  eventmachine
   logging (>= 1.5.0)
-  nats (>= 0.4.10)
+  nats
   rack
   rake
   rcov
   rspec
   thin
-  vcap_common!
+  vcap_common (~> 1.0.3)
   vcap_logging
   yajl-ruby

dea/config/dea.yml

@@ -55,7 +55,7 @@ runtimes:
     environment:
   node:
     executable: node
-    version: 0.4.[2-9]
+    version: 0.4.12
     version_flag: '-v'
     environment:
   java:

dea/config/dea2.yml

@@ -31,7 +31,7 @@ runtimes:
     environment:
   node:
     executable: node
-    version: 0.4.[2-9]
+    version: 0.4.12
     version_flag: '-v'
     environment:
   java:

dea/lib/dea/agent.rb

@@ -569,25 +569,33 @@ module DEA
         instance[:secure_user] = user[:user]
       end
 
-      start_operation = proc do
+      start_operation = lambda do
         @logger.debug('Completed download')
 
-        port = VCAP.grab_ephemeral_port
-        instance[:port] = port
-
-        starting = "Starting up instance #{instance[:log_id]} on port:#{port}"
-
-        if debug
-          debug_port = VCAP.grab_ephemeral_port
-          instance[:debug_ip] = VCAP.local_ip
-          instance[:debug_port] = debug_port
-          instance[:debug_mode] = debug
-
-          @logger.info("#{starting} with debugger:#{debug_port}")
+        port = grab_port
+        if port
+          instance[:port] = port
         else
-          @logger.info(starting)
+          @logger.warn("Unable to allocate port for instance#{instance[:log_id]}")
+          stop_droplet(instance)
+          return
         end
 
+        if debug
+          debug_port = grab_port
+          if debug_port
+            instance[:debug_ip] = VCAP.local_ip
+            instance[:debug_port] = debug_port
+            instance[:debug_mode] = debug
+          else
+            @logger.warn("Unable to allocate debug port for instance#{instance[:log_id]}")
+            stop_droplet(instance)
+            return
+          end
+        end
+
+        @logger.info("Starting up instance #{instance[:log_id]} on port:#{instance[:port]} " +
+                     "#{"debuger:" if instance[:debug_port]}#{instance[:debug_port]}")
         @logger.debug("Clients: #{@num_clients}")
         @logger.debug("Reserved Memory Usage: #{@reserved_mem} MB of #{@max_memory} MB TOTAL")
 
@@ -644,7 +652,7 @@ module DEA
             process.send_data("umask 077\n")
           end
           app_env.each { |env| process.send_data("export #{env}\n") }
-          process.send_data("./startup -p #{port}\n")
+          process.send_data("./startup -p #{instance[:port]}\n")
           process.send_data("exit\n")
         end
 
@@ -845,13 +853,8 @@ module DEA
       (instance[:mem_quota] / (1024*1024)).to_i
     end
 
-    def grab_ephemeral_port
-      socket = TCPServer.new('0.0.0.0', 0)
-      socket.setsockopt(Socket::SOL_SOCKET, Socket::SO_REUSEADDR, true)
-      Socket.do_not_reverse_lookup = true
-      port = socket.addr[1]
-      socket.close
-      return port
+    def grab_port
+      VCAP.grab_ephemeral_port
     end
 
     def detect_app_ready(instance, manifest, &block)

dev_setup/README

@@ -63,5 +63,12 @@ o To use a custom deployment config
    To Stop:
       $HOME/projects/vcap/dev_setup/bin/vcap_dev -d $HOME/projects -n dea stop
 
+o To use a custom domain
+  e.g. If you do not want your CloudFoundry domain as vcap.me
+  $HOME/projects/vcap/dev_setup/bin/vcap_dev_setup -D myowndomain.com
+
+  Later, you will target your CloudFoundry installation with:
+    vmc target api.myowndomain.com
+
 NOTE: To learn more about custom deployment config files and multi host setups
 see the README file vcap/dev_setup/deployments/README.

dev_setup/bin/vcap

@@ -146,7 +146,7 @@ class Component
         # should help for now.
         if is_cloud_controller?
           cc_dir = File.expand_path(File.join(DIR, '..', 'cloud_controller'))
-          Dir.chdir(cc_dir) { `rake db:migrate` }
+          Dir.chdir(cc_dir) { `bundle exec rake db:migrate` }
         end
         exec("#{component_start_path}")
       end
@@ -299,7 +299,7 @@ module Run
     end
 
     cc_dir = File.expand_path(File.join(DIR, '..', 'cloud_controller'))
-    run_command("Resetting the CloudController database", "cd #{cc_dir} 2>&1 && rake db:drop 2>&1")
+    run_command("Resetting the CloudController database", "cd #{cc_dir} 2>&1 && bundle exec rake db:drop 2>&1")
     puts
 
     cc_log_dir = File.join(cc_dir, 'log')

dev_setup/bin/vcap_dev_setup

@@ -12,6 +12,7 @@ OPTIONS:
   -p           http proxy i.e. -p http://username:password@host:port/
   -c           deployment config
   -d           cloudfoundry home
+  -D           cloudfoundry domain (default: vcap.me)
   -r           cloud foundry repo
   -b           cloud foundry repo branch/tag/SHA
 EOF
@@ -27,7 +28,7 @@ fi
 
 APT_CONFIG="-o Acquire::http::No-Cache=True -o Acquire::BrokenProxy=true -o Acquire::Retries=3"
 
-while getopts "had:p:c:r:b:" OPTION
+while getopts "had:p:c:D:r:b:" OPTION
 do
   case $OPTION in
     h)
@@ -43,6 +44,9 @@ do
     d)
       CLOUDFOUNDRY_HOME=$OPTARG
       ;;
+    D)
+      CLOUDFOUNDRY_DOMAIN=$OPTARG
+      ;;
     r)
       VCAP_REPO=$OPTARG
       ;;
@@ -60,6 +64,10 @@ if [ -z "$CLOUDFOUNDRY_HOME" ]; then
   CLOUDFOUNDRY_HOME=~/cloudfoundry
 fi
 
+if [ -z "$CLOUDFOUNDRY_DOMAIN" ]; then
+  CLOUDFOUNDRY_DOMAIN=vcap.me
+fi
+
 if [ -z "$VCAP_REPO" ]; then
   VCAP_REPO=https://github.com/cloudfoundry/vcap.git
 fi
@@ -130,8 +138,12 @@ if [ -n "$CLOUDFOUNDRY_HOME" ]; then
   ARGS="-d $CLOUDFOUNDRY_HOME"
 fi
 
+if [ -n "$CLOUDFOUNDRY_DOMAIN" ]; then
+  ARGS="$ARGS -D $CLOUDFOUNDRY_DOMAIN"
+fi
+
 if [ -n "$CONFIG_FILE" ]; then
   ARGS="$ARGS -c $CONFIG_FILE"
 fi
 
-$CLOUDFOUNDRY_HOME/vcap/dev_setup/lib/chefsolo_launch.rb $ARGS
+$CLOUDFOUNDRY_HOME/vcap/dev_setup/lib/chefsolo_launch.rb $ARGS

dev_setup/cookbooks/cloud_controller/templates/default/cloud_controller.yml.erb

@@ -112,7 +112,7 @@ runtimes:
   ruby19:
     version: 1.9.2
   node:
-    version: 0.4.[2-9]
+    version: 0.4.12
   java:
     version: 1.6.0
     debug_modes:

dev_setup/cookbooks/cloud_controller/templates/default/node.yml.erb

@@ -2,7 +2,7 @@
 name: "node"
 runtimes:
   - node:
-      version: '0.4.5'
+      version: '0.4.12'
       description: 'Node.js'
       executable: <%= File.join(node[:nodejs][:path], "bin", "node") %>
       default: true

dev_setup/cookbooks/dea/templates/default/dea.yml.erb

@@ -62,7 +62,7 @@ runtimes:
 <% if node[:dea][:runtimes].include?("nodejs") %>
   node:
     executable: <%= File.join(node[:nodejs][:path], "bin", "node") %>
-    version: 0.4.[2-9]
+    version: 0.4.12
     version_flag: '-v'
     environment:
 <% end %>

dev_setup/cookbooks/nginx/templates/default/ubuntu-nginx.conf.erb

@@ -44,10 +44,9 @@ http {
     server <%= node[:nginx][:uls_ip] %>:<%= node[:nginx][:uls_port] %>;
   }
 
-
   server {
     listen       80;
-    server_name  <%= node[:deployment][:domain] %>;
+    server_name  _;
     server_name_in_redirect off;
 
     #TODO: how to make this internal location totally transparent to outside
@@ -83,9 +82,9 @@ http {
         # if we have multiple workers, there will be still few stats not synced for
         # the workers which don't get this monitor request.
         access_by_lua '
-	  -- add package.path and package.cpath
-	  package.path = package.path..";<%= node[:lua][:module_path] %>/?.lua"
-	  package.cpath = package.cpath..";<%= node[:lua][:module_path] %>/?.so"
+          -- add package.path and package.cpath
+          package.path = package.path..";<%= node[:lua][:module_path] %>/?.lua"
+          package.cpath = package.cpath..";<%= node[:lua][:module_path] %>/?.so"
 
           local uls = require ("uls")
           local cjson = require("cjson")
@@ -122,9 +121,9 @@ http {
         set $trace '';
 
         access_by_lua '
-	  -- add package.path and package.cpath
-	  package.path = package.path..";<%= node[:lua][:module_path] %>/?.lua"
-	  package.cpath = package.cpath..";<%= node[:lua][:module_path] %>/?.so"
+          -- add package.path and package.cpath
+          package.path = package.path..";<%= node[:lua][:module_path] %>/?.lua"
+          package.cpath = package.cpath..";<%= node[:lua][:module_path] %>/?.so"
 
           local uls = require ("uls")
           local cjson = require("cjson")
@@ -175,9 +174,9 @@ http {
 
         # Handling response from backend servers
         header_filter_by_lua '
-	  -- add package.path and package.cpath
-	  package.path = package.path..";<%= node[:lua][:module_path] %>/?.lua"
-	  package.cpath = package.cpath..";<%= node[:lua][:module_path] %>/?.so"
+          -- add package.path and package.cpath
+          package.path = package.path..";<%= node[:lua][:module_path] %>/?.lua"
+          package.cpath = package.cpath..";<%= node[:lua][:module_path] %>/?.so"
 
           local uls = require ("uls")
 

dev_setup/cookbooks/nodejs/attributes/default.rb

@@ -1,4 +1,4 @@
 include_attribute "deployment"
-default[:nodejs][:version] = "0.4.8"
+default[:nodejs][:version] = "0.4.12"
 default[:nodejs][:path] = File.join(node[:deployment][:home], "deploy", "nodejs")
 default[:nodejs][:source] = "http://nodejs.org/dist/node-v#{nodejs[:version]}.tar.gz"

dev_setup/cookbooks/ruby/libraries/ruby_install.rb

@@ -0,0 +1,83 @@
+module RubyInstall
+  def cf_ruby_install(ruby_version, ruby_source, ruby_path)
+    rubygems_version = node[:rubygems][:version]
+    bundler_version = node[:rubygems][:bundler][:version]
+    rake_version = node[:rubygems][:rake][:version]
+
+    %w[ build-essential libssl-dev zlib1g-dev libreadline5-dev libxml2-dev libpq-dev].each do |pkg|
+      package pkg
+    end
+
+    remote_file File.join("", "tmp", "ruby-#{ruby_version}.tar.gz") do
+      owner node[:deployment][:user]
+      source ruby_source
+      not_if { ::File.exists?(File.join("", "tmp", "ruby-#{ruby_version}.tar.gz")) }
+    end
+
+    directory ruby_path do
+      owner node[:deployment][:user]
+      group node[:deployment][:group]
+      mode "0755"
+      recursive true
+      action :create
+    end
+
+    bash "Install Ruby #{ruby_path}" do
+      cwd File.join("", "tmp")
+      user node[:deployment][:user]
+      code <<-EOH
+      tar xzf ruby-#{ruby_version}.tar.gz
+      cd ruby-#{ruby_version}
+      ./configure --disable-pthread --prefix=#{ruby_path}
+      make
+      make install
+      EOH
+      not_if do
+        ::File.exists?(File.join(ruby_path, "bin", "ruby"))
+      end
+    end
+
+    remote_file File.join("", "tmp", "rubygems-#{rubygems_version}.tgz") do
+      owner node[:deployment][:user]
+      source "http://production.cf.rubygems.org/rubygems/rubygems-#{rubygems_version}.tgz"
+      not_if { ::File.exists?(File.join("", "tmp", "rubygems-#{rubygems_version}.tgz")) }
+    end
+
+    bash "Install RubyGems #{ruby_path}" do
+      cwd File.join("", "tmp")
+      user node[:deployment][:user]
+      code <<-EOH
+      tar xzf rubygems-#{rubygems_version}.tgz
+      cd rubygems-#{rubygems_version}
+      #{File.join(ruby_path, "bin", "ruby")} setup.rb
+      EOH
+      not_if do
+        ::File.exists?(File.join(ruby_path, "bin", "gem")) &&
+            system("#{File.join(ruby_path, "bin", "gem")} -v | grep -q '#{rubygems_version}$'")
+      end
+    end
+
+    gem_package "bundler" do
+      version bundler_version
+      gem_binary File.join(ruby_path, "bin", "gem")
+    end
+
+    gem_package "rake" do
+      version rake_version
+      gem_binary File.join(ruby_path, "bin", "gem")
+    end
+
+    # The default chef installed with Ubuntu 10.04 does not support the "retries" option
+    # for gem_package. It may be a good idea to add/use that option once the ubuntu
+    # chef package gets updated.
+    %w[ rack eventmachine thin sinatra mysql pg vmc ].each do |gem|
+      gem_package gem do
+        gem_binary File.join(ruby_path, "bin", "gem")
+      end
+    end
+  end
+end
+
+class Chef::Recipe
+  include RubyInstall
+end

dev_setup/cookbooks/ruby/recipes/default.rb

@@ -1,78 +1,6 @@
+# convenience variables
 ruby_version = node[:ruby][:version]
 ruby_source = node[:ruby][:source]
 ruby_path = node[:ruby][:path]
-rubygems_version = node[:rubygems][:version]
-bundler_version = node[:rubygems][:bundler][:version]
-rake_version = node[:rubygems][:rake][:version]
 
-%w[ build-essential libssl-dev zlib1g-dev libreadline5-dev libxml2-dev libpq-dev].each do |pkg|
-  package pkg
-end
-
-remote_file File.join("", "tmp", "ruby-#{ruby_version}.tar.gz") do
-  owner node[:deployment][:user]
-  source ruby_source
-  not_if { ::File.exists?(File.join("", "tmp", "ruby-#{ruby_version}.tar.gz")) }
-end
-
-directory ruby_path do
-  owner node[:deployment][:user]
-  group node[:deployment][:group]
-  mode "0755"
-  recursive true
-  action :create
-end
-
-bash "Install Ruby" do
-  cwd File.join("", "tmp")
-  user node[:deployment][:user]
-  code <<-EOH
-  tar xzf ruby-#{ruby_version}.tar.gz
-  cd ruby-#{ruby_version}
-  ./configure --disable-pthread --prefix=#{ruby_path}
-  make
-  make install
-  EOH
-  not_if do
-    ::File.exists?(File.join(ruby_path, "bin", "ruby"))
-  end
-end
-
-remote_file File.join("", "tmp", "rubygems-#{rubygems_version}.tgz") do
-  owner node[:deployment][:user]
-  source "http://production.cf.rubygems.org/rubygems/rubygems-#{rubygems_version}.tgz"
-  not_if { ::File.exists?(File.join("", "tmp", "rubygems-#{rubygems_version}.tgz")) }
-end
-
-bash "Install RubyGems" do
-  cwd File.join("", "tmp")
-  user node[:deployment][:user]
-  code <<-EOH
-  tar xzf rubygems-#{rubygems_version}.tgz
-  cd rubygems-#{rubygems_version}
-  #{File.join(ruby_path, "bin", "ruby")} setup.rb
-  EOH
-  not_if do
-    ::File.exists?(File.join(ruby_path, "bin", "gem")) &&
-        system("#{File.join(ruby_path, "bin", "gem")} -v | grep -q '#{rubygems_version}$'")
-  end
-end
-
-gem_package "bundler" do
-  version bundler_version
-  gem_binary File.join(ruby_path, "bin", "gem")
-end
-
-gem_package "rake" do
-  version rake_version
-  gem_binary File.join(ruby_path, "bin", "gem")
-end
-
-# The default chef installed with Ubuntu 10.04 does not support the "retries" option
-# for gem_package. It may be a good idea to add/use that option once the ubuntu
-# chef package gets updated.
-%w[ rack eventmachine thin sinatra mysql pg vmc ].each do |gem|
-  gem_package gem do
-    gem_binary File.join(ruby_path, "bin", "gem")
-  end
-end
+cf_ruby_install(ruby_version, ruby_source, ruby_path)

dev_setup/cookbooks/ruby/recipes/ruby18.rb

@@ -1,13 +1,6 @@
-orig_version = node[:ruby][:version]
-orig_source = node[:ruby][:source]
-orig_path = node[:ruby][:path]
+# convenience variables
+ruby_version = node[:ruby18][:version]
+ruby_source = node[:ruby18][:source]
+ruby_path = node[:ruby18][:path]
 
-node[:ruby][:version] = node[:ruby18][:version]
-node[:ruby][:source] = node[:ruby18][:source]
-node[:ruby][:path] = node[:ruby18][:path]
-
-include_recipe "ruby::default"
-
-node[:ruby][:version] = orig_version
-node[:ruby][:source] = orig_source
-node[:ruby][:path] = orig_path
+cf_ruby_install(ruby_version, ruby_source, ruby_path)

dev_setup/lib/chefsolo_launch.rb

@@ -15,12 +15,14 @@ require File.expand_path('job_manager', File.dirname(__FILE__))
 
 script_dir = File.expand_path(File.dirname(__FILE__))
 cloudfoundry_home = Deployment.get_cloudfoundry_home
+cloudfoundry_domain = Deployment.get_cloudfoundry_domain
 deployment_spec = File.expand_path(File.join(script_dir, "..", DEPLOYMENT_DEFAULT_SPEC))
 
 args = ARGV.dup
 opts_parser = OptionParser.new do |opts|
   opts.on('--config CONFIG_FILE', '-c CONFIG_FILE') { |file| deployment_spec = File.expand_path(file.to_s) }
   opts.on('--dir CLOUDFOUNDRY_HOME', '-d CLOUDFOUNDRY_HOME') { |dir| cloudfoundry_home = File.expand_path(dir.to_s) }
+  opts.on('--domain CLOUDFOUNDRY_DOMAIN', '-D CLOUDFOUNDRY_DOMAIN') { |domain| cloudfoundry_domain = domain }
 end
 args = opts_parser.parse!(args)
 
@@ -37,6 +39,7 @@ spec["deployment"] ||= {}
 spec["deployment"]["name"] ||= DEPLOYMENT_DEFAULT_NAME
 spec["deployment"]["user"] ||= ENV["USER"]
 spec["deployment"]["group"] ||= `id -g`.strip
+spec["deployment"]["domain"] ||= cloudfoundry_domain
 spec["cloudfoundry"] ||= {}
 spec["cloudfoundry"]["home"] ||= cloudfoundry_home
 spec["cloudfoundry"]["home"] = File.expand_path(spec["cloudfoundry"]["home"])

dev_setup/lib/vcap_defs.rb

@@ -34,6 +34,7 @@ end
 
 DEPLOYMENT_DEFAULT_SPEC = File.join("deployments", "devbox.yml")
 DEPLOYMENT_DEFAULT_NAME = "devbox"
+DEPLOYMENT_DEFAULT_DOMAIN = "vcap.me"
 DEPLOYMENT_CONFIG_DIR_NAME = "config"
 DEPLOYMENT_CONFIG_FILE_NAME = "deploy.json"
 DEPLOYMENT_VCAP_CONFIG_FILE_NAME = "vcap_components.json"
@@ -48,6 +49,10 @@ class Deployment
       File.expand_path(File.join(ENV["HOME"], "cloudfoundry"))
     end
 
+    def get_cloudfoundry_domain
+      DEPLOYMENT_DEFAULT_DOMAIN
+    end
+
     def get_config_path(name, cloudfoundry_home=nil)
       cloudfoundry_home ||= get_cloudfoundry_home
       File.expand_path(File.join(cloudfoundry_home, ".deployments", name, DEPLOYMENT_CONFIG_DIR_NAME))

health_manager/Gemfile

@@ -1,8 +1,8 @@
 source "http://rubygems.org"
 
 gem 'bundler', '>= 1.0.10'
-gem 'nats', '>= 0.4.10', :require => 'nats/client'
-gem 'eventmachine',  '~> 0.12.11.cloudfoundry.2'
+gem 'nats', :require => 'nats/client'
+gem 'eventmachine'
 gem 'em-http-request', '~> 1.0.0.beta.3', :require => 'em-http'
 
 gem 'rack', :require => ["rack/utils", "rack/mime"]
@@ -11,7 +11,7 @@ gem 'thin'
 gem 'yajl-ruby', :require => ['yajl', 'yajl/json_gem']
 gem 'logging', '>= 1.5.0'
 
-gem 'vcap_common', :path => '../common'
+gem 'vcap_common', '~> 1.0.3'
 gem 'vcap_logging', :require => ['vcap/logging']
 
 group :test do

health_manager/Gemfile.lock

@@ -1,14 +1,3 @@
-PATH
-  remote: ../common
-  specs:
-    vcap_common (0.99)
-      eventmachine (~> 0.12.11.cloudfoundry.2)
-      logging (>= 1.5.0)
-      nats
-      posix-spawn
-      thin
-      yajl-ruby
-
 GEM
   remote: http://rubygems.org/
   specs:
@@ -16,7 +5,7 @@ GEM
     builder (3.0.0)
     ci_reporter (1.6.5)
       builder (>= 2.1.2)
-    daemons (1.1.4)
+    daemons (1.1.5)
     diff-lcs (1.1.3)
     em-http-request (1.0.0.beta.3)
       addressable (>= 2.2.3)
@@ -25,18 +14,19 @@ GEM
       http_parser.rb (>= 0.5.1)
     em-socksify (0.1.0)
       eventmachine
-    eventmachine (0.12.11.cloudfoundry.2)
+    eventmachine (0.12.11.cloudfoundry.3)
     http_parser.rb (0.5.3)
-    json_pure (1.6.1)
-    little-plugger (1.1.2)
+    json_pure (1.6.4)
+    little-plugger (1.1.3)
     logging (1.6.1)
       little-plugger (>= 1.1.2)
-    nats (0.4.10)
-      daemons (>= 1.1.0)
+    nats (0.4.22.beta.4)
+      daemons (>= 1.1.4)
       eventmachine (>= 0.12.10)
-      json_pure (>= 1.5.1)
+      json_pure (>= 1.6.1)
+      thin (>= 1.3.1)
     posix-spawn (0.3.6)
-    rack (1.3.4)
+    rack (1.4.0)
     rake (0.9.2)
     rcov (0.9.10)
     rspec (2.6.0)
@@ -47,12 +37,19 @@ GEM
     rspec-expectations (2.6.0)
       diff-lcs (~> 1.1.2)
     rspec-mocks (2.6.0)
-    thin (1.2.11)
+    thin (1.3.1)
       daemons (>= 1.0.9)
       eventmachine (>= 0.12.6)
       rack (>= 1.0.0)
+    vcap_common (1.0.3)
+      eventmachine (~> 0.12.11.cloudfoundry.3)
+      logging (>= 1.5.0)
+      nats (~> 0.4.22.beta.4)
+      posix-spawn (~> 0.3.6)
+      thin (~> 1.3.1)
+      yajl-ruby (~> 0.8.3)
     vcap_logging (0.1.3)
-    yajl-ruby (1.0.0)
+    yajl-ruby (0.8.3)
 
 PLATFORMS
   ruby
@@ -61,14 +58,14 @@ DEPENDENCIES
   bundler (>= 1.0.10)
   ci_reporter
   em-http-request (~> 1.0.0.beta.3)
-  eventmachine (~> 0.12.11.cloudfoundry.2)
+  eventmachine
   logging (>= 1.5.0)
-  nats (>= 0.4.10)
+  nats
   rack
   rake
   rcov
   rspec
   thin
-  vcap_common!
+  vcap_common (~> 1.0.3)
   vcap_logging
   yajl-ruby

health_manager/config/health_manager.yml

@@ -51,10 +51,10 @@ intervals:
   # Time to wait before analyzing the state of an application that has been
   # started/restarted
   stable_state:         60
-  # Minimum time between sending start requests. This is the rate at which start requests
-  # are dequed.  Default value is 0.02 (20ms)
-  # Set this value to 0 to revert to non-queuing behavior
-  request_queue:        0.02
+
+#number of start requests send each second (subject to EM timer limitations)
+#default value is 50.
+dequeueing_rate: 50
 
 # Used for /healthz and /vars endpoints. If not provided random
 # values will be generated on component start. Uncomment to use

health_manager/lib/health_manager.rb

@@ -77,6 +77,10 @@ class HealthManager
   RUNNING_STATES    = Set.new([STARTING, RUNNING])
   RESTART_REASONS   = Set.new([CRASHED, DEA_SHUTDOWN, DEA_EVACUATION])
 
+
+  INFINITE_PRIORITY = 2_000_000_000
+
+
   def self.start(options)
     health_manager = new(options)
     health_manager.run
@@ -95,7 +99,7 @@ class HealthManager
     @restart_timeout = config['intervals']['restart_timeout']
     @stable_state = config['intervals']['stable_state']
     @nats_ping = config['intervals']['nats_ping'] || 10
-    @request_queue_interval = config['intervals']['request_queue'] || 0.02
+    @dequeueing_rate = config['dequeueing_rate'] || 50
     @database_environment = config['database_environment']
 
     @droplets = {}
@@ -149,6 +153,7 @@ class HealthManager
       @logger.error "Eventmachine problem, #{e}"
       @logger.error("#{e.backtrace.join("\n")}")
       @logger.error(e)
+      exit!
     end
 
     NATS.start(:uri => @config['mbus']) do
@@ -412,8 +417,11 @@ class HealthManager
               index_entry[:state] = DOWN
               index_entry[:state_timestamp] = Time.now.to_i
               index_entry[:last_action] = now
+
+              high_priority = (exit_message['reason'] == DEA_EVACUATION)
+
               @logger.info("Preparing to start instance (app_id=#{droplet_id}, index=#{index}). Reason: Instance exited with reason '#{exit_message['reason']}'.")
-              start_instances(droplet_id, [index])
+              start_instances(droplet_id, [index], high_priority)
             end
           end
         end
@@ -632,7 +640,7 @@ class HealthManager
     entry_updated
   end
 
-  def start_instances(droplet_id, indices)
+  def start_instances(droplet_id, indices, high_priority = false)
     droplet_entry = @droplets[droplet_id]
     start_message = {
       :droplet => droplet_id,
@@ -643,28 +651,25 @@ class HealthManager
     }
 
     if queue_requests?
-      queue_request(start_message)
+      queue_request(start_message, high_priority)
     else
       #old behavior: send the message immediately
       NATS.publish('cloudcontrollers.hm.requests', start_message.to_json)
       @logger.info("Requesting the start of extra instances: #{start_message}")
     end
-
-
   end
 
-  def queue_request message
+  def queue_request(message, high_priority)
     #the priority is higher for older items, to de-prioritize flapping items
     priority = Time.now.to_i - message[:last_updated]
     priority = 0 if priority < 0 #avoid timezone drama
-
+    priority = INFINITE_PRIORITY if high_priority
     key = message.clone
     key.delete :last_updated
     @logger.info("Queueing priority '#{priority}' request: #{message}, using key: #{key}.  Queue size: #{@request_queue.size}")
     @request_queue.insert(message, priority, key)
   end
 
-
   def stop_instances(droplet_id, instances)
     droplet_entry = @droplets[droplet_id]
     last_updated = droplet_entry ? droplet_entry[:last_updated] : 0
@@ -707,14 +712,20 @@ class HealthManager
     end
 
     if queue_requests?
-      EM.add_periodic_timer(@request_queue_interval) do
-        unless @request_queue.empty?
-          #TODO: if STOP requests are also queued, refactor this to be generic, particularly the log message
-          start_message = encode_json(@request_queue.remove)
-          NATS.publish('cloudcontrollers.hm.requests', start_message)
-          @logger.info("Requesting the start of missing instances: #{start_message}")
-          VCAP::Component.varz[:queue_length] = @request_queue.size
-        end
+      EM.add_periodic_timer(1) do
+        deque_a_batch_of_requests(@dequeueing_rate)
+      end
+    end
+  end
+
+  def deque_a_batch_of_requests(num_requests)
+    num_requests.times do
+      unless @request_queue.empty?
+        #TODO: if STOP requests are also queued, refactor this to be generic, particularly the log message
+        start_message = encode_json(@request_queue.remove)
+        NATS.publish('cloudcontrollers.hm.requests', start_message)
+        @logger.info("Requesting the start of missing instances: #{start_message}")
+        VCAP::Component.varz[:queue_length] = @request_queue.size
       end
     end
   end
@@ -800,7 +811,7 @@ class HealthManager
   end
 
   def queue_requests?
-    @request_queue_interval != 0
+    @dequeueing_rate != 0
   end
 end
 

health_manager/spec/health_manager_spec.rb

@@ -59,9 +59,10 @@ describe HealthManager do
         'flapping_death' => 3,
         'flapping_timeout' => 5,
         'restart_timeout' => 2,
-        'stable_state' => 1,
-        'request_queue' => 0
-      },
+        'stable_state' => -1,
+
+                              },
+      'dequeueing_rate' => 0,
       'rails_environment' => 'test',
       'database_environment' => {
         'test' => {
@@ -140,7 +141,7 @@ describe HealthManager do
   it "should detect instances that are down and send a START request" do
     stats = { :frameworks => {}, :runtimes => {}, :down => 0 }
     should_publish_to_nats "cloudcontrollers.hm.requests", {
-          'droplet' => 1,
+          'droplet' => @app.id,
           'op' => 'START',
           'last_updated' => @app.last_updated.to_i,
           'version' => "#{@app.staged_package_hash}-#{@app.run_count}",
@@ -164,7 +165,7 @@ describe HealthManager do
         3 => { :state => 'RUNNING', :timestamp => timestamp, :last_action => @app.last_updated, :instance => '3' }
     }}
     should_publish_to_nats "cloudcontrollers.hm.requests", {
-          'droplet' => 1,
+          'droplet' => @app.id,
           'op' => 'STOP',
           'last_updated' => @app.last_updated.to_i,
           'instances' => [ version_entry[:indices][3][:instance] ]
@@ -193,7 +194,7 @@ describe HealthManager do
 
   it "should restart an instance that exits unexpectedly" do
     should_publish_to_nats "cloudcontrollers.hm.requests", {
-          'droplet' => 1,
+          'droplet' => @app.id,
           'op' => 'START',
           'last_updated' => @app.last_updated.to_i,
           'version' => "#{@app.staged_package_hash}-#{@app.run_count}",
@@ -202,7 +203,7 @@ describe HealthManager do
 
     @hm.process_heartbeat_message(make_heartbeat_message([0], "RUNNING"))
     droplet_entry = @hm.process_exited_message({
-                                                     'droplet' => 1,
+                                                     'droplet' => @app.id,
                                                      'version' => "#{@app.staged_package_hash}-#{@app.run_count}",
                                                      'index' => 0,
                                                      'instance' => 0,