- Add new standalone framework that supports any runtime

- Allow a framework to not have a default runtime

- Allow sending a start command on app create/update

- Fix cc staging json parsing issue to allow quotes
in start command (EM.system strips escaped backslashes)

- Do not assign a web port if no URLs specified

- Confirm app is started by PID if no web port assigned

Change-Id: Ib4e51abf2495ff72dd586727bae85c0997bac28e
This commit is contained in:
Jennifer Hickey 2012-03-26 10:46:32 -07:00
Родитель 07b448e065
Коммит e46c15485d
24 изменённых файлов: 511 добавлений и 24 удалений

Просмотреть файл

@ -142,7 +142,7 @@ GEM
yajl-ruby (~> 0.8.3)
vcap_logging (0.1.3)
vcap_stager (0.1.11)
vcap_staging (0.1.47)
vcap_staging (0.1.48)
nokogiri (>= 1.4.4)
rake
rspec

Просмотреть файл

@ -407,6 +407,7 @@ class AppsController < ApplicationController
app.framework = body_params[:staging][:framework]
app.runtime = body_params[:staging][:runtime]
end
app.metadata[:command] = body_params[:staging][:command] if body_params[:staging][:command]
end
unless app.framework
CloudController.logger.error "app: #{app.id} No app framework indicated"

Просмотреть файл

@ -23,8 +23,10 @@ class App < ActiveRecord::Base
AppStates = %w[STOPPED STARTED]
PackageStates = %w[PENDING STAGED FAILED]
Runtimes = %w[ruby18 ruby19 java node node06 php erlangR14B02 python2]
Frameworks = %w[sinatra rack rails3 java_web spring grails node php otp_rebar lift wsgi django unknown]
Frameworks = %w[sinatra rack rails3 java_web spring grails node php otp_rebar lift wsgi django standalone unknown]
validates_presence_of :name, :framework, :runtime
@ -119,8 +121,9 @@ class App < ActiveRecord::Base
services = service_bindings(true).map {|sb| sb.for_staging}
{ :services => services,
:framework => framework,
:runtime => runtime,
:resources => resource_requirements }
:runtime => runtime,
:resources => resource_requirements,
:meta => metadata }
end
def staging_task_properties
@ -129,7 +132,8 @@ class App < ActiveRecord::Base
:framework => framework,
:runtime => runtime,
:resources => resource_requirements,
:environment => environment}
:environment => environment,
:meta => metadata }
end
# Returns an array of the URLs that point to this application

Просмотреть файл

@ -89,6 +89,7 @@ class AppManager
AppManager.secure_staging_dir(job[:user], job[:staging_dir])
AppManager.secure_staging_dir(job[:user], job[:exploded_dir])
AppManager.secure_staging_dir(job[:user], job[:work_dir])
end
Bundler.with_clean_env do
@ -117,6 +118,7 @@ class AppManager
ensure
FileUtils.rm_rf(job[:staging_dir])
FileUtils.rm_rf(job[:exploded_dir])
FileUtils.rm_rf(job[:work_dir]) if job[:work_dir]
end
end.resume
@ -136,11 +138,15 @@ class AppManager
end
def run_staging_command(script, exploded_dir, staging_dir, env_json)
work_dir = Dir.mktmpdir
plugin_env_path = File.join(work_dir, 'plugin_env.json')
File.open(plugin_env_path, 'w') {|f| f.write(env_json) }
job = {
:app => @app,
:cmd => "#{script} #{exploded_dir} #{staging_dir} #{env_json} #{AppManager.staging_manifest_directory}",
:cmd => "#{script} #{exploded_dir} #{staging_dir} #{plugin_env_path} #{AppManager.staging_manifest_directory}",
:staging_dir => staging_dir,
:exploded_dir => exploded_dir
:exploded_dir => exploded_dir,
:work_dir => work_dir
}
CloudController.logger.debug("Queueing staging command #{job[:cmd]}", :tags => [:staging])
@ -288,15 +294,15 @@ class AppManager
end
runtime = nil
manifest['runtimes'].each do |hash|
runtime ||= hash[app.runtime]
end
unless runtime
raise CloudError.new(CloudError::APP_INVALID_RUNTIME, app.runtime, app.framework)
end
env_json = Yajl::Encoder.encode(app.staging_environment)
env_json = app.staging_environment
app_source_dir = Dir.mktmpdir
app.explode_into(app_source_dir)

Просмотреть файл

@ -18,7 +18,10 @@ args = ARGV.dup
if args.length > 2
begin
args[2] = Yajl::Parser.parse(args[2], :symbolize_keys => true)
File.open(args[2], "r") { |f|
env_json = f.read
args[2] = Yajl::Parser.parse(env_json, :symbolize_keys => true)
}
rescue => e
puts "ERROR DECODING ENVIRONMENT: #{e}"
exit 1

Двоичный файл не отображается.

Просмотреть файл

@ -618,16 +618,18 @@ module DEA
start_operation = lambda do
@logger.debug('Completed download')
port = grab_port
if port
instance[:port] = port
if not instance[:uris].empty?
port = grab_port
if port
instance[:port] = port
else
@logger.warn("Unable to allocate port for instance#{instance[:log_id]}")
stop_droplet(instance)
return
end
else
@logger.warn("Unable to allocate port for instance#{instance[:log_id]}")
stop_droplet(instance)
return
@logger.info("No URIs found for application. Not assigning a port")
end
if debug
debug_port = grab_port
if debug_port
@ -709,7 +711,11 @@ module DEA
process.send_data("umask 077\n")
end
app_env.each { |env| process.send_data("export #{env}\n") }
process.send_data("./startup -p #{instance[:port]}\n")
if instance[:port]
process.send_data("./startup -p #{instance[:port]}\n")
else
process.send_data("./startup\n")
end
process.send_data("exit\n")
end
@ -729,7 +735,7 @@ module DEA
# Send the start message, which will bind the router, when we have established the
# connection..
detect_port_ready(instance) do |detected|
detect_app_ready(instance) do |detected|
if detected and not instance[:stop_processed]
@logger.info("Instance #{instance[:log_id]} is ready for connections, notifying system of status")
instance[:state] = :RUNNING
@ -916,6 +922,14 @@ module DEA
VCAP.grab_ephemeral_port
end
def detect_app_ready(instance, &block)
if instance[:port]
detect_port_ready(instance, &block)
else
block.call(true)
end
end
def detect_port_ready(instance, &block)
port = instance[:port]
attempts = 0
@ -1150,7 +1164,6 @@ module DEA
env << "#{k}=#{v}"
end
end
return env
end

Просмотреть файл

@ -21,6 +21,7 @@ default[:cloud_controller][:staging][:java_web] = "java_web.yml"
default[:cloud_controller][:staging][:php] = "php.yml"
default[:cloud_controller][:staging][:django] = "django.yml"
default[:cloud_controller][:staging][:wsgi] = "wsgi.yml"
default[:cloud_controller][:staging][:standalone] = "standalone.yml"
# Default builtin services
default[:cloud_controller][:builtin_services] = ["redis", "mongodb", "mysql", "neo4j", "rabbitmq"]

Просмотреть файл

@ -0,0 +1,55 @@
---
name: "standalone"
runtimes:
- "python2":
version: "2.6.5"
description: <%= "Python #{node[:python2][:version]}" %>
executable: python
environment:
path: /usr/bin:$PATH
- "java":
description: "Java 6"
version: "1.6"
executable: "java"
environment:
path: /usr/bin:$PATH
- node:
version: '0.4.12'
description: 'Node.js'
executable: <%= File.join(node[:node04][:path], "bin", "node") %>
environment:
path: <%= File.join(node[:node04][:path], "bin") %>:$PATH
- node06:
version: '0.6.8'
description: 'Node.js'
executable: <%= File.join(node[:node06][:path], "bin", "node") %>
environment:
path: <%= File.join(node[:node06][:path], "bin") %>:$PATH
- erlangR14B02:
version: 'R14B02'
description: 'Erlang R14B02'
executable: <%= File.join(node[:erlang][:path], "bin", "erl") %>
environment:
path: <%= File.join(node[:erlang][:path], "bin") %>:$PATH
- "php":
description: "PHP 5"
version: "5.3"
executable: "php"
environment:
path: /usr/bin:$PATH
- "ruby18":
version: "1.8.7" # FIXME change to 1.8.7-p334
description: "Ruby 1.8.7"
executable: "<%= File.join(node[:ruby18][:path], "bin", "ruby") %>"
environment:
path: <%= File.join(node[:ruby18][:path], "bin") %>:$PATH
bundle_gemfile:
- "ruby19":
version: "1.9.2p180"
description: "Ruby 1.9.2"
executable: "<%= File.join(node[:ruby][:path], "bin", "ruby") %>"
environment:
path: <%= File.join(node[:ruby][:path], "bin") %>:$PATH
bundle_gemfile:
# vim: filetype=yaml

Просмотреть файл

@ -46,7 +46,7 @@ GEM
thin (~> 1.3.1)
yajl-ruby (~> 0.8.3)
vcap_logging (0.1.3)
vcap_staging (0.1.47)
vcap_staging (0.1.48)
nokogiri (>= 1.4.4)
rake
rspec

Двоичный файл не отображается.

Просмотреть файл

@ -1,7 +1,7 @@
PATH
remote: .
specs:
vcap_staging (0.1.47)
vcap_staging (0.1.48)
nokogiri (>= 1.4.4)
rake
rspec

Просмотреть файл

@ -201,6 +201,7 @@ class StagingPlugin
return name if rt_info['default']
end
end
return nil
end
# Exits the process with a nonzero status if ARGV does not contain valid

Просмотреть файл

@ -0,0 +1,41 @@
---
name: "standalone"
runtimes:
- "python2":
version: "2.6.5"
description: "Python 2.6.5"
executable: python
- "java":
description: "Java 6"
version: "1.6"
executable: "java"
- node:
version: '0.4.12'
description: 'Node.js'
executable: node
- node06:
version: '0.6.8'
description: 'Node.js'
executable: node
- erlangR14B02:
version: 'R14B02'
description: 'Erlang R14B02'
executable: /var/vcap/runtimes/erlang-R14B02/bin/erl
- "php":
description: "PHP 5"
version: "5.3"
executable: "php"
- "ruby18":
version: "1.8.7" # FIXME change to 1.8.7-p334
description: "Ruby 1.8.7"
executable: "/usr/bin/ruby"
environment:
bundle_gemfile:
- "ruby19":
version: "1.9.2p180"
description: "Ruby 1.9.2"
executable: "ruby"
environment:
bundle_gemfile:
# vim: filetype=yaml

Просмотреть файл

@ -0,0 +1,90 @@
class StandalonePlugin < StagingPlugin
include GemfileSupport
include RubyAutoconfig
def framework
'standalone'
end
def stage_application
Dir.chdir(destination_directory) do
create_app_directories
copy_source_files
#Give everything executable perms, as start command may be a script
FileUtils.chmod_R(0744, File.join(destination_directory, 'app'))
runtime_specific_staging
create_startup_script
create_stop_script
end
end
private
def runtime_specific_staging
if environment[:runtime] =~ /\Aruby/
compile_gems
install_autoconfig_gem if autoconfig_enabled?
elsif environment[:runtime] =~ /\Ajava/
#Make a temp dir for java.io.tmpdir
FileUtils.mkdir_p File.join(destination_directory, 'temp')
end
end
def start_command
environment[:meta][:command]
end
def startup_script
vars = environment_hash
if environment[:runtime] =~ /\Aruby/
ruby_startup_script vars
elsif environment[:runtime] =~ /\Ajava/
java_startup_script vars
elsif environment[:runtime] =~ /\Apython/
python_startup_script vars
else
generate_startup_script(vars)
end
end
def ruby_startup_script vars
if uses_bundler?
path = vars['PATH'] ? vars['PATH'] : "$PATH"
vars['PATH'] = "$PWD/app/rubygems/ruby/#{library_version}/bin:#{path}"
vars['GEM_PATH'] = vars['GEM_HOME'] = "$PWD/app/rubygems/ruby/#{library_version}"
if autoconfig_enabled?
vars['RUBYOPT'] = "-I$PWD/ruby #{autoconfig_load_path} -rcfautoconfig -rstdsync"
else
vars['RUBYOPT'] = "-I$PWD/ruby -rstdsync"
end
else
vars['RUBYOPT'] = "-rubygems -I$PWD/ruby -rstdsync"
end
generate_startup_script(vars) do
ruby_stdsync_startup
end
end
def ruby_stdsync_startup
cmds = []
cmds << "mkdir ruby"
cmds << 'echo "\$stdout.sync = true" >> ./ruby/stdsync.rb'
cmds.join("\n")
end
def java_startup_script vars
java_sys_props = "-Djava.io.tmpdir=$PWD/temp"
vars['JAVA_OPTS'] = "$JAVA_OPTS -Xms#{application_memory}m -Xmx#{application_memory}m #{java_sys_props}"
generate_startup_script(vars)
end
def python_startup_script vars
#setup python scripts to sync stdout/stderr to files
vars['PYTHONUNBUFFERED'] = "true"
generate_startup_script(vars)
end
def stop_script
vars = environment_hash
generate_stop_script(vars)
end
end

Просмотреть файл

@ -0,0 +1,7 @@
#!/usr/bin/env ruby
require File.expand_path('../../common', __FILE__)
plugin_class = StagingPlugin.load_plugin_for('standalone')
plugin_class.validate_arguments!
plugin_class.new(*ARGV).stage_application
# vim: ts=2 sw=2 filetype=ruby

Просмотреть файл

@ -1,5 +1,5 @@
module VCAP
module Staging
VERSION = '0.1.47'
VERSION = '0.1.48'
end
end

4
staging/spec/fixtures/apps/standalone_gemfile/source/Gemfile поставляемый Normal file
Просмотреть файл

@ -0,0 +1,4 @@
source "http://rubygems.org"
gem "sinatra"
gem "thin"
gem "json"

Просмотреть файл

@ -0,0 +1,26 @@
GEM
remote: http://rubygems.org/
specs:
daemons (1.1.8)
eventmachine (0.12.10)
json (1.6.5)
rack (1.4.1)
rack-protection (1.2.0)
rack
sinatra (1.3.2)
rack (>= 1.3.6, ~> 1.3)
rack-protection (~> 1.2)
tilt (>= 1.3.3, ~> 1.3)
thin (1.3.1)
daemons (>= 1.0.9)
eventmachine (>= 0.12.6)
rack (>= 1.0.0)
tilt (1.3.3)
PLATFORMS
ruby
DEPENDENCIES
json
sinatra
thin

12
staging/spec/fixtures/apps/standalone_gemfile/source/app.rb поставляемый Normal file
Просмотреть файл

@ -0,0 +1,12 @@
require 'rubygems'
require 'sinatra'
get '/' do
results = <<-OUT
<pre>
#{%x[gem env]}
#{%x[gem list -l -d].strip}
</pre>
OUT
end

Просмотреть файл

@ -0,0 +1,20 @@
public class HelloCloud {
public static void main(String[] args) {
String javaopts = System.getenv("JAVA_OPTS");
if(javaopts != null) {
String userDir = System.getProperty("user.dir");
//Strip the /app off the current working dir
userDir = userDir.substring(0,userDir.length()-4);
javaopts = javaopts.replaceAll(userDir,"appdir");
}
System.out.print("Hello from the cloud. Java opts: " + javaopts);
while(true) {
try {
Thread.sleep(120000);
} catch (InterruptedException e) {
}
}
}
}

Просмотреть файл

@ -0,0 +1,4 @@
import time
print "Hello, World!",
while 1:
time.sleep(120000)

Просмотреть файл

@ -0,0 +1 @@
puts "Hello World"

Просмотреть файл

@ -0,0 +1,198 @@
require 'spec_helper'
describe "A Standalone app being staged" do
describe "when bundled" do
before do
app_fixture :standalone_gemfile
end
describe "and using Ruby 1.8" do
it "is packaged with a startup script" do
stage(:standalone,{:meta=>{:command=> "ruby app.rb"}, :runtime=> "ruby18"}) do |staged_dir|
start_script = File.join(staged_dir, 'startup')
start_script.should be_executable_file
script_body = File.read(start_script)
script_body.should == <<-EXPECTED
#!/bin/bash
export GEM_HOME="$PWD/app/rubygems/ruby/1.8"
export GEM_PATH="$PWD/app/rubygems/ruby/1.8"
export PATH="$PWD/app/rubygems/ruby/1.8/bin:$PATH"
export RUBYOPT="-I$PWD/ruby -I$PWD/app/rubygems/ruby/1.8/gems/cf-autoconfig-0.0.2/lib -rcfautoconfig -rstdsync"
unset BUNDLE_GEMFILE
mkdir ruby
echo "\\$stdout.sync = true" >> ./ruby/stdsync.rb
cd app
ruby app.rb > ../logs/stdout.log 2> ../logs/stderr.log &
STARTED=$!
echo "$STARTED" >> ../run.pid
wait $STARTED
EXPECTED
end
end
it "installs gems" do
stage(:standalone,{:meta=>{:command=> "ruby app.rb"}, :runtime=> "ruby18"}) do |staged_dir|
gemdir = File.join(staged_dir,'app','rubygems','ruby','1.8')
Dir.entries(gemdir).should_not == []
end
end
it "installs autoconfig gem" do
stage :sinatra do |staged_dir|
gemfile = File.join(staged_dir,'app','Gemfile')
gemfile_body = File.read(gemfile)
gemfile_body.should == <<-EXPECTED
source "http://rubygems.org"
gem "sinatra"
gem "thin"
gem "json"
gem "cf-autoconfig"
EXPECTED
end
end
end
describe "and using Ruby 1.9" do
it "is packaged with a startup script" do
stage(:standalone,{:meta=>{:command=> "ruby app.rb"}, :runtime=> "ruby19"}) do |staged_dir|
start_script = File.join(staged_dir, 'startup')
start_script.should be_executable_file
script_body = File.read(start_script)
script_body.should == <<-EXPECTED
#!/bin/bash
export GEM_HOME="$PWD/app/rubygems/ruby/1.9.1"
export GEM_PATH="$PWD/app/rubygems/ruby/1.9.1"
export PATH="$PWD/app/rubygems/ruby/1.9.1/bin:$PATH"
export RUBYOPT="-I$PWD/ruby -rcfautoconfig -rstdsync"
unset BUNDLE_GEMFILE
mkdir ruby
echo "\\$stdout.sync = true" >> ./ruby/stdsync.rb
cd app
ruby app.rb > ../logs/stdout.log 2> ../logs/stderr.log &
STARTED=$!
echo "$STARTED" >> ../run.pid
wait $STARTED
EXPECTED
end
end
it "installs gems" do
stage(:standalone,{:meta=>{:command=> "ruby app.rb"}, :runtime=> "ruby19"}) do |staged_dir|
gemdir = File.join(staged_dir,'app','rubygems','ruby','1.9.1')
Dir.entries(gemdir).should_not == []
end
end
it "installs autoconfig gem" do
stage :sinatra do |staged_dir|
gemfile = File.join(staged_dir,'app','Gemfile')
gemfile_body = File.read(gemfile)
gemfile_body.should == <<-EXPECTED
source "http://rubygems.org"
gem "sinatra"
gem "thin"
gem "json"
gem "cf-autoconfig"
EXPECTED
end
end
end
end
describe "when using Ruby and not bundled" do
before do
app_fixture :standalone_simple_ruby
end
describe "and using Ruby 1.8" do
it "is packaged with a startup script" do
stage(:standalone,{:meta=>{:command=> "ruby hello.rb"}, :runtime=> "ruby18"}) do |staged_dir|
start_script = File.join(staged_dir, 'startup')
start_script.should be_executable_file
script_body = File.read(start_script)
script_body.should == <<-EXPECTED
#!/bin/bash
export RUBYOPT="-rubygems -I$PWD/ruby -rstdsync"
unset BUNDLE_GEMFILE
mkdir ruby
echo "\\$stdout.sync = true" >> ./ruby/stdsync.rb
cd app
ruby hello.rb > ../logs/stdout.log 2> ../logs/stderr.log &
STARTED=$!
echo "$STARTED" >> ../run.pid
wait $STARTED
EXPECTED
end
end
end
describe "and using Ruby 1.9" do
it "is packaged with a startup script" do
stage(:standalone,{:meta=>{:command=> "ruby hello.rb"}, :runtime=> "ruby19"}) do |staged_dir|
start_script = File.join(staged_dir, 'startup')
start_script.should be_executable_file
script_body = File.read(start_script)
script_body.should == <<-EXPECTED
#!/bin/bash
export RUBYOPT="-rubygems -I$PWD/ruby -rstdsync"
unset BUNDLE_GEMFILE
mkdir ruby
echo "\\$stdout.sync = true" >> ./ruby/stdsync.rb
cd app
ruby hello.rb > ../logs/stdout.log 2> ../logs/stderr.log &
STARTED=$!
echo "$STARTED" >> ../run.pid
wait $STARTED
EXPECTED
end
end
end
end
describe "with Java runtime" do
before do
app_fixture :standalone_java
end
it "is packaged with a startup script" do
stage(:standalone,{:meta=>{:command=> "java $JAVA_OPTS HelloWorld"}, :runtime=> "java", :environment=>{:resources=>{:memory=>512}}}) do |staged_dir|
start_script = File.join(staged_dir, 'startup')
start_script.should be_executable_file
script_body = File.read(start_script)
script_body.should == <<-EXPECTED
#!/bin/bash
export JAVA_OPTS="$JAVA_OPTS -Xms512m -Xmx512m -Djava.io.tmpdir=$PWD/temp"
cd app
java $JAVA_OPTS HelloWorld > ../logs/stdout.log 2> ../logs/stderr.log &
STARTED=$!
echo "$STARTED" >> ../run.pid
wait $STARTED
EXPECTED
end
end
it "creates a temp dir" do
stage(:standalone,{:meta=>{:command=> "java $JAVA_OPTS HelloWorld"}, :runtime=> "java", :environment=>{:resources=>{:memory=>512}}}) do |staged_dir|
tmp_dir = File.join(staged_dir, 'temp')
File.exists?(tmp_dir).should == true
end
end
end
describe "with Python runtime" do
before do
app_fixture :standalone_python
end
it "is packaged with a startup script" do
stage(:standalone,{:meta=>{:command=> "python HelloWorld.py"}, :runtime=> "python2", :environment=>{:resources=>{:memory=>512}}}) do |staged_dir|
start_script = File.join(staged_dir, 'startup')
start_script.should be_executable_file
script_body = File.read(start_script)
script_body.should == <<-EXPECTED
#!/bin/bash
export PYTHONUNBUFFERED="true"
cd app
python HelloWorld.py > ../logs/stdout.log 2> ../logs/stderr.log &
STARTED=$!
echo "$STARTED" >> ../run.pid
wait $STARTED
EXPECTED
end
end
end
end