#------------------------------------------------------------------------- # # Copyright (c) Microsoft and contributors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. #-------------------------------------------------------------------------- # # This rakefile consists of several tasks. This section will provide an overview of the # tasks and their dependencies. # # ------------------------------------------------------------------------------------------------------------------------------------------- # | No | Task | Description | # |----|---------------------------------|--------------------------------------------------------------------------------------------------| # | 1 | regen | Regenerates all the SDK versions for all the services and all the profiles for all the services. | # | | | This task has regen_sdk_versions and regen_all_profiles as prereqs. | # |----|---------------------------------|--------------------------------------------------------------------------------------------------| # | 2 | regen_sdk_versions | Regenerates all the SDK versions for all the services. At the end, this task calls | # | | | regen_individual_require_files as an enhancement. This task has clean_generated as prereq. This | # |. |. | task is focused on code generation using autorest and will generate api versions as indicated in | # | | |.in the config.json file. | # |----|---------------------------------|--------------------------------------------------------------------------------------------------| # | 3 | regen_all_profiles | Regenerates all the profiles for all the services. This task has regen_rollup_profile and | # | | | regen_individual_profiles as prereqs. This task uses the profile_generator to generate the | # | | | with inputs from profiles.json and dir_metadata.json. | # |----|---------------------------------|--------------------------------------------------------------------------------------------------| # | 4 | clean_generated | Cleans all the generated folders in multiple SDK versions. | # |----|---------------------------------|--------------------------------------------------------------------------------------------------| # | 5 | regen_rollup_profile | Regenerates all profiles within the rollup gem. At the end, this task calls | # | | | regen_rollup_require_files as an enhancement. This task has clean_rollup_profiles as prereq. This| #.|. | | task requires correct definition of rollup profiles in profiles.json file. | # |----|---------------------------------|--------------------------------------------------------------------------------------------------| # | 6 | regen_individual_profiles | Regenerates all profiles with all the services. At the end, this task calls | # | | | regen_individual_require_files as an enhancement. This task calls clean_individual_profiles | # | | | as prereq. This task requires correct definition of individual profiles in profiles.json file. | # |----|---------------------------------|--------------------------------------------------------------------------------------------------| # | 7 | clean_all_profiles | Cleans all profiles inside the rollup gem and individual gems.This task has clean_rollup_profiles| # | | | and clean_individual_profiles as prereqs. | # |----|---------------------------------|--------------------------------------------------------------------------------------------------| # | 8 | clean_rollup_profiles | Cleans all profiles inside the rollup gem. | # |----|---------------------------------|--------------------------------------------------------------------------------------------------| # | 9 | clean_individual_profiles | Cleans all profiles inside individual gems. | # |----|---------------------------------|--------------------------------------------------------------------------------------------------| # | 10 | regen_all_require_files | Regenerates the require files for the rollup gem and individual gems. This task has | # | | | regen_rollup_require_files and regen_individual_require_files as prereqs. A require file is the | # | | | base file for a gem which includes other file. A sample could be found here: | # |. | | https://github.com/Azure/azure-sdk-for-ruby/blob/master/azure_sdk/lib/azure_sdk.rb | # |----|---------------------------------|--------------------------------------------------------------------------------------------------| # | 11 | regen_rollup_require_files | Regenerates the require files for the rollup gem | # |----|---------------------------------|--------------------------------------------------------------------------------------------------| # | 12 | regen_individual_require_files | Regenerates the require files for the individual gems | # |----|---------------------------------|--------------------------------------------------------------------------------------------------| # | 13 | release | Release gems. This task has build as a prereq. The gems to be released must be indicated in the | # |. | | GEMS_TO_RELEASE file. | # |----|---------------------------------|--------------------------------------------------------------------------------------------------| # | 14 | build | Builds all gems. This task has clean as a prereq. | # |----|---------------------------------|--------------------------------------------------------------------------------------------------| # | 15 | clean | Cleans the pks folder inside all the gems. | # |----|---------------------------------|--------------------------------------------------------------------------------------------------| # | 16 | bundle:update | Performs bundle update for all gems. | # |----|---------------------------------|--------------------------------------------------------------------------------------------------| # | 17 | spec | Executes tests for all gems. | # ------------------------------------------------------------------------------------------------------------------------------------------- # # regen # | # ----------------------------------------------------------------------------------------------------- # | | # regen_sdk_versions -->enhancement--> regen_individual_require_files | # | regen_all_profiles # | | # clean_generated ------------------------------------------------------------------------------------------------------------------------------ # | | # | | # | | # | | # | regen_all_require_files | # | | | # | ----------------------------------------- | # | | | | # | | | | # | | | | # regen_rollup_profile -->enhancement--> regen_rollup_require_files | | # | | | # | | | # | | | # | | | # | | | # | | | # | regen_individual_require_files<--enhancement<--regen_individual_profiles # | | # | clean_all_profiles | # | | | # | --------------------------------------------------------------------------------------------------------------- | # | | | | # | | | | # clean_rollup_profiles clean_individual_profiles # # release # | # build # | # clean # # bundle:update # # spec # require 'dotenv/tasks' require 'rspec/core/rake_task' require 'open3' require 'os' require 'json' require 'fileutils' def get_config_files_folder "#{__dir__}/config" end gem_versions = JSON.parse(File.read(File.expand_path("#{get_config_files_folder}/GEM_VERSIONS", __FILE__)).strip) gems_to_release = JSON.parse(File.read(File.expand_path("#{get_config_files_folder}/GEMS_TO_RELEASE", __FILE__)).strip) GEMS_TO_IGNORE = %w{azure_mgmt_graph, azure_mgmt_mobile_engagement, azure_mgmt_server_management} REGEN_EXCLUDES = ['azure_sdk'] desc 'Azure Resource Manager related tasks which often traverse each of the arm gems' namespace :arm do desc 'Delete ./pkg for each of the Azure Resource Manager projects' task :clean, [:gem_name] do |_, args| each_gem do |dir| if !should_i_continue(args[:gem_name], dir) next end FileUtils.rm_rf('pkg') end end desc 'Delete multiple version folders for each sdk' task :clean_generated, [:gem_name] do |_, args| clean_generated('management', args[:gem_name]) clean_generated('data', args[:gem_name]) Dir.chdir(File.expand_path('..', __FILE__)) end desc 'Build gems for each of the Azure Resource Manager projects' task :build, [:gem_name] => :clean do |_, args| each_gem do |dir| if !should_i_continue(args[:gem_name], dir) next end execute_and_stream('rake build') end end desc 'Push gem for each of the Azure projects' task :release, [:key] => :build do |_, args| gems_to_release.each do |key, gems| gems.each do |gem| if(key == 'rollup') Dir.chdir("#{__dir__}/azure_sdk") version = gem_versions['rollup'][gem] elsif(key == 'management') Dir.chdir("#{__dir__}/management/#{gem}") version = gem_versions['management'][gem] elsif(key == 'data') Dir.chdir("#{__dir__}/data/#{gem}") version = gem_versions['data'][gem] end # Delay for 10 seconds before publishing gem puts "Going to sleep for 10 seconds before publishing #{gem} ..." sleep(10) execute("gem push ./pkg/#{gem}-#{version}.gem" + (args[:key].nil? ? '' : " -k #{args[:key]}")) end end end desc 'Regen code for each sdk with all its api versions' task :regen_sdk_versions, [:gem_name] => [:regen_folder_structure_if_required, :clean_generated] do |_, args| json = get_config_file each_gem do |dir| # dir corresponds to each azure_* folder if REGEN_EXCLUDES.include?(dir.to_s) update_gem_version('lib/azure_sdk/version.rb', gem_versions['rollup'][dir]) next end if !should_i_continue(args[:gem_name], dir) next end mode = get_mode(dir) puts "\nGenerating #{dir}\n" ar_base_command = "#{ENV.fetch('AUTOREST_LOC', 'autorest')}" ar_base_command = "#{ar_base_command} --use=#{ENV.fetch('AUTOREST_RUBY_LOC')}" unless ENV.fetch('AUTOREST_RUBY_LOC', nil).nil? puts "ar_base_command #{ar_base_command}" md = json[dir] # there should be an entry in the metadata for each of the api versions to generate package_name = dir md.each do |api_version_pkg, api_version_value| ar_arguments = '' output_folder = '' api_version_value.each do |argument_name, argument_value| if argument_name.casecmp("output-folder") == 0 output_folder = argument_value else if argument_name.casecmp("package-name") == 0 package_name = argument_value else if argument_name.casecmp("markdown") == 0 ar_arguments = ar_arguments + " #{argument_value}" else if argument_name.casecmp("input-file") == 0 input_files = argument_value.map {|file| "--input-file=#{file}"} ar_arguments = ar_arguments + input_files.join(" ") else ar_arguments = ar_arguments + " --#{argument_name}=#{argument_value}" end end end end end command = "#{ar_base_command} --package-name=#{package_name} #{ar_arguments} --package-version=#{gem_versions[mode][dir]} --output-folder=#{File.join(Dir.pwd, 'lib', output_folder)} --ruby --azure-arm --version=V2" execute_and_stream(command) end update_gem_version('lib/version.rb', gem_versions[mode][dir]) end end desc 'Bundler related helper' namespace :bundle do desc 'bundle update for each of the Azure Resource Manager projects' task :update, [:gem_name] do |_, args| each_gem do if !should_i_continue(args[:gem_name], dir) next end execute_and_stream('bundle update') end end end desc 'run specs for each of the Azure Resource Manager projects' task :spec, [:gem_name] => :dotenv do |_, args| each_gem do |gem_dir| if !should_i_continue(args[:gem_name], gem_dir) next end puts "Executing spec on #{gem_dir}" execute_and_stream('bundle install') execute_and_stream('bundle exec rspec') end end desc 'Clean Rollup Profiles' task :clean_rollup_profiles do Dir.chdir(File.expand_path('../azure_sdk/lib', __FILE__)) subdir_list = Dir['*'].reject{|o| not File.directory?(o)} subdir_list.each do |subdir| if subdir != 'azure_sdk' folder_to_be_cleaned = File.expand_path("../azure_sdk/lib/#{subdir}", __FILE__) puts "Cleaning folder - #{folder_to_be_cleaned}" FileUtils.rm_rf(folder_to_be_cleaned) end end end desc 'Clean Individual Profiles' task :clean_individual_profiles, [:gem_name] do |_, args| each_gem_dir do |gem| if !should_i_continue(args[:gem_name], gem) next end if REGEN_EXCLUDES.include?gem next end mode = get_mode(gem) Dir.chdir("#{__dir__}/#{mode}/#{gem}/lib/profiles") subdir_list = Dir['*'].reject{|o| not File.directory?(o)} subdir_list.each do |subdir| folder_to_be_cleaned = "#{__dir__}/#{mode}/#{gem}/lib/profiles/#{subdir}" puts "Cleaning folder - #{folder_to_be_cleaned}" FileUtils.rm_rf(folder_to_be_cleaned) end end end desc 'Clean All profiles' task :clean_all_profiles, [:gem_name] => [:clean_rollup_profiles, :clean_individual_profiles] do puts 'Cleaned all profiles' end desc 'Regen rollup profiles' task :regen_rollup_profile => :clean_rollup_profiles do Dir.chdir(__dir__) # Sample Command # bundle exec ruby profile_generator_client.rb --dir_metadata=dir_metadata.json --profile=profiles.json --mode=rollup --key=azure_sdk command = "#{get_base_profile_generation_cmd} --dir_metadata=#{get_config_files_folder}/dir_metadata.json --profile=#{get_config_files_folder}/profiles.json --mode=rollup --key=azure_sdk --sdk_path=#{__dir__}" execute_and_stream(command) end desc 'Regen individual profiles' task :regen_individual_profiles, [:gem_name] => :clean_individual_profiles do |_, args| each_gem_dir do |gem| if !should_i_continue(args[:gem_name], gem) next end if REGEN_EXCLUDES.include?gem next end mode = get_mode(gem) # Sample Command # bundle exec ruby profile_generator_client.rb --dir_metadata=dir_metadata.json --profile=authorization_profiles.json --mode=management --key=azure_mgmt_authorization command = "#{get_base_profile_generation_cmd} --dir_metadata=#{get_config_files_folder}/dir_metadata.json --profile=#{get_config_files_folder}/profiles.json --mode=#{mode} --key=#{gem} --sdk_path=#{__dir__}" execute_and_stream(command) end end desc 'Generate individual require files' task :regen_individual_require_files, [:gem_name] do |_, args| key = args[:gem_name].nil? ? '*' : args[:gem_name] command = "bundle exec ruby #{__dir__}/generators/requirefilegen/src/require_file_generator.rb --mode=individual --sdk_path=#{__dir__} --gem_name=#{key}" execute_and_stream(command) end desc 'Generate rollup require files' task :regen_rollup_require_files do command = "bundle exec ruby #{__dir__}/generators/requirefilegen/src/require_file_generator.rb --mode=rollup --sdk_path=#{__dir__} --gem_name=azure_sdk" execute_and_stream(command) end task :regen_folder_structure_if_required, [:gem_name] do |_, args| if args[:gem_name] && !REGEN_EXCLUDES.include?(args[:gem_name]) plane = 'data' if(args[:gem_name].include?'_mgmt_') plane = 'management' end command = "ruby #{__dir__}/generators/metadatagen/src/metadatagen.rb --plane=#{plane} --gem_name=#{args[:gem_name]} --sdk_path=#{__dir__}" execute_and_stream(command) end end desc 'Generate all require files' task :regen_all_require_files, [:gem_name] => [:regen_individual_require_files, :regen_rollup_require_files] do puts 'Generated all require files' end desc 'Regen all profiles' task :regen_all_profiles, [:gem_name] => [:regen_rollup_profile, :regen_individual_profiles] do puts 'Regenerated all profiles' end desc 'Regen all versions of sdk and profiles' task :regen, [:gem_name] => [:regen_sdk_versions, :regen_all_profiles] do puts 'Regenerated all versions of sdk and profiles' end end Rake::Task['arm:regen_rollup_profile'].enhance do Rake::Task['arm:regen_rollup_require_files'].invoke Rake::Task['arm:regen_rollup_require_files'].reenable end Rake::Task['arm:regen_individual_profiles'].enhance do |_, args| Rake::Task['arm:regen_individual_require_files'].invoke(args[:gem_name]) Rake::Task['arm:regen_individual_require_files'].reenable end Rake::Task['arm:regen_sdk_versions'].enhance do |_, args| Rake::Task['arm:regen_individual_require_files'].invoke(args[:gem_name]) Rake::Task['arm:regen_individual_require_files'].reenable end task :default => :spec def should_i_continue(key, dir) return (key.nil? || key == dir) end def get_mode(dir) mode = '' if (dir.include? '_mgmt_') mode = 'management' else mode = 'data' end mode end def clean_generated(parent_dir, key) Dir.chdir(File.expand_path("../#{parent_dir}", __FILE__)) gem_folders = Dir['*'].reject{|o| not File.directory?(o)} gem_folders.each do |gem| if !should_i_continue(key, gem) next end if GEMS_TO_IGNORE.include?gem next end Dir.chdir(File.expand_path("../#{parent_dir}/#{gem}/lib", __FILE__)) subdir_list = Dir['*'].reject{|o| not File.directory?(o)} subdir_list.each do |subdir| if subdir.to_s == 'profiles' next end folder_to_be_cleaned = File.expand_path("../#{parent_dir}/#{gem}/lib/#{subdir}", __FILE__) puts "Cleaning folder - #{folder_to_be_cleaned}" FileUtils.rm_rf(folder_to_be_cleaned) end end end def get_base_profile_generation_cmd "bundle exec ruby #{__dir__}/generators/profilegen/src/profile_generator_client.rb" end def execute_and_stream(cmd) puts "running: #{cmd}" execute(cmd) end def execute(cmd) Open3.popen2e(cmd) do |_, stdout_err, wait_thr| while line = stdout_err.gets puts line end exit_status = wait_thr.value unless exit_status.success? abort "FAILED !!!" end end end def get_config_file config_file = File.read(File.expand_path("#{get_config_files_folder}/config.json", __FILE__)) JSON.parse(config_file) end def each_child each_child_mode_specific('management') {|dir| yield dir } each_child_mode_specific('data') {|dir| yield dir } Dir.chdir(File.expand_path('../azure_sdk', __FILE__)) yield('azure_sdk') end def each_child_mode_specific(parent_dir, change_dir = true) Dir.chdir(File.expand_path("../#{parent_dir}", __FILE__)) sub_dirs = Dir['*'].reject{|o| not File.directory?(o)} sub_dirs.each do |dir| if GEMS_TO_IGNORE.include?dir next end if (change_dir) Dir.chdir(dir) do yield(dir) end else yield dir end end end def each_gem each_child do |dir| gem_dir = dir.split('/').last yield gem_dir end end def each_gem_dir each_child_mode_specific('management', false) {|dir| yield dir } each_child_mode_specific('data', false) {|dir| yield dir } end def update_gem_version(version_file, new_version) existing_contents = File.read(version_file) content_to_replace = existing_contents.gsub(/VERSION = '.*'/, "VERSION = '#{new_version}'") File.open(version_file, 'w') { |file| file.puts content_to_replace } end