azure-storage-fuse/blobfuse2-release.yaml

655 строки
26 KiB
YAML

parameters:
- name: tag
displayName: 'New Release Tag'
type: string
default: 'blobfuse2-'
- name: unit_test
displayName: 'Execute Unit Tests'
type: boolean
default: false
- name: post_release
displayName: 'Post Release on Github'
type: boolean
default: false
- name: draft
displayName: 'Post as Draft Release'
type: boolean
default: false
- name: prerelease
displayName: 'Post as PreRelease'
type: boolean
default: false
- name: update_version
displayName: 'Update Version'
type: boolean
default: false
# Do not trigger this pipeline automatically
trigger: none
pr: none
stages:
- stage: BuildArtifact
jobs:
# Ubuntu builds
- job: Set_1
strategy:
matrix:
Ubuntu-18:
container_image: 'ubuntu-18.04'
fuselib: 'libfuse-dev'
tags: 'fuse2'
Ubuntu-20:
container_image: 'ubuntu-20.04'
fuselib: 'libfuse3-dev'
tags: 'fuse3'
pool:
vmImage: $(container_image)
variables:
- group: NightlyBlobFuse
- name: root_dir
value: '$(System.DefaultWorkingDirectory)'
- name: work_dir
value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse'
steps:
- checkout: none
- script: |
git clone https://github.com/Azure/azure-storage-fuse
displayName: 'Checkout Code & List commits'
workingDirectory: $(root_dir)
# list commits from past 12hrs
- script: |
git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`
git --no-pager log --since="12 hours ago" --stat
displayName: 'List Commits'
workingDirectory: $(work_dir)
# install dependencies required for compiling blobfuse
- script: |
sudo apt-get update --fix-missing
sudo apt-get install ruby-dev build-essential $(fuselib) -y
sudo gem install fpm -V
displayName: "Installing Dependencies"
# build blobfuse2 and generate binary
- template: 'azure-pipeline-templates/build-release.yml'
parameters:
work_dir: $(work_dir)
root_dir: $(root_dir)
unit_test: ${{ parameters.unit_test }}
tags: $(tags)
# place the generated binary files & any additional files in appropriate locations
- script: |
mkdir -p pkgDir/usr/bin/
mkdir -p pkgDir/usr/share/blobfuse2/
cp azure-storage-fuse/blobfuse2 pkgDir/usr/bin/blobfuse2
cp azure-storage-fuse/setup/baseConfig.yaml pkgDir/usr/share/blobfuse2/
cp azure-storage-fuse/sampleFileCacheConfig.yaml pkgDir/usr/share/blobfuse2/
cp azure-storage-fuse/sampleStreamingConfig.yaml pkgDir/usr/share/blobfuse2/
mkdir -p pkgDir/etc/rsyslog.d
mkdir -p pkgDir/etc/logrotate.d
cp azure-storage-fuse/setup/11-blobfuse2.conf pkgDir/etc/rsyslog.d
cp azure-storage-fuse/setup/blobfuse2-logrotate pkgDir/etc/logrotate.d/blobfuse2
workingDirectory: $(root_dir)
displayName: 'Accumulate pkg files'
# using fpm tool for packaging of our binary & performing post-install operations
# for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/
- script: |
fpm -s dir -t deb -n blobfuse2 -C pkgDir/ -v `./pkgDir/usr/bin/blobfuse2 --version | cut -d " " -f 3` -d fuse \
--maintainer "Blobfuse v-Team <blobfusevteam@microsoft.com>" --url "https://github.com/Azure/azure-storage-fuse" \
--description "An user-space filesystem for interacting with Azure Storage" \
--before-install ./azure-storage-fuse/tools/preinstall.sh \
--after-install ./azure-storage-fuse/tools/postinstall.sh
mv ./blobfuse2*.deb ./blobfuse2-`./pkgDir/usr/bin/blobfuse2 --version | cut -d " " -f 3`-$(container_image)-x86-64.deb
cp ./blobfuse2*.deb $(Build.ArtifactStagingDirectory)
workingDirectory: $(root_dir)
displayName: 'Make Package'
# publishing the artifacts generated
- task: PublishBuildArtifacts@1
inputs:
artifactName: 'blobfuse2'
displayName: 'Publish Artifacts'
# Ubuntu builds end here
# Debian Builds start here
- job: Set_2
strategy:
matrix:
Debian-9.0:
DistroVer: 'Debian9.0'
AgentName: 'DEB 9.0'
fuselib: 'libfuse-dev'
tags: 'fuse2'
Debian-10.0:
DistroVer: 'Debian10.0'
AgentName: 'DEB 10.0'
fuselib: 'libfuse-dev'
tags: 'fuse2'
pool:
name: 'Blobfuse Pool'
demands:
- Agent.Name -equals $(AgentName)
variables:
- group: NightlyBlobFuse
- name: root_dir
value: '/home/vsts/workv2'
- name: work_dir
value: "/home/vsts/workv2/azure-storage-fuse"
steps:
- checkout: none
# clearing out remnants from past build sessions
- script: |
sudo rm -rf $(root_dir)
sudo mkdir -p $(root_dir)
sudo chown -R `whoami` $(root_dir)
chmod 777 $(root_dir)
displayName: 'Create Directory Structure'
- script: |
git clone https://github.com/Azure/azure-storage-fuse
displayName: 'Checkout Code & List commits'
workingDirectory: $(root_dir)
# listing commits from last 12hrs
- script: |
git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`
git --no-pager log --since="12 hours ago" --stat
displayName: 'List Commits'
workingDirectory: $(work_dir)
# installing dependencies for compiling blobfuse2
- script: |
sudo apt-get update --fix-missing
sudo apt-get install ruby-dev build-essential $(fuselib) -y
sudo gem install fpm -V
displayName: "Installing Dependencies"
# generating blobfuse2 binary
- template: 'azure-pipeline-templates/build-release.yml'
parameters:
work_dir: $(work_dir)
root_dir: $(root_dir)
unit_test: ${{ parameters.unit_test }}
tags: $(tags)
# accumulating necessary binary & additional scripts to be packaged
- script: |
mkdir -p pkgDir/usr/bin/
mkdir -p pkgDir/usr/share/blobfuse2/
cp azure-storage-fuse/blobfuse2 pkgDir/usr/bin/blobfuse2
cp azure-storage-fuse/setup/baseConfig.yaml pkgDir/usr/share/blobfuse2/
cp azure-storage-fuse/sampleFileCacheConfig.yaml pkgDir/usr/share/blobfuse2/
cp azure-storage-fuse/sampleStreamingConfig.yaml pkgDir/usr/share/blobfuse2/
mkdir -p pkgDir/etc/rsyslog.d
mkdir -p pkgDir/etc/logrotate.d
cp azure-storage-fuse/setup/11-blobfuse2.conf pkgDir/etc/rsyslog.d
cp azure-storage-fuse/setup/blobfuse2-logrotate pkgDir/etc/logrotate.d/blobfuse2
workingDirectory: $(root_dir)
displayName: 'Accumulate pkg files'
# using fpm tool for building the install package
- script: |
fpm -s dir -t deb -n blobfuse2 -C pkgDir/ -v `./pkgDir/usr/bin/blobfuse2 --version | cut -d " " -f 3` -d fuse \
--maintainer "Blobfuse v-Team <blobfusevteam@microsoft.com>" --url "https://github.com/Azure/azure-storage-fuse" \
--description "An user-space filesystem for interacting with Azure Storage" \
--before-install ./azure-storage-fuse/tools/preinstall.sh \
--after-install ./azure-storage-fuse/tools/postinstall.sh
mv ./blobfuse2*.deb ./blobfuse2-`./pkgDir/usr/bin/blobfuse2 --version | cut -d " " -f 3`-$(DistroVer)-x86-64.deb
cp ./blobfuse2*.deb $(Build.ArtifactStagingDirectory)
workingDirectory: $(root_dir)
displayName: 'Make Package'
# publishing artifacts
- task: PublishBuildArtifacts@1
inputs:
artifactName: 'blobfuse2'
displayName: 'Publish Artifacts'
# Debian builds end here
# RHEL 7.x builds begin here
- job: Set_3
strategy:
matrix:
CentOS-7.0:
DistroVer: 'CentOS-7.0'
AgentName: 'COS 7.0'
RHEL-7.5:
DistroVer: 'RHEL-7.5'
AgentName: 'RHEL 7.5'
RHEL-7.8:
DistroVer: 'RHEL-7.8'
AgentName: 'RHEL 7.8'
pool:
name: 'BlobFuse Pool'
demands:
- Agent.Name -equals $(AgentName)
variables:
- group: NightlyBlobFuse
- name: root_dir
value: '/usr/pipeline/workv2'
- name: work_dir
value: '/usr/pipeline/workv2/azure-storage-fuse'
steps:
- checkout: none
# clearing out remnants from past builds
- script: |
sudo rm -rf $(root_dir)
sudo mkdir -p $(root_dir)
sudo chown -R `whoami` $(root_dir)
chmod 777 $(root_dir)
displayName: 'Create Directory Structure'
- script: |
git clone https://github.com/Azure/azure-storage-fuse
displayName: 'Checkout Code & List commits'
workingDirectory: $(root_dir)
# list commits from past 12hrs
- script: |
git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`
git --no-pager log --since="12 hours ago" --stat
displayName: 'List Commits'
workingDirectory: $(work_dir)
# installing dependencies for building & packaging blobfuse
- script: |
sudo yum install gcc-c++ patch readline readline-devel zlib zlib-devel libffi-devel \
openssl-devel make bzip2 autoconf automake libtool bison sqlite-devel -y
sudo yum install fuse3-devel -y
curl -sSL https://rvm.io/mpapis.asc | gpg2 --import -
curl -sSL https://rvm.io/pkuczynski.asc | gpg2 --import -
curl -L get.rvm.io | bash -s stable
source ~/.rvm/scripts/rvm
rvm reload
rvm reinstall 2.5
rvm use 2.5 --default
displayName: 'Installing dependencies'
# generating binary
- template: 'azure-pipeline-templates/build-release.yml'
parameters:
work_dir: $(work_dir)
root_dir: $(root_dir)
unit_test: ${{ parameters.unit_test }}
# accumulating binary & necessary scripts for packaging
- script: |
mkdir -p pkgDir/usr/bin/
mkdir -p pkgDir/usr/share/blobfuse2/
cp azure-storage-fuse/blobfuse2 pkgDir/usr/bin/blobfuse2
cp azure-storage-fuse/setup/baseConfig.yaml pkgDir/usr/share/blobfuse2/
cp azure-storage-fuse/sampleFileCacheConfig.yaml pkgDir/usr/share/blobfuse2/
cp azure-storage-fuse/sampleStreamingConfig.yaml pkgDir/usr/share/blobfuse2/
mkdir -p pkgDir/etc/rsyslog.d
mkdir -p pkgDir/etc/logrotate.d
cp azure-storage-fuse/setup/11-blobfuse2.conf pkgDir/etc/rsyslog.d
cp azure-storage-fuse/setup/blobfuse2-logrotate pkgDir/etc/logrotate.d/blobfuse2
workingDirectory: $(root_dir)
displayName: 'Accumulate pkg files'
# using fpm tool to generate package along with post-install scripts
- script: |
source ~/.rvm/scripts/rvm
gem install fpm
fpm -s dir -t rpm -n blobfuse2 -C pkgDir/ -v `./pkgDir/usr/bin/blobfuse2 --version | cut -d " " -f 3` -d fuse \
--maintainer "Blobfuse v-Team <blobfusevteam@microsoft.com>" --url "https://github.com/Azure/azure-storage-fuse" \
--description "An user-space filesystem for interacting with Azure Storage" \
--before-install ./azure-storage-fuse/tools/preinstall.sh \
--after-install ./azure-storage-fuse/tools/postinstall.sh
mv ./blobfuse2*.rpm ./blobfuse2-`./pkgDir/usr/bin/blobfuse2 --version | cut -d " " -f 3`-$(DistroVer)-x86-64.rpm
cp ./blobfuse2*.rpm $(Build.ArtifactStagingDirectory)
workingDirectory: $(root_dir)
displayName: 'Make Package'
# publish artifacts
- task: PublishBuildArtifacts@1
inputs:
artifactName: 'blobfuse2'
displayName: 'Publish Artifacts'
# RHEL 7.x builds end here
# RHEL builds begin here
- job: Set_4
strategy:
matrix:
CentOS-8.0:
DistroVer: 'CentOS-8.0'
AgentName: 'COS 8.0'
Oracle-8.1:
DistroVer: 'Oracle-8.1'
AgentName: 'ORA 8.1'
RHEL-8.1:
DistroVer: 'RHEL-8.1'
AgentName: 'RHEL 8.1'
RHEL-8.2:
DistroVer: 'RHEL-8.2'
AgentName: 'RHEL 8.2'
pool:
name: 'BlobFuse Pool'
demands:
- Agent.Name -equals $(AgentName)
variables:
- group: NightlyBlobFuse
- name: root_dir
value: '/usr/pipeline/workv2'
- name: work_dir
value: '/usr/pipeline/workv2/azure-storage-fuse'
steps:
- checkout: none
# clearing out remnants from past builds
- script: |
sudo rm -rf $(root_dir)
sudo mkdir -p $(root_dir)
sudo chown -R `whoami` $(root_dir)
chmod 777 $(root_dir)
displayName: 'Create Directory Structure'
- script: |
git clone https://github.com/Azure/azure-storage-fuse
displayName: 'Checkout Code & List commits'
workingDirectory: $(root_dir)
# listing commits from past 12hrs
- script: |
git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`
git --no-pager log --since="12 hours ago" --stat
displayName: 'List Commits'
workingDirectory: $(work_dir)
# installing dependencies for building blobfuse
- script: |
sudo yum install gcc gcc-c++ make gnutls-devel rpm-build -y
sudo yum install fuse3-devel -y
sudo yum install ruby-devel -y
sudo gem install fpm -V
displayName: "Installing Dependencies"
# generating binaries
- template: 'azure-pipeline-templates/build-release.yml'
parameters:
work_dir: $(work_dir)
root_dir: $(root_dir)
unit_test: ${{ parameters.unit_test }}
# accumulating binaries & necessary scripts for packaging
- script: |
mkdir -p pkgDir/usr/bin/
mkdir -p pkgDir/usr/share/blobfuse2/
cp azure-storage-fuse/blobfuse2 pkgDir/usr/bin/blobfuse2
cp azure-storage-fuse/setup/baseConfig.yaml pkgDir/usr/share/blobfuse2/
cp azure-storage-fuse/sampleFileCacheConfig.yaml pkgDir/usr/share/blobfuse2/
cp azure-storage-fuse/sampleStreamingConfig.yaml pkgDir/usr/share/blobfuse2/
mkdir -p pkgDir/etc/rsyslog.d
mkdir -p pkgDir/etc/logrotate.d
cp azure-storage-fuse/setup/11-blobfuse2.conf pkgDir/etc/rsyslog.d
cp azure-storage-fuse/setup/blobfuse2-logrotate pkgDir/etc/logrotate.d/blobfuse2
workingDirectory: $(root_dir)
displayName: 'Accumulate pkg files'
# using fpm tool for packaging binaries & running post-install scripts
- script: |
fpm -s dir -t rpm -n blobfuse2 -C pkgDir/ -v `./pkgDir/usr/bin/blobfuse2 --version | cut -d " " -f 3` -d fuse \
--maintainer "Blobfuse v-Team <blobfusevteam@microsoft.com>" --url "https://github.com/Azure/azure-storage-fuse" \
--description "An user-space filesystem for interacting with Azure Storage" \
--before-install ./azure-storage-fuse/tools/preinstall.sh \
--after-install ./azure-storage-fuse/tools/postinstall.sh
mv ./blobfuse2*.rpm ./blobfuse2-`./pkgDir/usr/bin/blobfuse2 --version | cut -d " " -f 3`-$(DistroVer)-x86-64.rpm
cp ./blobfuse2*.rpm $(Build.ArtifactStagingDirectory)
workingDirectory: $(root_dir)
displayName: 'Make Package'
# publishing artifacts
- task: PublishBuildArtifacts@1
inputs:
artifactName: 'blobfuse2'
displayName: 'Publish Artifacts'
# RHEL builds end here.
- job: Set_5
strategy:
matrix:
SUSE-15G2:
DistroVer: 'Suse-15Gen2'
AgentName: 'SUSE 15 Blobfuse2-Release'
pool:
name: 'Blobfuse Pool'
demands:
- Agent.Name -equals $(AgentName)
variables:
- group: NightlyBlobFuse
- name: root_dir
value: '/usr/pipeline/workv2'
- name: work_dir
value: '/usr/pipeline/workv2/azure-storage-fuse'
steps:
- checkout: none
- script: |
sudo rm -rf $(root_dir)
sudo mkdir -p $(root_dir)
sudo chown -R `whoami` $(root_dir)
chmod 777 $(root_dir)
displayName: 'Create Directory Structure'
- script: |
git clone https://github.com/Azure/azure-storage-fuse
displayName: 'Checkout Code & List commits'
workingDirectory: $(root_dir)
- script: |
git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`
git --no-pager log --since="12 hours ago" --stat
displayName: 'List Commits'
workingDirectory: $(work_dir)
- script: |
curl -sSL https://rvm.io/mpapis.asc | gpg2 --import -
curl -sSL https://rvm.io/pkuczynski.asc | gpg2 --import -
curl -L get.rvm.io | bash -s stable
source ~/.rvm/scripts/rvm
yes | rvm requirements run
rvm reload
displayName: "Installing Dependencies"
- template: 'azure-pipeline-templates/build-release.yml'
parameters:
work_dir: $(work_dir)
root_dir: $(root_dir)
unit_test: ${{ parameters.unit_test }}
tags: fuse3
- script: |
mkdir -p pkgDir/usr/bin/
cp azure-storage-fuse/blobfuse2 pkgDir/usr/bin/blobfuse2
workingDirectory: $(root_dir)
displayName: 'Accumulate pkg files'
- script: |
sudo gem install fpm -v 1.14.1
sudo zypper in rpmbuild
export PATH=$PATH:/usr/lib64/ruby/gems/2.5.0/gems/fpm-1.14.1/bin
fpm -s dir -t rpm -n blobfuse2 -C pkgDir/ -v `./pkgDir/usr/bin/blobfuse2 --version | cut -d " " -f 3` -d fuse \
--maintainer "Blobfuse v-Team <blobfusevteam@microsoft.com>" --url "https://github.com/Azure/azure-storage-fuse" \
--description "An user-space filesystem for interacting with Azure Storage" \
--after-install ./azure-storage-fuse/tools/postinstall.sh
mv ./blobfuse2*.rpm ./blobfuse2-`./pkgDir/usr/bin/blobfuse2 --version | cut -d " " -f 3`-$(DistroVer)-x86-64.rpm
cp ./blobfuse2*.rpm $(Build.ArtifactStagingDirectory)
workingDirectory: $(root_dir)
displayName: 'Make Package'
- task: PublishBuildArtifacts@1
inputs:
artifactName: 'blobfuse2'
displayName: 'Publish Artifacts'
- stage: ReleaseArtifact
dependsOn: BuildArtifact
condition: succeeded('BuildArtifact')
jobs:
- job: ReleaseBlobfuse
pool:
vmImage: 'ubuntu-20.04'
variables:
- group: NightlyBlobFuse
steps:
- checkout: none
- script: |
echo ${{ parameters.tag }}
displayName: 'Tag Name'
# download artifacts that need to be published
- task: DownloadBuildArtifacts@0
displayName: 'Download Build Artifacts'
inputs:
artifactName: 'blobfuse2'
downloadPath: $(Build.ArtifactStagingDirectory)
- script: |
sudo ls -lRt $(Build.ArtifactStagingDirectory)
md5sum $(Build.ArtifactStagingDirectory)/blobfuse2/*.deb
md5sum $(Build.ArtifactStagingDirectory)/blobfuse2/*.rpm
displayName: 'List Artifacts'
- ${{ if eq(parameters.post_release, true) }}:
# Send images for signing
- task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1
displayName: 'ESRP CodeSigning blobfuse2'
inputs:
ConnectedServiceName: 'ESRP Code signing blobfuse'
FolderPath: '$(Build.ArtifactStagingDirectory)/blobfuse2'
Pattern: '*.rpm, *.deb'
signConfigType: inlineSignParams
VerboseLogin: true
inlineOperation: |
[
{
"KeyCode" : $(ESRP_BLOBFUSE_KEY_CODE),
"OperationCode" : "LinuxSign",
"Parameters" : {},
"ToolName" : "sign",
"ToolVersion" : "1.0"
}
]
# Validate signed images have md5sum changed
- script: |
chmod 755 $(Build.ArtifactStagingDirectory)/blobfuse2/*.rpm
chmod 755 $(Build.ArtifactStagingDirectory)/blobfuse2/*.deb
rm -rf $(Build.ArtifactStagingDirectory)/blobfuse2/*.md
mv $(Build.ArtifactStagingDirectory)/blobfuse2/* $(Build.ArtifactStagingDirectory)/
rm -rf $(Build.ArtifactStagingDirectory)/blobfuse2/
displayName: 'Make Artifacts executable'
- script: |
sudo ls -lRt $(Build.ArtifactStagingDirectory)
md5sum $(Build.ArtifactStagingDirectory)/*.deb
md5sum $(Build.ArtifactStagingDirectory)/*.rpm
displayName: 'List Signed Artifacts'
# Push signed images to artifact directory
- task: PublishBuildArtifacts@1
inputs:
artifactName: 'blobfuse2-signed'
displayName: 'Publish Signed Artifacts'
# add release tags & push to github
- task: GithubRelease@1
inputs:
githubConnection: 'blobfuse-git-rel'
repositoryName: 'Azure/azure-storage-fuse'
action: 'edit'
target: '$(Build.SourceVersion)'
tagSource: 'userSpecifiedTag'
title: ${{ parameters.tag }}
tag: ${{ parameters.tag }}
assets: |
$(Build.ArtifactStagingDirectory)/*
changeLogCompareToRelease: 'lastFullRelease'
changeLogType: 'commitBased'
isDraft: ${{ parameters.draft }}
isPreRelease: ${{ parameters.prerelease }}
assetUploadMode: replace
- stage: UpdateLatestVersion
dependsOn: ReleaseArtifact
condition: succeeded('ReleaseArtifact')
jobs:
- job: UpdateVersion
pool:
vmImage: 'ubuntu-18.04'
variables:
- group: NightlyBlobFuse
steps:
- checkout: none
- ${{ if eq(parameters.update_version, true) }}:
- script: |
sudo apt-get install python3 -y
python3 --version
displayName: 'Installing Python'
# download artifacts that need to be published
- task: DownloadBuildArtifacts@0
displayName: 'Download Build Artifacts'
inputs:
artifactName: 'blobfuse2'
downloadPath: $(Build.ArtifactStagingDirectory)
# install blobfuse2
- script: |
cd $(Build.ArtifactStagingDirectory)/blobfuse2
ls | grep ubuntu-18.04
sudo apt-get install ./`ls | grep ubuntu-18.04` -y
blobfuse2 version
displayName: 'Installing blobfuse2'
# Add '-preview' to pre-release versions. For example, 2.0.0-preview.1
# Also considering that the tag and version are same
- script: |
python3 -c "import json; import requests; import sys;
if(len(sys.argv)!=3):
print('Less arguments')
sys.exit(1)
url = sys.argv[1]
version = sys.argv[2].split(' ')[2]
print('Version: ' + version)
if(len(version)==0):
print('Incorrect Release Tag')
sys.exit(1)
dataDict = requests.get(url).json()
dataDict['blobfuse2'] = version
resp = requests.put(url, data=json.dumps(dataDict, indent=4), headers={'Content-Type': 'application/json', 'x-ms-blob-type': 'BlockBlob'})
print('Status Code: ' + str(resp.status_code))
sys.exit(1) if(resp.status_code<200 or resp.status_code>202) else print('SUCCESS')" \
'$(LATEST_VERSION_SAS)' "`blobfuse2 version`"
displayName: 'Updating version number'