azure-storage-fuse/blobfuse2-gen1-artifacts.yaml

# Pipeline to build artifact for gen1 compliant adlsgen1fuse

# Do not trigger this pipeline automatically
trigger: none
pr: none

stages:

  # ---------------------- Build adlsgen1fuse here and copy binary to artifact ------------------------------
  - stage: BuildAdlsGen1Fuse
    jobs:
      - job: AdlsGen1FuseBuild

        # Define on what all OS we want to run this Pipeline
        strategy:
          matrix:
            Ubuntu-18:
              container_image: 'ubuntu-18.04'
              build_type: Release
        pool:
          vmImage: $(container_image)

        variables:
        - group: NightlyBlobFuse
        - name: ROOT_DIR
          value: '$(System.DefaultWorkingDirectory)'
        - name: WORK_DIR
          value: '$(System.DefaultWorkingDirectory)/BlobFuse-ADLSGen1'

        steps:
          - checkout: none

          # Install all dependencies
          - script: |
              sudo apt-get update --fix-missing
              sudo apt-get install pkg-config cmake libcurl4-gnutls-dev libgnutls28-dev uuid-dev libgcrypt20-dev libboost-all-dev gcc g++ -y
              sudo apt-get install libfuse-dev -y              
            displayName: "Basic Tools Setup"

          - script: |
              sudo apt update
              sudo apt upgrade
              sudo apt-get install pkg-config clang-9 -y
              curl https://sh.rustup.rs -sSf | sh -s -- -y
              rustup install nightly
              rustup install nightly-2021-09-01
              rustup default nightly-2021-09-01-x86_64-unknown-linux-gnu
              sudo apt install llvm
              llvm-config --prefix              
            displayName: "Rust Setup"
            
          - script: |
              git clone https://$(DEVOPS_UNAME):$(DEVOPS_PWD)@dev.azure.com/azstorage/_git/BlobFuse-ADLSGen1              
            displayName: "Checkout code"
            workingDirectory: $(ROOT_DIR)

          # List commits consumed for this build
          - script: |
              echo "Below commits were done in last 12 hours : " `date` 
              git --no-pager log --since="12 hours ago" --stat              
            displayName: "List Commits"
            workingDirectory: $(WORK_DIR)
            
          # Build adlsgen1fuse binary
          - script: |
              cd call-datasetfuse-gen1
              { echo 'cargo-features = ["edition2021"]'; cat Cargo.toml; } > Cargo_new.toml
              mv Cargo_new.toml Cargo.toml
              cargo build --release              
            displayName: "Build the binary"
            workingDirectory: $(WORK_DIR)
          
          - script: |
              cd call-datasetfuse-gen1
              cp ../target/release/call-gen1datasetfuse $(Build.ArtifactStagingDirectory)/adlsgen1fuse              
            displayName: "Copy Artifact"
            workingDirectory: $(WORK_DIR)
          
          # Publish the Artifacts
          - task: PublishBuildArtifacts@1
            inputs:
              artifactName: 'rustbin'
            displayName: 'Publish Artifacts for adlsgen1fuse'
            condition: succeeded()

  # ---------------------- Build blobfuse and create a .deb package ------------------------------
  - stage: BuildArtifact
    dependsOn: BuildAdlsGen1Fuse
    condition: succeeded('BuildAdlsGen1Fuse')
    
    jobs:
      # Ubuntu builds
      - job: BlobfuseBuild
        strategy:
          matrix:
            Ubuntu-18:
              container_image: 'ubuntu-18.04'
              fuselib: 'libfuse-dev'
              tags: 'fuse2'
        
        pool:
          vmImage: $(container_image)

        variables:
          - group: NightlyBlobFuse
          - name: root_dir
            value: '$(System.DefaultWorkingDirectory)'
          - name: work_dir
            value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse'
        
        steps:
          - checkout: none
          - script: |
              git clone https://github.com/Azure/azure-storage-fuse              
            displayName: 'Checkout Code & List commits'
            workingDirectory: $(root_dir)
          
          # list commits from past 12hrs
          - script: |
              git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`
              git --no-pager log --since="12 hours ago" --stat              
            displayName: 'List Commits'
            workingDirectory: $(work_dir)
          
          # install dependencies required for compiling blobfuse
          - script: |
              sudo apt-get update --fix-missing
              sudo apt-get install ruby-dev build-essential $(fuselib) -y
              sudo gem install fpm -V              
            displayName: "Installing Dependencies"

          # Copy adlsgen1fuse binary to blobfuse root
          - task: DownloadBuildArtifacts@0
            displayName: 'Download AdlsGen1Fuse Artifacts'
            inputs:
              artifactName: 'rustbin'
              downloadPath: $(work_dir)

          - script: |
              cp ./rustbin/* ./
              rm -rf ./rustbin/*
              chmod 777 ./adlsgen1fuse
              ls -l ./adlsgen1fuse              
            displayName: "Prepare for packaging"
            workingDirectory: $(work_dir)
          
          # build blobfuse2 and generate binary
          - template: 'azure-pipeline-templates/build-release.yml'
            parameters:
              work_dir: $(work_dir)
              root_dir: $(root_dir)
              tags: $(tags)

          # place the generated binary files & any additional files in appropriate locations
          - script: |
              mkdir -p pkgDir/usr/bin/
              mkdir -p pkgDir/usr/share/blobfuse2/
              cp azure-storage-fuse/blobfuse2 pkgDir/usr/bin/blobfuse2
              cp azure-storage-fuse/adlsgen1fuse pkgDir/usr/bin/adlsgen1fuse
              cp azure-storage-fuse/setup/baseConfig.yaml pkgDir/usr/share/blobfuse2/
              cp azure-storage-fuse/sampleFileCacheConfig.yaml pkgDir/usr/share/blobfuse2/
              cp azure-storage-fuse/sampleStreamingConfig.yaml pkgDir/usr/share/blobfuse2/
              mkdir -p pkgDir/etc/rsyslog.d
              mkdir -p pkgDir/etc/logrotate.d
              cp azure-storage-fuse/setup/11-blobfuse2.conf pkgDir/etc/rsyslog.d
              cp azure-storage-fuse/setup/blobfuse2-logrotate pkgDir/etc/logrotate.d/blobfuse2              
            workingDirectory: $(root_dir)
            displayName: 'Accumulate pkg files'
          
          # using fpm tool for packaging of our binary & performing post-install operations
          # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/
          - script: |
              fpm -s dir -t deb -n blobfuse2 -C pkgDir/ -v `./pkgDir/usr/bin/blobfuse2 --version | cut -d " " -f 3` -d fuse \
              --maintainer "Blobfuse v-Team <blobfusevteam@microsoft.com>" --url "https://github.com/Azure/azure-storage-fuse" \
              --description "An user-space filesystem for interacting with Azure Storage" \
              --before-install ./azure-storage-fuse/tools/preinstall.sh \
              --after-install ./azure-storage-fuse/tools/postinstall.sh
              mv ./blobfuse2*.deb ./blobfuse2-`./pkgDir/usr/bin/blobfuse2 --version | cut -d " " -f 3`-$(container_image)-x86-64.deb
              cp ./blobfuse2*.deb $(Build.ArtifactStagingDirectory)              
            workingDirectory: $(root_dir)
            displayName: 'Make Package'
          
          # publishing the artifacts generated
          - task: PublishBuildArtifacts@1
            inputs:
              artifactName: 'blobfuse2'
            displayName: 'Publish Artifacts'