azure-storage-fuse/blobfuse2-data-validation.yaml

parameters:
  - name: verbose_log
    displayName: 'Verbose Log'
    type: boolean
    default: false


stages:
  - stage: KernelBuild 
    jobs:
      # Ubuntu Tests
      - job: Set_1
        timeoutInMinutes: 900
        strategy:
          matrix:
            Ubuntu-22:
              containerName: 'validation'
        pool:
          name: "blobfuse-validation-pool"

        variables:
          - group: NightlyBlobFuse
          - name: ROOT_DIR
            value: "/usr/pipeline/workv2"
          - name: WORK_DIR
            value: "/usr/pipeline/workv2/go/src/azure-storage-fuse"
          - name: MOUNT_DIR
            value: "/usr/pipeline/workv2/blob_mnt"
          - name: TEMP_DIR
            value: "/usr/pipeline/workv2/temp"
          - name: BLOBFUSE2_CFG
            value: "/usr/pipeline/workv2/blobfuse2.yaml"
          - name: GOPATH
            value: "/usr/pipeline/workv2/go"

        steps:
          - template: 'azure-pipeline-templates/setup.yml'
            parameters:
              tags: $(tags)
              installStep:
                script: |
                  sudo apt-get update --fix-missing
                  sudo apt update
                  sudo apt-get install cmake gcc libfuse3-dev git parallel -y
                  sudo apt-get install fuse3 -y
                displayName: 'Install fuse'

          - script: |
              sudo apt-get install git fakeroot build-essential ncurses-dev xz-utils libssl-dev bc flex libelf-dev bison -y
            displayName: 'Install kernel build dependencies'

          - script: |
              cd $(WORK_DIR)
              $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_block_bench.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG)
            displayName: "Create Config File"
            env:
                NIGHTLY_STO_ACC_NAME: $(BLOBFUSE_BENCHMARK_ACCOUNT)
                NIGHTLY_STO_ACC_KEY: $(BLOBFUSE_BENCHMARK_ACC_KEY)
                ACCOUNT_TYPE: "block"
                ACCOUNT_ENDPOINT: https://$(BLOBFUSE_BENCHMARK_ACCOUNT).blob.core.windows.net
            continueOnError: false
              
          - script: |
              cat $(BLOBFUSE2_CFG)
            displayName: 'Print config file'

          - template: 'azure-pipeline-templates/blobfuse2-kernel-test.yml'
            parameters:
              mount_dir: $(MOUNT_DIR)
              temp_dir: $(TEMP_DIR)
              kversion: "6.10.2"
              idstring: 'File-Cache'
              config_file: $(BLOBFUSE2_CFG)

  - stage: StripedWrite
    jobs:
      # Ubuntu Tests
      - job: Set_1
        timeoutInMinutes: 900
        strategy:
          matrix:
            Ubuntu-22:
              containerName: 'validation'
        pool:
          name: "blobfuse-validation-pool"

        variables:
          - group: NightlyBlobFuse
          - name: ROOT_DIR
            value: "/usr/pipeline/workv2"
          - name: WORK_DIR
            value: "/usr/pipeline/workv2/go/src/azure-storage-fuse"
          - name: MOUNT_DIR
            value: "/usr/pipeline/workv2/blob_mnt"
          - name: TEMP_DIR
            value: "/usr/pipeline/workv2/temp"
          - name: BLOBFUSE2_CFG
            value: "/usr/pipeline/workv2/blobfuse2.yaml"
          - name: GOPATH
            value: "/usr/pipeline/workv2/go"

        steps:
          - template: 'azure-pipeline-templates/setup.yml'
            parameters:
              tags: $(tags)
              installStep:
                script: |
                  sudo apt-get update --fix-missing
                  sudo apt update
                  sudo apt-get install cmake gcc libfuse3-dev git parallel -y
                  sudo apt-get install fuse3 -y
                displayName: 'Install fuse'

          - script: |
              cd $(WORK_DIR)
              $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_block_bench.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG)
            displayName: Create Key Config File
            env:
              AZURE_STORAGE_ACCOUNT: $(BLOBFUSE_BENCHMARK_ACCOUNT)
              AZURE_STORAGE_ACCESS_KEY: $(BLOBFUSE_BENCHMARK_ACC_KEY)
              continueOnError: false

          - script: cat $(BLOBFUSE2_CFG)
            displayName: Print config file

          - script:
              mkdir -p $(MOUNT_DIR)
              mkdir -p $(TEMP_DIR)
            displayName: "Create directory structure"

          - template: 'azure-pipeline-templates/mount.yml'
            parameters:
              working_dir: $(WORK_DIR)
              mount_dir: $(MOUNT_DIR)
              temp_dir: $(TEMP_DIR)
              ro_mount: false
              mountStep: 
                script: |
                  $(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) --default-working-dir=$(WORK_DIR)
          
          - script: |
              rm -rf $(MOUNT_DIR)/striped
              mkdir -p $(MOUNT_DIR)/striped
              cd $(MOUNT_DIR)/striped
              $(WORK_DIR)/test/scripts/striped_write.sh
            displayName: 'Run striped write script'  

          - script: |
              rm -rf $(MOUNT_DIR)/striped
            displayName: 'Cleanup data' 

          # Cleanup
          - template: 'azure-pipeline-templates/cleanup.yml'
            parameters:
              working_dir: $(WORK_DIR)
              mount_dir: $(MOUNT_DIR)
              temp_dir: $(TEMP_DIR)

  - stage: VScodeBuild
    jobs:
      # Ubuntu Tests
      - job: Set_1
        timeoutInMinutes: 900
        strategy:
          matrix:
            Ubuntu-22:
              containerName: 'validation'
        pool:
          name: "blobfuse-validation-pool"

        variables:
          - group: NightlyBlobFuse
          - name: ROOT_DIR
            value: "/usr/pipeline/workv2"
          - name: WORK_DIR
            value: "/usr/pipeline/workv2/go/src/azure-storage-fuse"
          - name: MOUNT_DIR
            value: "/usr/pipeline/workv2/blob_mnt"
          - name: TEMP_DIR
            value: "/usr/pipeline/workv2/temp"
          - name: BLOBFUSE2_CFG
            value: "/usr/pipeline/workv2/blobfuse2.yaml"
          - name: GOPATH
            value: "/usr/pipeline/workv2/go"

        steps:
          - template: 'azure-pipeline-templates/setup.yml'
            parameters:
              tags: $(tags)
              installStep:
                script: |
                  sudo apt-get update --fix-missing
                  sudo apt update
                  sudo apt-get install cmake gcc libfuse3-dev git parallel -y
                  sudo apt-get install fuse3 -y
                displayName: 'Install fuse'

          - script: |
              cd $(WORK_DIR)
              $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_block_bench.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG)
            displayName: Create Key Config File
            env:
              AZURE_STORAGE_ACCOUNT: $(BLOBFUSE_BENCHMARK_ACCOUNT)
              AZURE_STORAGE_ACCESS_KEY: $(BLOBFUSE_BENCHMARK_ACC_KEY)
              continueOnError: false

          - script: cat $(BLOBFUSE2_CFG)
            displayName: Print config file

          - script:
              mkdir -p $(MOUNT_DIR)
              mkdir -p $(TEMP_DIR)
            displayName: "Create directory structure"

          - template: 'azure-pipeline-templates/mount.yml'
            parameters:
              working_dir: $(WORK_DIR)
              mount_dir: $(MOUNT_DIR)
              temp_dir: $(TEMP_DIR)
              ro_mount: false
              mountStep: 
                script: |
                  $(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) --default-working-dir=$(WORK_DIR)
          
          - script: |
              cd $(MOUNT_DIR)
              git clone https://github.com/microsoft/vscode.git
            displayName: 'Clone VScode'

          - script: |
              cd $(MOUNT_DIR)/vscode
              curl -fsSL https://deb.nodesource.com/setup_20.x | sudo -E bash -
              sudo apt-get install -y nodejs
            displayName: 'Install dependencies'

          - script: |
              cd $(MOUNT_DIR)/vscode
              sudo npm install -g yarn
            displayName: 'Install yarn'

          - script: | 
              cd $(MOUNT_DIR)/vscode
              yarn add ternary-stream json-stable-stringify-without-jsonify
            displayName: 'Add modules'

          - script: |
              cd $(MOUNT_DIR)/vscode
              sudo yarn 
            displayName: 'Build VScode'

          - script: |
              cd $(MOUNT_DIR)/vscode
              yarn compile
            displayName: 'Compile VScode'
          
          - script: |
              cd $(MOUNT_DIR)/
              rm -rf vscode
              rm -rf vscode
              rm -rf vscode
            displayName: 'Cleanup Mount'

          # Cleanup
          - template: 'azure-pipeline-templates/cleanup.yml'
            parameters:
              working_dir: $(WORK_DIR)
              mount_dir: $(MOUNT_DIR)
              temp_dir: $(TEMP_DIR)

  - stage: FineTuningTests
    jobs:
      # Ubuntu Tests
      - job: Set_1
        timeoutInMinutes: 900
        strategy:
          matrix:
            Ubuntu-22:
              containerName: 'finetuning'
        pool:
          name: "blobfuse-validation-pool"

        variables:
          - group: NightlyBlobFuse
          - name: ROOT_DIR
            value: "/usr/pipeline/workv2"
          - name: WORK_DIR
            value: "/usr/pipeline/workv2/go/src/azure-storage-fuse"
          - name: MOUNT_DIR
            value: "/mnt/blobfusemount"
          - name: TEMP_DIR
            value: "/usr/pipeline/workv2/temp"
          - name: BLOBFUSE2_CFG
            value: "/usr/pipeline/workv2/blobfuse2.yaml"
          - name: GOPATH
            value: "/usr/pipeline/workv2/go"

        steps:
          - template: 'azure-pipeline-templates/setup.yml'
            parameters:
              tags: $(tags)
              installStep:
                script: |
                  sudo apt-get update --fix-missing
                  sudo apt update
                  sudo apt-get install cmake gcc libfuse3-dev git parallel -y
                  sudo apt-get install fuse3 -y
                displayName: 'Install fuse'

          - script: |
              sudo apt-get install python3 -y
              python3 --version
            displayName: 'Installing Python'

          - script: |
              cd $(WORK_DIR)
              $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_block_bench.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG)
            displayName: Create Key Config File
            env:
              AZURE_STORAGE_ACCOUNT: $(BLOBFUSE_BENCHMARK_ACCOUNT)
              AZURE_STORAGE_ACCESS_KEY: $(BLOBFUSE_BENCHMARK_ACC_KEY)
            continueOnError: false

          - script: cat $(BLOBFUSE2_CFG)
            displayName: Print config file

          - script:
              mkdir -p $(MOUNT_DIR)
              mkdir -p $(TEMP_DIR)
            displayName: "Create directory structure"

          - template: 'azure-pipeline-templates/mount.yml'
            parameters:
              working_dir: $(WORK_DIR)
              mount_dir: $(MOUNT_DIR)
              temp_dir: $(TEMP_DIR)
              ro_mount: false
              mountStep: 
                script: |
                  $(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) --default-working-dir=$(WORK_DIR)
          
          - script: |
              cd $(MOUNT_DIR)
              python3 finetuning/finetuning.py
            displayName: 'Run FineTuning Tests'

          - script: |
              cd $(MOUNT_DIR)
              python3 finetuning/loadcheckpoint.py
              expect -c 'spawn python3 loadcheckpoint.py; send "3\r"; interact'
            displayName: 'Load checkpoints'

          # Cleanup
          - template: 'azure-pipeline-templates/cleanup.yml'
            parameters:
              working_dir: $(WORK_DIR)
              mount_dir: $(MOUNT_DIR)
              temp_dir: $(TEMP_DIR)