firefox-translations-models/.circleci/config.yml

version: 2.1

jobs:
  deploy:
    docker:
      - image: gcr.io/google.com/cloudsdktool/cloud-sdk:348.0.0
    working_directory: ~/mozilla/firefox-translations-models
    steps:
      - run:
          name: Installing git lfs
          command: |
            curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash
            apt-get install git-lfs
            git lfs install            
      - checkout
      - run:
          name: Uploading to GCS
          command: |
            bash scripts/upload.sh            
  evaluate:
    machine:
      image: linux-cuda-11:default
    resource_class: gpu.nvidia.large
    working_directory: ~/mozilla/firefox-translations-models
    steps:
      - add_ssh_keys:
          fingerprints:
            - "0d:23:4b:98:32:fe:2a:24:f6:2b:0d:3a:bc:d6:c5:2f"
      - run:
          name: Installing git lfs and nvidia-container-toolkit
          command: |
            sudo apt-get update
            sudo apt-get install -y git-lfs nvidia-container-toolkit
            sudo git lfs install            
      - run:
          name: Configure the Docker daemon to recognize the NVIDIA Container Runtime
          command: | 
            sudo nvidia-ctk runtime configure --runtime=docker
            sudo systemctl restart docker
      - checkout
      - run:
          name: Running evaluation
          command: |
            find ./models -type f -exec git lfs pull {} \;
            bash scripts/update-results.sh            
      - run:
          name: Showing results
          command: |
            git add evaluation/*/*.bleu
            git --no-pager diff --staged evaluation/*/*.bleu
            git add evaluation/*/*.comet
            git --no-pager diff --staged evaluation/*/*.comet
            git add evaluation/*/*.cometcompare
            git --no-pager diff --staged evaluation/*/*.cometcompare            
      - run:
          name: Pushing results
          command: |
            git config user.email "ci-models-evaluation@firefox-translations"
            git config user.name "CircleCI evaluation job"
            git add evaluation

            if [[ $(git status --porcelain) ]]; then
              echo "### Commiting results"
              git commit -m "Update evaluation results [skip ci]"
              git push --set-upstream origin "$CIRCLE_BRANCH"
            else
              echo "### Nothing to commit"
            fi            
  test-python-formatting:
    docker:
      - image: cimg/python:3.10.2
    working_directory: ~/mozilla/firefox-translations-models
    steps:
      - checkout
      - run:
          name: Ensure black finds no changes
          command: |
            pip install poetry
            poetry install --no-root
            poetry run black --check .
            BLACK_RESULT=$?
            if [ $BLACK_RESULT -eq 0 ]; then
              echo "Black found no code-format changes... CI passed."
            else
              echo "Black found code-format changes... Failing CI."
              exit $BLACK_RESULT
            fi            
  test-code:
    docker:
      - image: cimg/python:3.10.2
    working_directory: ~/mozilla/firefox-translations-models
    steps:
      - checkout
      - run:
          name: Run tests
          command: |
            pip install poetry
            poetry install --no-root
            poetry run python -m pytest tests -vv
            TEST_RESULT=$?
            if [ $TEST_RESULT -eq 0 ]; then
              echo "All tests passed... CI passed."
            else
              echo "Some tests failed... Failing CI."
              exit $TEST_RESULT
            fi            
  generate-registry:
    docker:
      - image: cimg/python:3.10.2
    working_directory: ~/mozilla/firefox-translations-models
    steps:
      - add_ssh_keys:
          fingerprints:
            - "0d:23:4b:98:32:fe:2a:24:f6:2b:0d:3a:bc:d6:c5:2f"
      - run:
          name: Installing git lfs
          command: |
            sudo apt-get update
            sudo apt-get install -y git-lfs
            sudo git lfs install            
      - checkout
      - run:
          name: Updating model registry
          command: |
            python scripts/generate-registry.py            
      - run:
          name: Showing results
          command: |
            git add registry.json
            git --no-pager diff --staged registry.json            
      - run:
          name: Pushing results
          command: |
            git config user.email "ci-models-evaluation@firefox-translations"
            git config user.name "CircleCI evaluation job"
            git pull origin "$CIRCLE_BRANCH"
            git add registry.json

            if [[ $(git status --porcelain) ]]; then
              echo "### Commiting results"
              git commit -m "Update model registry [skip ci]"
              git push --set-upstream origin "$CIRCLE_BRANCH"
            else
              echo "### Nothing to commit"
            fi            
workflows:
  version: 2
  ci:
    jobs:
      - test-python-formatting
      - test-code:
          requires:
            - test-python-formatting
      - evaluate:
          requires:
            - test-python-formatting
          filters:
            branches:
              # Forked pull requests have CIRCLE_BRANCH set to pull/XXX
              ignore:
                - /pull\/[0-9]+/
                - main
      - generate-registry:
          requires:
            - evaluate
          filters:
            branches:
              # Forked pull requests have CIRCLE_BRANCH set to pull/XXX
              ignore:
                - /pull\/[0-9]+/
                - main
  publish:
    jobs:
      - deploy:
          filters:
            branches:
              ignore: /.*/
            tags:
              only: /\d*\.\d*\.\d*/