Fix GitHub Actions for GitHub Enterprise (#4)
* Reformat coalb files (#2) * Update GitHub actions for GitHub Enterprise Compat (#1) ### Proposed change(s) Update the GitHub actions to work in GitHub Enterprise (move from public to enterprise + public mirror). ### Useful links (Github issues, JIRA tickets, ML-Agents forum threads etc.) https://jira.unity3d.com/browse/MLA-2344 ### Types of change(s) - [x] Code refactor ### Depends on https://github.cds.internal.unity3d.com/unity/actions-syncer/pull/44 ### Depends on #2 to reformat some colab notebooks ### Verified working actions vs untested (need to test once this is checked in): - [x] colab - [x] pre-commit - [x] pytest - [ ] nightly (will automatically be tested upon submission) - [ ] publish pypi (this needs to be careful) - [ ] Lock Threads (should only run on public mirror of repo once mirroring is complete) * Reformat coalb files (#2) * Update GitHub actions for GitHub Enterprise Compat (#1) ### Proposed change(s) Update the GitHub actions to work in GitHub Enterprise (move from public to enterprise + public mirror). ### Useful links (Github issues, JIRA tickets, ML-Agents forum threads etc.) https://jira.unity3d.com/browse/MLA-2344 ### Types of change(s) - [x] Code refactor ### Depends on https://github.cds.internal.unity3d.com/unity/actions-syncer/pull/44 ### Depends on #2 to reformat some colab notebooks ### Verified working actions vs untested (need to test once this is checked in): - [x] colab - [x] pre-commit - [x] pytest - [ ] nightly (will automatically be tested upon submission) - [ ] publish pypi (this needs to be careful) - [ ] Lock Threads (should only run on public mirror of repo once mirroring is complete)
This commit is contained in:
Henry Peteet
GitHub Enterprise
Родитель
71b121ca7a
Коммит
b388f51b69
|
|
@ -1,46 +0,0 @@
|
|||
name: build and push Docker image
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "release_[0-9]+"
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: google-github-actions/setup-gcloud@v0
|
||||
with:
|
||||
version: '270.0.0'
|
||||
service_account_email: ${{ secrets.GCP_STG_EMAIL }}
|
||||
service_account_key: ${{ secrets.GCP_STG_PRIVATE_KEY }}
|
||||
export_default_credentials: true
|
||||
- name: Configure GCR Docker auth (stg)
|
||||
run: gcloud auth configure-docker
|
||||
- name: Build docker image
|
||||
run: docker build --tag ml-agents:${GITHUB_SHA} --build-arg SHA=${GITHUB_SHA} .
|
||||
- name: Tag docker image (stg)
|
||||
run: |
|
||||
docker tag ml-agents:${GITHUB_SHA} gcr.io/unity-ai-ml-agents-stg/ml-agents:${GITHUB_SHA}
|
||||
docker tag ml-agents:${GITHUB_SHA} gcr.io/unity-ai-ml-agents-stg/ml-agents:latest
|
||||
- name: Push docker image (stg)
|
||||
run: |
|
||||
docker push gcr.io/unity-ai-ml-agents-stg/ml-agents:${GITHUB_SHA}
|
||||
docker push gcr.io/unity-ai-ml-agents-stg/ml-agents:latest
|
||||
- uses: google-github-actions/setup-gcloud@v0
|
||||
with:
|
||||
version: '270.0.0'
|
||||
service_account_email: ${{ secrets.GCP_PRD_EMAIL }}
|
||||
service_account_key: ${{ secrets.GCP_PRD_PRIVATE_KEY }}
|
||||
export_default_credentials: true
|
||||
- name: Configure GCR Docker auth (prd)
|
||||
run: gcloud auth configure-docker
|
||||
- name: Tag docker image (prd)
|
||||
run: |
|
||||
docker tag ml-agents:${GITHUB_SHA} gcr.io/unity-ai-ml-agents-prd/ml-agents:${GITHUB_SHA}
|
||||
docker tag ml-agents:${GITHUB_SHA} gcr.io/unity-ai-ml-agents-prd/ml-agents:latest
|
||||
- name: Push docker image (prd)
|
||||
run: |
|
||||
docker push gcr.io/unity-ai-ml-agents-prd/ml-agents:${GITHUB_SHA}
|
||||
docker push gcr.io/unity-ai-ml-agents-prd/ml-agents:latest
|
||||
|
|
@ -9,16 +9,20 @@ on:
|
|||
- '.github/workflows/colab.yml'
|
||||
push:
|
||||
branches: [main]
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
colab:
|
||||
runs-on: ubuntu-latest
|
||||
# Only run on GitHub Enterprise
|
||||
if: ${{ github.server_url == 'https://github.cds.internal.unity3d.com' }}
|
||||
runs-on: [self-hosted, Linux, X64]
|
||||
env:
|
||||
COLAB_ALWAYS_INSTALL_XVFB: 1
|
||||
QLEARNING_NUM_TRAINING_STEPS: 5
|
||||
QLEARNING_NUM_NEW_EXP: 64
|
||||
QLEARNING_BUFFER_SIZE: 64
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
notebook_path: [Colab_UnityEnvironment_1_Run.ipynb, Colab_UnityEnvironment_2_Train.ipynb, Colab_UnityEnvironment_3_SideChannel.ipynb]
|
||||
steps:
|
||||
|
|
@ -29,17 +33,19 @@ jobs:
|
|||
python-version: 3.8.x
|
||||
- uses: actions/setup-dotnet@v1
|
||||
with:
|
||||
dotnet-version: '3.1.x'
|
||||
- name: Cache pip
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
# This path is specific to Ubuntu
|
||||
path: ~/.cache/pip
|
||||
# Look to see if there is a cache hit for the corresponding requirements file
|
||||
key: ${{ runner.os }}-pip-${{ hashFiles('ml-agents/setup.py', 'ml-agents-envs/setup.py', 'gym-unity/setup.py', 'colab_requirements.txt') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-pip-
|
||||
${{ runner.os }}-
|
||||
dotnet-version: '6.0.x'
|
||||
# Caching not supported on GitHub Enterprise
|
||||
# See https://github.com/actions/cache/issues/505
|
||||
# - name: Cache pip
|
||||
# uses: actions/cache@v2
|
||||
# with:
|
||||
# # This path is specific to Ubuntu
|
||||
# path: ~/.cache/pip
|
||||
# # Look to see if there is a cache hit for the corresponding requirements file
|
||||
# key: ${{ runner.os }}-pip-${{ hashFiles('ml-agents/setup.py', 'ml-agents-envs/setup.py', 'gym-unity/setup.py', 'colab_requirements.txt') }}
|
||||
# restore-keys: |
|
||||
# ${{ runner.os }}-pip-
|
||||
# ${{ runner.os }}-
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
|
|
@ -49,7 +55,7 @@ jobs:
|
|||
python -m pip install --progress-bar=off -e ./ml-agents
|
||||
python -m pip install --progress-bar=off -r colab_requirements.txt
|
||||
- name: Execute notebook
|
||||
run: jupyter nbconvert --to notebook --execute --log-level=INFO --ExecutePreprocessor.kernel_name=python3 --output output-${{ matrix.notebook_path }} colab/${{ matrix.notebook_path }}
|
||||
run: jupyter nbconvert --to notebook --execute --log-level=DEBUG --ExecutePreprocessor.kernel_name=python3 --output output-${{ matrix.notebook_path }} colab/${{ matrix.notebook_path }}
|
||||
- name: Upload colab results
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
|
|
|
|||
|
|
@ -3,12 +3,15 @@ name: 'Lock Threads'
|
|||
on:
|
||||
schedule:
|
||||
- cron: '0 0/4 * * *'
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
lock:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: dessant/lock-threads@v2
|
||||
# Only lock threads on public GitHub mirror, not enterprise.
|
||||
if: ${{ github.server_url == 'https://github.com' }}
|
||||
with:
|
||||
github-token: ${{ github.token }}
|
||||
issue-lock-inactive-days: '30'
|
||||
|
|
|
|||
|
|
@ -3,17 +3,24 @@ name: nightly
|
|||
on:
|
||||
schedule:
|
||||
- cron: '0 7 * * *' # run at 7 AM UTC == midnight PST
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
markdown-link-check-full:
|
||||
runs-on: ubuntu-latest
|
||||
# Only run on GitHub Enterprise
|
||||
if: ${{ github.server_url == 'https://github.cds.internal.unity3d.com' }}
|
||||
runs-on: [self-hosted, Linux, X64]
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-python@v1
|
||||
- uses: actions/setup-node@v2-beta
|
||||
- uses: actions/setup-python@v2
|
||||
- uses: actions/setup-node@v2
|
||||
with:
|
||||
node-version: '12'
|
||||
- run: sudo npm install -g markdown-link-check
|
||||
- uses: pre-commit/action@v2.0.0
|
||||
with:
|
||||
extra_args: --hook-stage manual markdown-link-check-full --all-files
|
||||
- name: Install manual dependencies
|
||||
run: |
|
||||
sudo npm install -g markdown-link-check
|
||||
python -m pip install pre-commit
|
||||
pre-commit install
|
||||
- name: Run markdown checker
|
||||
run: |
|
||||
pre-commit run --hook-stage manual markdown-link-check-full --all-files
|
||||
|
|
|
|||
|
|
@ -4,40 +4,56 @@ on:
|
|||
pull_request:
|
||||
push:
|
||||
branches: [main]
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
pre-commit:
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ github.server_url == 'https://github.cds.internal.unity3d.com' }}
|
||||
runs-on: [self-hosted, Linux, X64]
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: 3.7.x
|
||||
- uses: actions/setup-ruby@v1
|
||||
env:
|
||||
ImageOS: ubuntu20
|
||||
with:
|
||||
ruby-version: '2.6'
|
||||
- uses: actions/setup-dotnet@v1
|
||||
with:
|
||||
dotnet-version: '6.0.x'
|
||||
include-prerelease: true
|
||||
- uses: pre-commit/action@v2.0.0
|
||||
|
||||
- name: Install manual dependencies
|
||||
run: |
|
||||
python -m pip install pre-commit
|
||||
pre-commit install
|
||||
- name: Run pre-commit
|
||||
run: |
|
||||
pre-commit run --all-files
|
||||
markdown-link-check:
|
||||
runs-on: ubuntu-latest
|
||||
# Only run on GitHub Enterprise
|
||||
if: ${{ github.server_url == 'https://github.cds.internal.unity3d.com' }}
|
||||
runs-on: [self-hosted, Linux, X64]
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-python@v1
|
||||
- uses: actions/setup-node@v2-beta
|
||||
- uses: actions/setup-python@v2
|
||||
- uses: actions/setup-node@v2
|
||||
with:
|
||||
node-version: '12'
|
||||
- run: sudo npm install -g markdown-link-check
|
||||
- uses: pre-commit/action@v2.0.0
|
||||
with:
|
||||
extra_args: --hook-stage manual markdown-link-check --all-files
|
||||
|
||||
- name: Install manual dependencies
|
||||
run: |
|
||||
sudo npm install -g markdown-link-check
|
||||
python -m pip install pre-commit
|
||||
pre-commit install
|
||||
- name: Run markdown checker
|
||||
run: |
|
||||
pre-commit run --hook-stage manual markdown-link-check --all-files
|
||||
validate-meta-files:
|
||||
runs-on: ubuntu-latest
|
||||
# Only run on GitHub Enterprise
|
||||
if: ${{ github.server_url == 'https://github.cds.internal.unity3d.com' }}
|
||||
runs-on: [self-hosted, Linux, X64]
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-python@v1
|
||||
- uses: actions/setup-python@v2
|
||||
- run: python utils/validate_meta_files.py
|
||||
|
|
|
|||
|
|
@ -1,16 +1,19 @@
|
|||
# Adapted from https://packaging.python.org/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/
|
||||
name: publish to PyPI (or TestPyPI)
|
||||
name: Publish to PyPI (or TestPyPI)
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "release_[0-9]+_test[0-9]+"
|
||||
- "release_[0-9]+"
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
build-and-publish:
|
||||
# Only run on GitHub Enterprise
|
||||
if: ${{ github.server_url == 'https://github.cds.internal.unity3d.com' }}
|
||||
name: publish to PyPI (or TestPyPI)
|
||||
runs-on: ubuntu-18.04
|
||||
runs-on: [self-hosted, Linux, X64]
|
||||
strategy:
|
||||
matrix:
|
||||
package-path: [ml-agents, ml-agents-envs, gym-unity]
|
||||
|
|
@ -34,14 +37,14 @@ jobs:
|
|||
python setup.py bdist_wheel
|
||||
- name: Publish distribution 📦 to Test PyPI
|
||||
if: startsWith(github.ref, 'refs/tags') && contains(github.ref, 'test')
|
||||
uses: pypa/gh-action-pypi-publish@master
|
||||
uses: actions/gh-action-pypi-publish@717ba43cfbb0387f6ce311b169a825772f54d295
|
||||
with:
|
||||
password: ${{ secrets.test_pypi_password }}
|
||||
password: ${{ secrets.TEST_PYPI_PASSWORD }}
|
||||
repository_url: https://test.pypi.org/legacy/
|
||||
packages_dir: ${{ matrix.package-path }}/dist/
|
||||
- name: Publish distribution 📦 to Production PyPI
|
||||
if: startsWith(github.ref, 'refs/tags') && !contains(github.ref, 'test')
|
||||
uses: pypa/gh-action-pypi-publish@master
|
||||
uses: actions/gh-action-pypi-publish@717ba43cfbb0387f6ce311b169a825772f54d295
|
||||
with:
|
||||
password: ${{ secrets.pypi_password }}
|
||||
password: ${{ secrets.PYPI_PASSWORD }}
|
||||
packages_dir: ${{ matrix.package-path }}/dist/
|
||||
|
|
|
|||
|
|
@ -11,15 +11,22 @@ on:
|
|||
- '.github/workflows/pytest.yml'
|
||||
push:
|
||||
branches: [main]
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
pytest:
|
||||
runs-on: ubuntu-latest
|
||||
# Only run on GitHub Enterprise
|
||||
if: ${{ github.server_url == 'https://github.cds.internal.unity3d.com' }}
|
||||
runs-on: [self-hosted, Linux, X64]
|
||||
env:
|
||||
TEST_ENFORCE_BUFFER_KEY_TYPES: 1
|
||||
strategy:
|
||||
# If one test in the matrix fails we still want to run the others.
|
||||
fail-fast: false
|
||||
matrix:
|
||||
python-version: [3.6.x, 3.7.x, 3.8.x, 3.9.x]
|
||||
# NOTE: 3.9.10 was failing but is hard to test since it JUST came out. Delaying debugging for now.
|
||||
# TODO: Set back to 3.9.x once we can test with 3.9.10 via pyenv.
|
||||
python-version: [3.6.x, 3.7.x, 3.8.x, 3.9.9]
|
||||
include:
|
||||
- python-version: 3.6.x
|
||||
pip_constraints: test_constraints_min_version.txt
|
||||
|
|
@ -27,7 +34,7 @@ jobs:
|
|||
pip_constraints: test_constraints_mid_version.txt
|
||||
- python-version: 3.8.x
|
||||
pip_constraints: test_constraints_mid_version.txt
|
||||
- python-version: 3.9.x
|
||||
- python-version: 3.9.9
|
||||
pip_constraints: test_constraints_max_version.txt
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
|
@ -35,16 +42,18 @@ jobs:
|
|||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- name: Cache pip
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
# This path is specific to Ubuntu
|
||||
path: ~/.cache/pip
|
||||
# Look to see if there is a cache hit for the corresponding requirements file
|
||||
key: ${{ runner.os }}-pip-${{ hashFiles('ml-agents/setup.py', 'ml-agents-envs/setup.py', 'gym-unity/setup.py', 'test_requirements.txt', matrix.pip_constraints) }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-pip-
|
||||
${{ runner.os }}-
|
||||
# Caching not supported on GitHub Enterprise
|
||||
# See https://github.com/actions/cache/issues/505
|
||||
# - name: Cache pip
|
||||
# uses: actions/cache@v2
|
||||
# with:
|
||||
# # This path is specific to Ubuntu
|
||||
# path: ~/.cache/pip
|
||||
# # Look to see if there is a cache hit for the corresponding requirements file
|
||||
# key: ${{ runner.os }}-pip-${{ hashFiles('ml-agents/setup.py', 'ml-agents-envs/setup.py', 'gym-unity/setup.py', 'test_requirements.txt', matrix.pip_constraints) }}
|
||||
# restore-keys: |
|
||||
# ${{ runner.os }}-pip-
|
||||
# ${{ runner.os }}-
|
||||
- name: Display Python version
|
||||
run: python -c "import sys; print(sys.version)"
|
||||
- name: Install dependencies
|
||||
|
|
|
|||
|
|
@ -103,18 +103,17 @@
|
|||
"esac\n",
|
||||
"exit 0\n",
|
||||
" \"\"\")\n",
|
||||
" pro_bar.update(progress(5, 100))\n",
|
||||
" !apt-get install daemon >/dev/null 2>&1\n",
|
||||
" !sudo apt-get update\n",
|
||||
" pro_bar.update(progress(10, 100))\n",
|
||||
" !apt-get install wget >/dev/null 2>&1\n",
|
||||
" !sudo DEBIAN_FRONTEND=noninteractive apt install -y daemon wget gdebi-core build-essential libfontenc1 libfreetype6 xorg-dev xorg\n",
|
||||
" pro_bar.update(progress(20, 100))\n",
|
||||
" !wget http://security.ubuntu.com/ubuntu/pool/main/libx/libxfont/libxfont1_1.5.1-1ubuntu0.16.04.4_amd64.deb >/dev/null 2>&1\n",
|
||||
" !wget http://security.ubuntu.com/ubuntu/pool/main/libx/libxfont/libxfont1_1.5.1-1ubuntu0.16.04.4_amd64.deb 2>&1\n",
|
||||
" pro_bar.update(progress(30, 100))\n",
|
||||
" !wget --output-document xvfb.deb http://security.ubuntu.com/ubuntu/pool/universe/x/xorg-server/xvfb_1.18.4-0ubuntu0.12_amd64.deb >/dev/null 2>&1\n",
|
||||
" !wget --output-document xvfb.deb http://security.ubuntu.com/ubuntu/pool/universe/x/xorg-server/xvfb_1.18.4-0ubuntu0.12_amd64.deb 2>&1\n",
|
||||
" pro_bar.update(progress(40, 100))\n",
|
||||
" !dpkg -i libxfont1_1.5.1-1ubuntu0.16.04.4_amd64.deb >/dev/null 2>&1\n",
|
||||
" !sudo dpkg -i libxfont1_1.5.1-1ubuntu0.16.04.4_amd64.deb 2>&1\n",
|
||||
" pro_bar.update(progress(50, 100))\n",
|
||||
" !dpkg -i xvfb.deb >/dev/null 2>&1\n",
|
||||
" !sudo dpkg -i xvfb.deb 2>&1\n",
|
||||
" pro_bar.update(progress(70, 100))\n",
|
||||
" !rm libxfont1_1.5.1-1ubuntu0.16.04.4_amd64.deb\n",
|
||||
" pro_bar.update(progress(80, 100))\n",
|
||||
|
|
|
|||
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
|
|
@ -13,15 +13,6 @@
|
|||
"name": "python3",
|
||||
"language": "python",
|
||||
"display_name": "Python 3"
|
||||
},
|
||||
"pycharm": {
|
||||
"stem_cell": {
|
||||
"cell_type": "raw",
|
||||
"source": [],
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"cells": [
|
||||
|
|
@ -103,18 +94,17 @@
|
|||
"esac\n",
|
||||
"exit 0\n",
|
||||
" \"\"\")\n",
|
||||
" pro_bar.update(progress(5, 100))\n",
|
||||
" !apt-get install daemon >/dev/null 2>&1\n",
|
||||
" !sudo apt-get update\n",
|
||||
" pro_bar.update(progress(10, 100))\n",
|
||||
" !apt-get install wget >/dev/null 2>&1\n",
|
||||
" !sudo DEBIAN_FRONTEND=noninteractive apt install -y daemon wget gdebi-core build-essential libfontenc1 libfreetype6 xorg-dev xorg\n",
|
||||
" pro_bar.update(progress(20, 100))\n",
|
||||
" !wget http://security.ubuntu.com/ubuntu/pool/main/libx/libxfont/libxfont1_1.5.1-1ubuntu0.16.04.4_amd64.deb >/dev/null 2>&1\n",
|
||||
" !wget http://security.ubuntu.com/ubuntu/pool/main/libx/libxfont/libxfont1_1.5.1-1ubuntu0.16.04.4_amd64.deb 2>&1\n",
|
||||
" pro_bar.update(progress(30, 100))\n",
|
||||
" !wget --output-document xvfb.deb http://security.ubuntu.com/ubuntu/pool/universe/x/xorg-server/xvfb_1.18.4-0ubuntu0.12_amd64.deb >/dev/null 2>&1\n",
|
||||
" !wget --output-document xvfb.deb http://security.ubuntu.com/ubuntu/pool/universe/x/xorg-server/xvfb_1.18.4-0ubuntu0.12_amd64.deb 2>&1\n",
|
||||
" pro_bar.update(progress(40, 100))\n",
|
||||
" !dpkg -i libxfont1_1.5.1-1ubuntu0.16.04.4_amd64.deb >/dev/null 2>&1\n",
|
||||
" !sudo dpkg -i libxfont1_1.5.1-1ubuntu0.16.04.4_amd64.deb 2>&1\n",
|
||||
" pro_bar.update(progress(50, 100))\n",
|
||||
" !dpkg -i xvfb.deb >/dev/null 2>&1\n",
|
||||
" !sudo dpkg -i xvfb.deb 2>&1\n",
|
||||
" pro_bar.update(progress(70, 100))\n",
|
||||
" !rm libxfont1_1.5.1-1ubuntu0.16.04.4_amd64.deb\n",
|
||||
" pro_bar.update(progress(80, 100))\n",
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
# Test-only dependencies should go here, not in setup.py
|
||||
pytest>4.0.0,<6.0.0
|
||||
pytest>=6.2.5
|
||||
pytest-cov==2.6.1
|
||||
pytest-xdist==1.34.0
|
||||
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче