Initial checkin/move to the codeql-container registry
This commit is contained in:
Suraj Jacob
Родитель
8cca5f7eaf
Коммит
e79ce763e5
|
|
@ -3,6 +3,9 @@ __pycache__/
|
||||||
*.py[cod]
|
*.py[cod]
|
||||||
*$py.class
|
*$py.class
|
||||||
|
|
||||||
|
# VS
|
||||||
|
/.vs/**
|
||||||
|
|
||||||
# C extensions
|
# C extensions
|
||||||
*.so
|
*.so
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,68 @@
|
||||||
|
FROM ubuntu:20.04 AS codeql_base
|
||||||
|
LABEL maintainer="Github codeql team"
|
||||||
|
|
||||||
|
# tzdata install needs to be non-interactive
|
||||||
|
ENV DEBIAN_FRONTEND=noninteractive
|
||||||
|
|
||||||
|
# install/update basics and python
|
||||||
|
RUN apt-get update && \
|
||||||
|
apt-get upgrade -y && \
|
||||||
|
apt-get install -y --no-install-recommends \
|
||||||
|
software-properties-common \
|
||||||
|
vim \
|
||||||
|
curl \
|
||||||
|
wget \
|
||||||
|
git \
|
||||||
|
build-essential \
|
||||||
|
unzip \
|
||||||
|
apt-transport-https \
|
||||||
|
python3.8 \
|
||||||
|
python3-venv \
|
||||||
|
python3-pip \
|
||||||
|
python3-setuptools \
|
||||||
|
python3-dev \
|
||||||
|
gnupg \
|
||||||
|
g++ \
|
||||||
|
make \
|
||||||
|
gcc \
|
||||||
|
apt-utils \
|
||||||
|
rsync \
|
||||||
|
file \
|
||||||
|
gettext && \
|
||||||
|
apt-get clean
|
||||||
|
|
||||||
|
# Clone our setup and run scripts
|
||||||
|
#RUN git clone https://github.com/microsoft/codeql-container /usr/local/startup_scripts
|
||||||
|
RUN mkdir -p /usr/local/startup_scripts
|
||||||
|
RUN ls -al /usr/local/startup_scripts
|
||||||
|
COPY container /usr/local/startup_scripts/
|
||||||
|
RUN pip3 install --upgrade pip \
|
||||||
|
&& pip3 install -r /usr/local/startup_scripts/requirements.txt
|
||||||
|
|
||||||
|
# Install latest codeQL
|
||||||
|
ENV CODEQL_HOME /usr/local/codeql-home
|
||||||
|
# record the latest version of the codeql-cli
|
||||||
|
RUN python3 /usr/local/startup_scripts/get-latest-codeql-version.py > /tmp/codeql_version
|
||||||
|
RUN mkdir -p ${CODEQL_HOME} \
|
||||||
|
${CODEQL_HOME}/codeql-cli \
|
||||||
|
${CODEQL_HOME}/codeql-repo \
|
||||||
|
${CODEQL_HOME}/codeql-go-repo \
|
||||||
|
/opt/codeql
|
||||||
|
|
||||||
|
RUN CODEQL_VERSION=$(cat /tmp/codeql_version) && \
|
||||||
|
wget -q https://github.com/github/codeql-cli-binaries/releases/download/${CODEQL_VERSION}/codeql-linux64.zip -O /tmp/codeql_linux.zip && \
|
||||||
|
unzip /tmp/codeql_linux.zip -d ${CODEQL_HOME}/codeql-cli && \
|
||||||
|
rm /tmp/codeql_linux.zip
|
||||||
|
|
||||||
|
# get the latest codeql queries and record the HEAD
|
||||||
|
RUN git clone https://github.com/github/codeql ${CODEQL_HOME}/codeql-repo && \
|
||||||
|
git --git-dir ${CODEQL_HOME}/codeql-repo/.git log --pretty=reference -1 > /opt/codeql/codeql-repo-last-commit
|
||||||
|
RUN git clone https://github.com/github/codeql-go ${CODEQL_HOME}/codeql-go-repo && \
|
||||||
|
git --git-dir ${CODEQL_HOME}/codeql-go-repo/.git log --pretty=reference -1 > /opt/codeql/codeql-go-repo-last-commit
|
||||||
|
|
||||||
|
ENV PATH="${CODEQL_HOME}/codeql-cli/codeql:${PATH}"
|
||||||
|
|
||||||
|
# Pre-compile our queries to save time later
|
||||||
|
#RUN codeql query compile --threads=0 ${CODEQL_HOME}/codelq-repo/*/ql/src/codeql-suites/*-.qls
|
||||||
|
#RUN codeql query compile --threads=0 ${CODEQL_HOME}/codelq-go-repo/ql/src/codeql-suites/*-.qls
|
||||||
|
#ENTRYPOINT ["python3", "/usr/local/startup_scripts/setup.py"]
|
||||||
|
|
@ -0,0 +1,14 @@
|
||||||
|
#!/usr/bin/python3
|
||||||
|
# get the parent directory of the script, to link libs
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
sys.path.append(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
|
||||||
|
|
||||||
|
from libs.github import get_latest_github_repo_version
|
||||||
|
|
||||||
|
def main():
|
||||||
|
latest_release = get_latest_github_repo_version("github/codeql-cli-binaries")
|
||||||
|
print(latest_release.title)
|
||||||
|
|
||||||
|
main()
|
||||||
|
|
@ -0,0 +1,86 @@
|
||||||
|
from sys import exit
|
||||||
|
from re import search
|
||||||
|
from zipfile import ZipFile
|
||||||
|
from libs.utils import *
|
||||||
|
from logging import getLogger
|
||||||
|
from libs.github import get_latest_github_repo_version
|
||||||
|
|
||||||
|
logger = getLogger('codeql-container')
|
||||||
|
|
||||||
|
class CodeQL:
|
||||||
|
"""
|
||||||
|
Codeql related logic, downloading, installing etc
|
||||||
|
"""
|
||||||
|
CODEQL_HOME = None
|
||||||
|
|
||||||
|
CODEQL_GITHUB_URL = 'https://github.com/github/codeql-cli-binaries'
|
||||||
|
CODEQL_QUERIES_URL = 'https://github.com/github/codeql'
|
||||||
|
CODEQL_GO_QUERIES_URL = 'https://github.com/github/codeql-go'
|
||||||
|
|
||||||
|
TEMP_DIR ='/tmp'
|
||||||
|
|
||||||
|
# Error codes
|
||||||
|
ERROR_EXECUTING_COMMAND = 1
|
||||||
|
ERROR_EXECUTING_CODEQL = 2
|
||||||
|
ERROR_UNKNOWN_OS = 3
|
||||||
|
ERROR_GIT_COMMAND = 4
|
||||||
|
|
||||||
|
def __init__(self, codeql_base_dir):
|
||||||
|
self.CODEQL_HOME = codeql_base_dir
|
||||||
|
|
||||||
|
def download_and_install_latest_codeql(self, github_version):
|
||||||
|
"""
|
||||||
|
Download and install the latest codeql-cli from the github repo releases
|
||||||
|
"""
|
||||||
|
download_url = None
|
||||||
|
download_path = None
|
||||||
|
if os_name == 'posix':
|
||||||
|
download_url = f'https://github.com/github/codeql-cli-binaries/releases/download/{github_version.title}/codeql-linux64.zip'
|
||||||
|
download_path = f'{self.TEMP_DIR}/codeql_linux.zip'
|
||||||
|
elif os_name == 'nt':
|
||||||
|
download_url = f'https://github.com/github/codeql-cli-binaries/releases/download/{github_version.title}/codeql-win64.zip'
|
||||||
|
download_path = f'{self.TEMP_DIR}/codeql_windows.zip'
|
||||||
|
else:
|
||||||
|
exit(ERROR_UNKNOWN_OS)
|
||||||
|
|
||||||
|
logger.info(f'Downloading codeql-cli version {github_version.title}...')
|
||||||
|
check_output_wrapper(f"wget -q {download_url} -O {download_path}", shell=True).decode("utf-8")
|
||||||
|
self.install_codeql_cli(download_path)
|
||||||
|
#rm /tmp/codeql_linux.zip
|
||||||
|
|
||||||
|
def download_and_install_latest_codeql_queries(self):
|
||||||
|
"""
|
||||||
|
Download and install the latest codeql queries from the github repo
|
||||||
|
"""
|
||||||
|
logger.info("Downloading codeql queries...")
|
||||||
|
codeql_repo_dir = f'{self.CODEQL_HOME}/codeql-repo'
|
||||||
|
wipe_and_create_dir(codeql_repo_dir)
|
||||||
|
ret1 = check_output_wrapper(f'git clone {self.CODEQL_QUERIES_URL} {codeql_repo_dir}', shell=True)
|
||||||
|
|
||||||
|
codeql_go_repo_dir = f'{self.CODEQL_HOME}/codeql-go-repo'
|
||||||
|
wipe_and_create_dir(codeql_go_repo_dir)
|
||||||
|
ret2 = check_output_wrapper(f'git clone {self.CODEQL_GO_QUERIES_URL} {codeql_go_repo_dir}', shell=True)
|
||||||
|
if ret1 is CalledProcessError or ret2 is CalledProcessError:
|
||||||
|
logger.error("Could not run git command")
|
||||||
|
exit(ERROR_GIT_COMMAND)
|
||||||
|
|
||||||
|
def get_current_local_version(self):
|
||||||
|
ret_string = check_output_wrapper(f'{self.CODEQL_HOME}/codeql-cli/codeql/codeql version', shell=True).decode("utf-8")
|
||||||
|
if ret_string is CalledProcessError:
|
||||||
|
logger.error("Could not run codeql command")
|
||||||
|
exit(ERROR_EXECUTING_CODEQL)
|
||||||
|
version_match = search("Version: ([0-9.]+)\.", ret_string)
|
||||||
|
if not version_match:
|
||||||
|
logger.error("Could not determine existing codeql version")
|
||||||
|
exit(ERROR_EXECUTING_CODEQL)
|
||||||
|
version = f'v{version_match.group(1)}'
|
||||||
|
return version
|
||||||
|
|
||||||
|
def get_latest_codeql_github_version(self):
|
||||||
|
return get_latest_github_repo_version("github/codeql-cli-binaries")
|
||||||
|
|
||||||
|
def install_codeql_cli(self, download_path):
|
||||||
|
logger.info("Installing codeql-cli...")
|
||||||
|
codeql_dir = f'{self.CODEQL_HOME}/codeql-cli'
|
||||||
|
wipe_and_create_dir(codeql_dir)
|
||||||
|
ret1 = check_output_wrapper(f'unzip {download_path} -d {codeql_dir}', shell=True)
|
||||||
|
|
@ -0,0 +1,18 @@
|
||||||
|
from datetime import datetime, MINYEAR
|
||||||
|
from github import Github, GitRelease, Repository, GithubException
|
||||||
|
|
||||||
|
def get_latest_github_repo_version(repo):
|
||||||
|
client = Github()
|
||||||
|
repo = client.get_repo(repo)
|
||||||
|
releases = repo.get_releases()
|
||||||
|
latest_release = get_latest_github_release(releases)
|
||||||
|
return latest_release
|
||||||
|
|
||||||
|
def get_latest_github_release(releases):
|
||||||
|
latest_release = None
|
||||||
|
latest_date = datetime(MINYEAR, 1, 1)
|
||||||
|
for release in releases:
|
||||||
|
if release.created_at > latest_date:
|
||||||
|
latest_date = release.created_at
|
||||||
|
latest_release = release
|
||||||
|
return latest_release
|
||||||
|
|
@ -0,0 +1,39 @@
|
||||||
|
from sys import exit
|
||||||
|
from shutil import rmtree
|
||||||
|
from os import environ, mkdir, name as os_name
|
||||||
|
from subprocess import check_output, CalledProcessError
|
||||||
|
from logging import getLogger
|
||||||
|
|
||||||
|
logger = getLogger('codeql-container')
|
||||||
|
|
||||||
|
# get secrets from the env
|
||||||
|
def get_env_variable(self, var_name, optional=False):
|
||||||
|
"""
|
||||||
|
Retrieve an environment variable. Any failures will cause an exception
|
||||||
|
to be thrown.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
return environ[var_name]
|
||||||
|
except KeyError:
|
||||||
|
if optional:
|
||||||
|
return False
|
||||||
|
else:
|
||||||
|
error_msg = f'Error: You must set the {var_name} environment variable.'
|
||||||
|
raise Exception(error_msg)
|
||||||
|
|
||||||
|
def check_output_wrapper(*args, **kwargs):
|
||||||
|
"""
|
||||||
|
Thin wrapper around subprocess
|
||||||
|
"""
|
||||||
|
|
||||||
|
logger.debug('Executing %s, %s', args, kwargs)
|
||||||
|
try:
|
||||||
|
return check_output(*args, **kwargs)
|
||||||
|
except CalledProcessError as msg:
|
||||||
|
logger.warning('Error %s,%s,%s from command.', msg.returncode, msg.output, msg.stderr)
|
||||||
|
logger.debug('Output: %s', msg.output)
|
||||||
|
sys.exit(ERROR_EXECUTING_COMMAND);
|
||||||
|
|
||||||
|
def wipe_and_create_dir(dirname):
|
||||||
|
rmtree(dirname)
|
||||||
|
mkdir(dirname)
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
PyGithub==1.43.7
|
||||||
|
|
@ -0,0 +1,46 @@
|
||||||
|
#!/usr/bin/python3
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
from logging import Logger, getLogger, INFO
|
||||||
|
from sys import path as syspath
|
||||||
|
from libs.utils import *
|
||||||
|
from libs.github import *
|
||||||
|
from libs.codeql import *
|
||||||
|
|
||||||
|
# get the parent directory of the script, to link libs
|
||||||
|
sys.path.append(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
|
||||||
|
|
||||||
|
CODEQL_HOME = environ['CODEQL_HOME']
|
||||||
|
|
||||||
|
logger = getLogger('codeql-container')
|
||||||
|
logger.setLevel(INFO)
|
||||||
|
|
||||||
|
def setup():
|
||||||
|
"""
|
||||||
|
Download and install the latest codeql cli
|
||||||
|
Download and install the latest codeql queries
|
||||||
|
"""
|
||||||
|
|
||||||
|
# check version and download the latest version
|
||||||
|
get_latest_codeql()
|
||||||
|
# install vscode?
|
||||||
|
# clone codeql libs
|
||||||
|
# setup vscode + codeql
|
||||||
|
# wait for user
|
||||||
|
|
||||||
|
|
||||||
|
def get_latest_codeql():
|
||||||
|
# what version do we have?
|
||||||
|
codeql = CodeQL(CODEQL_HOME)
|
||||||
|
current_installed_version = codeql.get_current_local_version()
|
||||||
|
logger.info(f'Current codeql version: {current_installed_version}')
|
||||||
|
latest_online_version = codeql.get_latest_codeql_github_version()
|
||||||
|
if current_installed_version != latest_online_version.title:
|
||||||
|
# we got a newer version online, download and install it
|
||||||
|
codeql.download_and_install_latest_codeql(latest_online_version)
|
||||||
|
# get the latest queries regardless (TODO: Optimize by storing and checking the last commit hash?)
|
||||||
|
codeql.download_and_install_latest_codeql_queries()
|
||||||
|
|
||||||
|
setup()
|
||||||
|
|
||||||
|
|
@ -0,0 +1,18 @@
|
||||||
|
import os
|
||||||
|
|
||||||
|
# should we update the local copy of codeql-cli if a new version is available?
|
||||||
|
CHECK_LATEST_CODEQL_CLI = False
|
||||||
|
|
||||||
|
# should we update the local copy of codeql queries if a new version is available?
|
||||||
|
CHECK_LATEST_QUERIES = False
|
||||||
|
|
||||||
|
|
||||||
|
# if we are downloading new queries, should we precompile them
|
||||||
|
#(makes query execution faster, but building the container build slower).
|
||||||
|
PRECOMPILE_QUERIES = False
|
||||||
|
|
||||||
|
def main():
|
||||||
|
# get all the command-line args/envs required
|
||||||
|
# check if the latest codeql cli need to be downloaded
|
||||||
|
# check if the latest codeql queries need to be downloaded
|
||||||
|
# check if we need to precompile the new queries
|
||||||
Загрузка…
Ссылка в новой задаче