Initial checkin/move to the codeql-container registry
This commit is contained in:
Suraj Jacob
Родитель
8cca5f7eaf
Коммит
e79ce763e5
|
|
@ -3,6 +3,9 @@ __pycache__/
|
|||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
# VS
|
||||
/.vs/**
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,68 @@
|
|||
FROM ubuntu:20.04 AS codeql_base
|
||||
LABEL maintainer="Github codeql team"
|
||||
|
||||
# tzdata install needs to be non-interactive
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
# install/update basics and python
|
||||
RUN apt-get update && \
|
||||
apt-get upgrade -y && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
software-properties-common \
|
||||
vim \
|
||||
curl \
|
||||
wget \
|
||||
git \
|
||||
build-essential \
|
||||
unzip \
|
||||
apt-transport-https \
|
||||
python3.8 \
|
||||
python3-venv \
|
||||
python3-pip \
|
||||
python3-setuptools \
|
||||
python3-dev \
|
||||
gnupg \
|
||||
g++ \
|
||||
make \
|
||||
gcc \
|
||||
apt-utils \
|
||||
rsync \
|
||||
file \
|
||||
gettext && \
|
||||
apt-get clean
|
||||
|
||||
# Clone our setup and run scripts
|
||||
#RUN git clone https://github.com/microsoft/codeql-container /usr/local/startup_scripts
|
||||
RUN mkdir -p /usr/local/startup_scripts
|
||||
RUN ls -al /usr/local/startup_scripts
|
||||
COPY container /usr/local/startup_scripts/
|
||||
RUN pip3 install --upgrade pip \
|
||||
&& pip3 install -r /usr/local/startup_scripts/requirements.txt
|
||||
|
||||
# Install latest codeQL
|
||||
ENV CODEQL_HOME /usr/local/codeql-home
|
||||
# record the latest version of the codeql-cli
|
||||
RUN python3 /usr/local/startup_scripts/get-latest-codeql-version.py > /tmp/codeql_version
|
||||
RUN mkdir -p ${CODEQL_HOME} \
|
||||
${CODEQL_HOME}/codeql-cli \
|
||||
${CODEQL_HOME}/codeql-repo \
|
||||
${CODEQL_HOME}/codeql-go-repo \
|
||||
/opt/codeql
|
||||
|
||||
RUN CODEQL_VERSION=$(cat /tmp/codeql_version) && \
|
||||
wget -q https://github.com/github/codeql-cli-binaries/releases/download/${CODEQL_VERSION}/codeql-linux64.zip -O /tmp/codeql_linux.zip && \
|
||||
unzip /tmp/codeql_linux.zip -d ${CODEQL_HOME}/codeql-cli && \
|
||||
rm /tmp/codeql_linux.zip
|
||||
|
||||
# get the latest codeql queries and record the HEAD
|
||||
RUN git clone https://github.com/github/codeql ${CODEQL_HOME}/codeql-repo && \
|
||||
git --git-dir ${CODEQL_HOME}/codeql-repo/.git log --pretty=reference -1 > /opt/codeql/codeql-repo-last-commit
|
||||
RUN git clone https://github.com/github/codeql-go ${CODEQL_HOME}/codeql-go-repo && \
|
||||
git --git-dir ${CODEQL_HOME}/codeql-go-repo/.git log --pretty=reference -1 > /opt/codeql/codeql-go-repo-last-commit
|
||||
|
||||
ENV PATH="${CODEQL_HOME}/codeql-cli/codeql:${PATH}"
|
||||
|
||||
# Pre-compile our queries to save time later
|
||||
#RUN codeql query compile --threads=0 ${CODEQL_HOME}/codelq-repo/*/ql/src/codeql-suites/*-.qls
|
||||
#RUN codeql query compile --threads=0 ${CODEQL_HOME}/codelq-go-repo/ql/src/codeql-suites/*-.qls
|
||||
#ENTRYPOINT ["python3", "/usr/local/startup_scripts/setup.py"]
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
#!/usr/bin/python3
|
||||
# get the parent directory of the script, to link libs
|
||||
|
||||
import os
|
||||
import sys
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
|
||||
|
||||
from libs.github import get_latest_github_repo_version
|
||||
|
||||
def main():
|
||||
latest_release = get_latest_github_repo_version("github/codeql-cli-binaries")
|
||||
print(latest_release.title)
|
||||
|
||||
main()
|
||||
|
|
@ -0,0 +1,86 @@
|
|||
from sys import exit
|
||||
from re import search
|
||||
from zipfile import ZipFile
|
||||
from libs.utils import *
|
||||
from logging import getLogger
|
||||
from libs.github import get_latest_github_repo_version
|
||||
|
||||
logger = getLogger('codeql-container')
|
||||
|
||||
class CodeQL:
|
||||
"""
|
||||
Codeql related logic, downloading, installing etc
|
||||
"""
|
||||
CODEQL_HOME = None
|
||||
|
||||
CODEQL_GITHUB_URL = 'https://github.com/github/codeql-cli-binaries'
|
||||
CODEQL_QUERIES_URL = 'https://github.com/github/codeql'
|
||||
CODEQL_GO_QUERIES_URL = 'https://github.com/github/codeql-go'
|
||||
|
||||
TEMP_DIR ='/tmp'
|
||||
|
||||
# Error codes
|
||||
ERROR_EXECUTING_COMMAND = 1
|
||||
ERROR_EXECUTING_CODEQL = 2
|
||||
ERROR_UNKNOWN_OS = 3
|
||||
ERROR_GIT_COMMAND = 4
|
||||
|
||||
def __init__(self, codeql_base_dir):
|
||||
self.CODEQL_HOME = codeql_base_dir
|
||||
|
||||
def download_and_install_latest_codeql(self, github_version):
|
||||
"""
|
||||
Download and install the latest codeql-cli from the github repo releases
|
||||
"""
|
||||
download_url = None
|
||||
download_path = None
|
||||
if os_name == 'posix':
|
||||
download_url = f'https://github.com/github/codeql-cli-binaries/releases/download/{github_version.title}/codeql-linux64.zip'
|
||||
download_path = f'{self.TEMP_DIR}/codeql_linux.zip'
|
||||
elif os_name == 'nt':
|
||||
download_url = f'https://github.com/github/codeql-cli-binaries/releases/download/{github_version.title}/codeql-win64.zip'
|
||||
download_path = f'{self.TEMP_DIR}/codeql_windows.zip'
|
||||
else:
|
||||
exit(ERROR_UNKNOWN_OS)
|
||||
|
||||
logger.info(f'Downloading codeql-cli version {github_version.title}...')
|
||||
check_output_wrapper(f"wget -q {download_url} -O {download_path}", shell=True).decode("utf-8")
|
||||
self.install_codeql_cli(download_path)
|
||||
#rm /tmp/codeql_linux.zip
|
||||
|
||||
def download_and_install_latest_codeql_queries(self):
|
||||
"""
|
||||
Download and install the latest codeql queries from the github repo
|
||||
"""
|
||||
logger.info("Downloading codeql queries...")
|
||||
codeql_repo_dir = f'{self.CODEQL_HOME}/codeql-repo'
|
||||
wipe_and_create_dir(codeql_repo_dir)
|
||||
ret1 = check_output_wrapper(f'git clone {self.CODEQL_QUERIES_URL} {codeql_repo_dir}', shell=True)
|
||||
|
||||
codeql_go_repo_dir = f'{self.CODEQL_HOME}/codeql-go-repo'
|
||||
wipe_and_create_dir(codeql_go_repo_dir)
|
||||
ret2 = check_output_wrapper(f'git clone {self.CODEQL_GO_QUERIES_URL} {codeql_go_repo_dir}', shell=True)
|
||||
if ret1 is CalledProcessError or ret2 is CalledProcessError:
|
||||
logger.error("Could not run git command")
|
||||
exit(ERROR_GIT_COMMAND)
|
||||
|
||||
def get_current_local_version(self):
|
||||
ret_string = check_output_wrapper(f'{self.CODEQL_HOME}/codeql-cli/codeql/codeql version', shell=True).decode("utf-8")
|
||||
if ret_string is CalledProcessError:
|
||||
logger.error("Could not run codeql command")
|
||||
exit(ERROR_EXECUTING_CODEQL)
|
||||
version_match = search("Version: ([0-9.]+)\.", ret_string)
|
||||
if not version_match:
|
||||
logger.error("Could not determine existing codeql version")
|
||||
exit(ERROR_EXECUTING_CODEQL)
|
||||
version = f'v{version_match.group(1)}'
|
||||
return version
|
||||
|
||||
def get_latest_codeql_github_version(self):
|
||||
return get_latest_github_repo_version("github/codeql-cli-binaries")
|
||||
|
||||
def install_codeql_cli(self, download_path):
|
||||
logger.info("Installing codeql-cli...")
|
||||
codeql_dir = f'{self.CODEQL_HOME}/codeql-cli'
|
||||
wipe_and_create_dir(codeql_dir)
|
||||
ret1 = check_output_wrapper(f'unzip {download_path} -d {codeql_dir}', shell=True)
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
from datetime import datetime, MINYEAR
|
||||
from github import Github, GitRelease, Repository, GithubException
|
||||
|
||||
def get_latest_github_repo_version(repo):
|
||||
client = Github()
|
||||
repo = client.get_repo(repo)
|
||||
releases = repo.get_releases()
|
||||
latest_release = get_latest_github_release(releases)
|
||||
return latest_release
|
||||
|
||||
def get_latest_github_release(releases):
|
||||
latest_release = None
|
||||
latest_date = datetime(MINYEAR, 1, 1)
|
||||
for release in releases:
|
||||
if release.created_at > latest_date:
|
||||
latest_date = release.created_at
|
||||
latest_release = release
|
||||
return latest_release
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
from sys import exit
|
||||
from shutil import rmtree
|
||||
from os import environ, mkdir, name as os_name
|
||||
from subprocess import check_output, CalledProcessError
|
||||
from logging import getLogger
|
||||
|
||||
logger = getLogger('codeql-container')
|
||||
|
||||
# get secrets from the env
|
||||
def get_env_variable(self, var_name, optional=False):
|
||||
"""
|
||||
Retrieve an environment variable. Any failures will cause an exception
|
||||
to be thrown.
|
||||
"""
|
||||
try:
|
||||
return environ[var_name]
|
||||
except KeyError:
|
||||
if optional:
|
||||
return False
|
||||
else:
|
||||
error_msg = f'Error: You must set the {var_name} environment variable.'
|
||||
raise Exception(error_msg)
|
||||
|
||||
def check_output_wrapper(*args, **kwargs):
|
||||
"""
|
||||
Thin wrapper around subprocess
|
||||
"""
|
||||
|
||||
logger.debug('Executing %s, %s', args, kwargs)
|
||||
try:
|
||||
return check_output(*args, **kwargs)
|
||||
except CalledProcessError as msg:
|
||||
logger.warning('Error %s,%s,%s from command.', msg.returncode, msg.output, msg.stderr)
|
||||
logger.debug('Output: %s', msg.output)
|
||||
sys.exit(ERROR_EXECUTING_COMMAND);
|
||||
|
||||
def wipe_and_create_dir(dirname):
|
||||
rmtree(dirname)
|
||||
mkdir(dirname)
|
||||
|
|
@ -0,0 +1 @@
|
|||
PyGithub==1.43.7
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
#!/usr/bin/python3
|
||||
|
||||
import os
|
||||
import sys
|
||||
from logging import Logger, getLogger, INFO
|
||||
from sys import path as syspath
|
||||
from libs.utils import *
|
||||
from libs.github import *
|
||||
from libs.codeql import *
|
||||
|
||||
# get the parent directory of the script, to link libs
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
|
||||
|
||||
CODEQL_HOME = environ['CODEQL_HOME']
|
||||
|
||||
logger = getLogger('codeql-container')
|
||||
logger.setLevel(INFO)
|
||||
|
||||
def setup():
|
||||
"""
|
||||
Download and install the latest codeql cli
|
||||
Download and install the latest codeql queries
|
||||
"""
|
||||
|
||||
# check version and download the latest version
|
||||
get_latest_codeql()
|
||||
# install vscode?
|
||||
# clone codeql libs
|
||||
# setup vscode + codeql
|
||||
# wait for user
|
||||
|
||||
|
||||
def get_latest_codeql():
|
||||
# what version do we have?
|
||||
codeql = CodeQL(CODEQL_HOME)
|
||||
current_installed_version = codeql.get_current_local_version()
|
||||
logger.info(f'Current codeql version: {current_installed_version}')
|
||||
latest_online_version = codeql.get_latest_codeql_github_version()
|
||||
if current_installed_version != latest_online_version.title:
|
||||
# we got a newer version online, download and install it
|
||||
codeql.download_and_install_latest_codeql(latest_online_version)
|
||||
# get the latest queries regardless (TODO: Optimize by storing and checking the last commit hash?)
|
||||
codeql.download_and_install_latest_codeql_queries()
|
||||
|
||||
setup()
|
||||
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
import os
|
||||
|
||||
# should we update the local copy of codeql-cli if a new version is available?
|
||||
CHECK_LATEST_CODEQL_CLI = False
|
||||
|
||||
# should we update the local copy of codeql queries if a new version is available?
|
||||
CHECK_LATEST_QUERIES = False
|
||||
|
||||
|
||||
# if we are downloading new queries, should we precompile them
|
||||
#(makes query execution faster, but building the container build slower).
|
||||
PRECOMPILE_QUERIES = False
|
||||
|
||||
def main():
|
||||
# get all the command-line args/envs required
|
||||
# check if the latest codeql cli need to be downloaded
|
||||
# check if the latest codeql queries need to be downloaded
|
||||
# check if we need to precompile the new queries
|
||||
Загрузка…
Ссылка в новой задаче