зеркало из https://github.com/mozilla/MozDef.git
Merge pull request #1214 from gene1wood/change-cicd-sequence
Change CI/CD sequence for v1.38.5
This commit is contained in:
Brandon Myers
GitHub
Коммит
33a770b9c1
16
CHANGELOG
16
CHANGELOG
|
|
@ -5,6 +5,19 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
|
|||
|
||||
## [Unreleased]
|
||||
|
||||
## [v1.38.5] - 2019-04-09
|
||||
### Added
|
||||
- Support for CSS themes
|
||||
|
||||
### Changed
|
||||
- The CI/CD order to now build docker images in CodeBuild, upload them
|
||||
to DockerHub and then pull them down in the packer instance. Updated docs.
|
||||
- Assert TravisCI Python version in advance of change of Travis default to 3.6
|
||||
|
||||
### Fixed
|
||||
- Dashboard error on docker spinup
|
||||
|
||||
|
||||
## [v1.38.4] - 2019-04-08
|
||||
### Fixed
|
||||
- Docker image tagging for git version tag builds
|
||||
|
|
@ -64,7 +77,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
|
|||
- Added checks on sending SQS messages to only accept intra-account messages
|
||||
- Improved docker performance and disk space requirements
|
||||
|
||||
[Unreleased]: https://github.com/mozilla/MozDef/compare/v1.38.4...HEAD
|
||||
[Unreleased]: https://github.com/mozilla/MozDef/compare/v1.38.5...HEAD
|
||||
[v1.38.5]: https://github.com/mozilla/MozDef/compare/v1.38.4...v1.38.5
|
||||
[v1.38.4]: https://github.com/mozilla/MozDef/compare/v1.38.3...v1.38.4
|
||||
[v1.38.3]: https://github.com/mozilla/MozDef/compare/v1.38.2...v1.38.3
|
||||
[v1.38.2]: https://github.com/mozilla/MozDef/compare/v1.38.1...v1.38.2
|
||||
|
|
|
|||
4
Makefile
4
Makefile
|
|
@ -143,7 +143,3 @@ rebuild: clean build-from-cwd
|
|||
.PHONY: new-alert
|
||||
new-alert: ## Create an example alert and working alert unit test
|
||||
python tests/alert_templater.py
|
||||
|
||||
.PHONY: set-version-and-fetch-docker-container
|
||||
set-version-and-fetch-docker-container: build-from-cwd tag-images # Lock the release of MozDef by pulling the docker containers on AMI build and caching replace all instances of latest in the compose override with the BRANCH
|
||||
sed -i s/latest/$(BRANCH)/g docker/compose/docker-compose-cloudy-mozdef.yml
|
||||
|
|
|
|||
|
|
@ -15,20 +15,20 @@ echo "It's dangerous to go alone. Take one of these: <%%%%|==========>"
|
|||
# Then again we probably do not need to run the test suite here because it has been run three times to get the code here.
|
||||
# echo "Tests complete.
|
||||
|
||||
echo "Processing webhook event for ${CODEBUILD_WEBHOOK_TRIGGER}."
|
||||
echo "Processing webhook event for '${CODEBUILD_WEBHOOK_TRIGGER}'."
|
||||
|
||||
if [[ "branch/master" == "$CODEBUILD_WEBHOOK_TRIGGER" \
|
||||
|| "$CODEBUILD_WEBHOOK_TRIGGER" =~ ^tag\/v[0-9]+\.[0-9]+\.[0-9]+(\-(prod|pre|testing))?$ ]]; then
|
||||
echo "Building a release"
|
||||
echo "C|_| This may take a bit. Might as well grab a coffee."
|
||||
make build-from-cwd
|
||||
cd cloudy_mozdef
|
||||
BRANCH="`echo $CODEBUILD_WEBHOOK_TRIGGER | cut -d '/' -f2`"
|
||||
make build-from-cwd
|
||||
make hub-login
|
||||
make BRANCH=${BRANCH} docker-push-tagged
|
||||
cd cloudy_mozdef
|
||||
make BRANCH=${BRANCH} packer-build-github
|
||||
make BRANCH=${BRANCH} publish-versioned-templates
|
||||
cd ..
|
||||
make hub-login
|
||||
make BRANCH=${BRANCH} docker-push-tagged
|
||||
fi
|
||||
|
||||
echo "End build of the MozDef codebase."
|
||||
|
|
|
|||
|
|
@ -1,72 +1,69 @@
|
|||
{
|
||||
"variables": {
|
||||
"aws_access_key": "{{env `AWS_ACCESS_KEY_ID`}}",
|
||||
"aws_secret_key": "{{env `AWS_SECRET_ACCESS_KEY`}}",
|
||||
"aws_security_token": "{{env `AWS_SESSION_TOKEN`}}"
|
||||
"aws_access_key": "{{env `AWS_ACCESS_KEY_ID`}}",
|
||||
"aws_secret_key": "{{env `AWS_SECRET_ACCESS_KEY`}}",
|
||||
"aws_security_token": "{{env `AWS_SESSION_TOKEN`}}"
|
||||
},
|
||||
"builders": [{
|
||||
"type": "amazon-ebs",
|
||||
"region": "us-west-2",
|
||||
"access_key": "{{user `aws_access_key`}}",
|
||||
"secret_key": "{{user `aws_secret_key`}}",
|
||||
"token": "{{user `aws_security_token`}}",
|
||||
"source_ami": "ami-0d1000aff9a9bad89",
|
||||
"instance_type": "t2.large",
|
||||
"ssh_pty" : "true",
|
||||
"ssh_username": "ec2-user",
|
||||
"ami_name": "mozdef_{{timestamp}}",
|
||||
"launch_block_device_mappings": [
|
||||
{
|
||||
"delete_on_termination": true,
|
||||
"device_name": "/dev/xvda",
|
||||
"volume_size": 14
|
||||
"builders": [
|
||||
{
|
||||
"type": "amazon-ebs",
|
||||
"region": "us-west-2",
|
||||
"access_key": "{{user `aws_access_key`}}",
|
||||
"secret_key": "{{user `aws_secret_key`}}",
|
||||
"token": "{{user `aws_security_token`}}",
|
||||
"source_ami": "ami-0d1000aff9a9bad89",
|
||||
"instance_type": "t2.large",
|
||||
"ssh_pty": "true",
|
||||
"ssh_username": "ec2-user",
|
||||
"ami_name": "mozdef_{{timestamp}}",
|
||||
"launch_block_device_mappings": [
|
||||
{
|
||||
"delete_on_termination": true,
|
||||
"device_name": "/dev/xvda",
|
||||
"volume_size": 14
|
||||
}
|
||||
],
|
||||
"ami_description": "An automated build of MozDef triggered via the makefile.",
|
||||
"ami_groups": [
|
||||
"all"
|
||||
],
|
||||
"run_tags": {
|
||||
"app": "packer-builder-mozdef"
|
||||
},
|
||||
"run_volume_tags": {
|
||||
"app": "packer-builder-mozdef"
|
||||
},
|
||||
"snapshot_tags": {
|
||||
"app": "packer-builder-mozdef"
|
||||
},
|
||||
"tags": {
|
||||
"github:Branch": "{{ user `github_branch`}}",
|
||||
"buildTimestamp": "{{timestamp}}",
|
||||
"app": "mozdef"
|
||||
}
|
||||
],
|
||||
"ami_description": "An automated build of MozDef triggered via the makefile.",
|
||||
"ami_groups": [
|
||||
"all"
|
||||
],
|
||||
"run_tags": {
|
||||
"app": "packer-builder-mozdef"
|
||||
},
|
||||
"run_volume_tags": {
|
||||
"app": "packer-builder-mozdef"
|
||||
},
|
||||
"snapshot_tags": {
|
||||
"app": "packer-builder-mozdef"
|
||||
},
|
||||
"tags": {
|
||||
"github:Branch": "{{ user `github_branch`}}",
|
||||
"buildTimestamp": "{{timestamp}}",
|
||||
"app": "mozdef"
|
||||
}
|
||||
}],
|
||||
"provisioners": [
|
||||
{ "type": "shell",
|
||||
],
|
||||
"provisioners": [
|
||||
{
|
||||
"type": "shell",
|
||||
"inline": [
|
||||
"set -e",
|
||||
"sudo yum update -y",
|
||||
"sudo yum makecache fast",
|
||||
"sudo yum install -y glibc-devel gcc libstdc++ libffi-devel zlib-devel make ",
|
||||
"sudo yum install -y mysql-devel python python-devel python-pip",
|
||||
"sudo yum install -y git",
|
||||
"sudo yum install -y docker",
|
||||
"sudo yum install -y python3",
|
||||
"sudo pip install virtualenv ",
|
||||
"sudo pip install docker-compose",
|
||||
"sudo yum install -y glibc-devel gcc libstdc++ libffi-devel zlib-devel make mysql-devel python python-devel python-pip git docker python3",
|
||||
"sudo pip install virtualenv docker-compose",
|
||||
"sudo systemctl enable docker",
|
||||
"sudo systemctl start docker",
|
||||
"sudo mkdir -p /opt/mozdef/",
|
||||
"sudo mkdir --verbose --parents /opt/mozdef/",
|
||||
"sudo git clone https://github.com/mozilla/MozDef /opt/mozdef",
|
||||
"cd /opt/mozdef && sudo git checkout {{ user `github_branch`}}",
|
||||
"cd /opt/mozdef && sudo git rev-parse HEAD",
|
||||
"cd /opt/mozdef && sudo touch docker/compose/cloudy_mozdef.env docker/compose/rabbitmq.env docker/compose/cloudy_mozdef_mq_cloudtrail.env docker/compose/cloudy_mozdef_mq_sns_sqs.env docker/compose/cloudy_mozdef_kibana.env",
|
||||
"cd /opt/mozdef && sudo make BRANCH={{ user `github_branch`}} set-version-and-fetch-docker-container",
|
||||
"cd /opt/mozdef && sudo docker-compose -f docker/compose/docker-compose-cloudy-mozdef.yml -p mozdef pull",
|
||||
"rm -rf /home/ec2-user/.ssh/authorized_keys",
|
||||
"rm -rf /home/ec2-user/.ssh/known_hosts",
|
||||
"sudo rm -rf /tmp/*",
|
||||
"sudo rm -rf /home/ec2-user/.bash_history",
|
||||
"sudo rm -rf /root/.ssh"
|
||||
]}
|
||||
]
|
||||
"cd /opt/mozdef",
|
||||
"sudo git checkout {{ user `github_branch`}}",
|
||||
"sudo git rev-parse HEAD",
|
||||
"sudo touch docker/compose/cloudy_mozdef.env docker/compose/rabbitmq.env docker/compose/cloudy_mozdef_mq_cloudtrail.env docker/compose/cloudy_mozdef_mq_sns_sqs.env docker/compose/cloudy_mozdef_kibana.env",
|
||||
"sudo sed --in-place s/latest/{{ user `github_branch`}}/g docker/compose/docker-compose-cloudy-mozdef.yml",
|
||||
"sudo docker-compose --file docker/compose/docker-compose-cloudy-mozdef.yml --project-name mozdef pull",
|
||||
"sudo rm --recursive --force --verbose /tmp/* /home/ec2-user/.bash_history /root/.ssh /home/ec2-user/.ssh/known_hosts /home/ec2-user/.ssh/authorized_keys"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -46,7 +46,8 @@ RUN mkdir -p /opt/mozdef/envs/meteor/mozdef
|
|||
RUN if [ "${METEOR_BUILD}" = "YES" ]; then \
|
||||
cd /opt/mozdef/envs/mozdef/meteor && \
|
||||
meteor npm install && \
|
||||
meteor build --server localhost:3002 --directory /opt/mozdef/envs/meteor/mozdef && \
|
||||
echo "Starting meteor build" && \
|
||||
time meteor build --server localhost:3002 --directory /opt/mozdef/envs/meteor/mozdef && \
|
||||
cp -r /opt/mozdef/envs/mozdef/meteor/node_modules /opt/mozdef/envs/meteor/mozdef/node_modules &&\
|
||||
cd /opt/mozdef/envs/meteor/mozdef/bundle/programs/server && \
|
||||
npm install ;\
|
||||
|
|
|
|||
|
|
@ -78,9 +78,9 @@ _________________________________________
|
|||
The Build Sequence
|
||||
__________________
|
||||
|
||||
* A branch is merged into `master` in the GitHub repo
|
||||
* GitHub emits a webhook event to AWS CodeBuild indicating that a commit was
|
||||
pushed to `master`
|
||||
* A branch is merged into `master` in the GitHub repo or a version git tag is
|
||||
applied to a commit
|
||||
* GitHub emits a webhook event to AWS CodeBuild indicating this
|
||||
* AWS CodeBuild reads the
|
||||
`buildspec.yml <https://github.com/mozilla/MozDef/blob/master/cloudy_mozdef/buildspec.yml>`_
|
||||
file to know what to do
|
||||
|
|
@ -98,8 +98,28 @@ __________________
|
|||
target of the `Makefile` which calls `docker-compose build` on the
|
||||
`docker-compose.yml <https://github.com/mozilla/MozDef/blob/master/docker/compose/docker-compose.yml>`_
|
||||
file, building the docker images in the AWS CodeBuild environment. These are
|
||||
built so they can be uploaded later to DockerHub for use by developers and
|
||||
the community.
|
||||
built both so they can be consumed later in the build by packer and also
|
||||
for use by developers and the community.
|
||||
* `deploy` then calls the
|
||||
`docker-push-tagged <https://github.com/mozilla/MozDef/blob/cfeafb77f9d4d4d8df02117a0ffca0ec9379a7d5/Makefile#L113>`_
|
||||
make target which calls
|
||||
|
||||
* the tag-images_
|
||||
make target which calls the
|
||||
`cloudy_mozdef/ci/docker_tag_or_push tag <https://github.com/mozilla/MozDef/blob/master/cloudy_mozdef/ci/docker_tag_or_push>`_
|
||||
script which applies a docker image tag to the local image that was just
|
||||
built by AWS CodeBuild.
|
||||
* the
|
||||
`hub-tagged <https://github.com/mozilla/MozDef/blob/cfeafb77f9d4d4d8df02117a0ffca0ec9379a7d5/Makefile#L116-L117>`_
|
||||
make target which calls the
|
||||
`cloudy_mozdef/ci/docker_tag_or_push push <https://github.com/mozilla/MozDef/blob/master/cloudy_mozdef/ci/docker_tag_or_push>`_
|
||||
script which
|
||||
|
||||
* Uploads the local image that was just built by AWS CodeBuild to DockerHub.
|
||||
If the branch being built is `master` then the image is uploaded both with
|
||||
a tag of `master` as well as with a tag of `latest`
|
||||
* If the branch being built is from a version tag (e.g. `v1.2.3`) then the
|
||||
image is uploaded with only that version tag applied
|
||||
* The `deploy` script next calls the
|
||||
`packer-build-github <https://github.com/mozilla/MozDef/blob/cfeafb77f9d4d4d8df02117a0ffca0ec9379a7d5/cloudy_mozdef/Makefile#L34-L36>`_
|
||||
make target in the
|
||||
|
|
@ -124,31 +144,14 @@ __________________
|
|||
* Within this ec2 instance, packer `clones the MozDef GitHub repo and checks
|
||||
out the branch that triggered this build
|
||||
<https://github.com/mozilla/MozDef/blob/cfeafb77f9d4d4d8df02117a0ffca0ec9379a7d5/cloudy_mozdef/packer/packer.json#L59-L60>`_
|
||||
* packer calls the `set-version-and-fetch-docker-container <https://github.com/mozilla/MozDef/blob/cfeafb77f9d4d4d8df02117a0ffca0ec9379a7d5/Makefile#L148-L149>`_
|
||||
target of the `Makefile` which
|
||||
|
||||
* Calls the
|
||||
`build-from-cwd <https://github.com/mozilla/MozDef/blob/cfeafb77f9d4d4d8df02117a0ffca0ec9379a7d5/Makefile#L78-L79>`_
|
||||
target of the `Makefile` which calls `docker-compose build` on the
|
||||
`docker-compose.yml <https://github.com/mozilla/MozDef/blob/master/docker/compose/docker-compose.yml>`_
|
||||
file, building the docker images in the packer ec2 environment
|
||||
* Calls the tag-images_
|
||||
make target which calls the `cloudy_mozdef/ci/docker_tag_or_push tag <https://github.com/mozilla/MozDef/blob/master/cloudy_mozdef/ci/docker_tag_or_push>`_
|
||||
script.
|
||||
|
||||
* This applies a docker image tag to the local image that was just built
|
||||
by packer. This tag will be referenced by the
|
||||
`docker-compose-cloudy-mozdef.yml` file when the MozDef ec2 instance is
|
||||
launched from the AMI.
|
||||
|
||||
* Replaces all instances of the word `latest` in the
|
||||
`docker-compose-cloudy-mozdef.yml <https://github.com/mozilla/MozDef/blob/master/docker/compose/docker-compose-cloudy-mozdef.yml>`_
|
||||
file with either the branch `master` or the version tag (e.g. `v1.2.3`)
|
||||
|
||||
* packer replaces all instances of the word `latest` in the
|
||||
`docker-compose-cloudy-mozdef.yml <https://github.com/mozilla/MozDef/blob/master/docker/compose/docker-compose-cloudy-mozdef.yml>`_
|
||||
file with either the branch `master` or the version tag (e.g. `v1.2.3`)
|
||||
* packer runs `docker-compose pull` on the
|
||||
`docker-compose-cloudy-mozdef.yml <https://github.com/mozilla/MozDef/blob/master/docker/compose/docker-compose-cloudy-mozdef.yml>`_
|
||||
file to pull down any remaining non MozDef container images that weren't
|
||||
just built in preceding packer steps
|
||||
file to pull down both the docker images that were just built by AWS
|
||||
CodeBuild and uploaded to Dockerhub as well as other non MozDef docker
|
||||
images
|
||||
|
||||
* After packer completes executing the steps laid out in `packer.json` inside
|
||||
the ec2 instance, it generates an AMI from that instance and continues with
|
||||
|
|
@ -168,26 +171,5 @@ __________________
|
|||
* uploads the CloudFormation templates to S3 in a directory either called
|
||||
`master` or the tag version that was built (e.g. `v1.2.3`)
|
||||
|
||||
* `deploy` then calls the
|
||||
`docker-push-tagged <https://github.com/mozilla/MozDef/blob/cfeafb77f9d4d4d8df02117a0ffca0ec9379a7d5/Makefile#L113>`_
|
||||
make target which calls
|
||||
|
||||
* the tag-images_
|
||||
make target which calls the
|
||||
`cloudy_mozdef/ci/docker_tag_or_push tag <https://github.com/mozilla/MozDef/blob/master/cloudy_mozdef/ci/docker_tag_or_push>`_
|
||||
script which applies a docker image tag to the local image that was just
|
||||
built by AWS CodeBuild.
|
||||
* the
|
||||
`hub-tagged <https://github.com/mozilla/MozDef/blob/cfeafb77f9d4d4d8df02117a0ffca0ec9379a7d5/Makefile#L116-L117>`_
|
||||
make target which calls the
|
||||
`cloudy_mozdef/ci/docker_tag_or_push push <https://github.com/mozilla/MozDef/blob/master/cloudy_mozdef/ci/docker_tag_or_push>`_
|
||||
script which
|
||||
|
||||
* Uploads the local image that was just built by AWS CodeBuild to DockerHub.
|
||||
If the branch being built is `master` then the image is uploaded both with
|
||||
a tag of `master` as well as with a tag of `latest`
|
||||
* If the branch being built is from a version tag (e.g. `v1.2.3`) then the
|
||||
image is uploaded with only that version tag applied
|
||||
|
||||
.. _docker/compose/docker-compose-tests.yml: https://github.com/mozilla/MozDef/blob/master/docker/compose/docker-compose-tests.yml
|
||||
.. _tag-images: https://github.com/mozilla/MozDef/blob/cfeafb77f9d4d4d8df02117a0ffca0ec9379a7d5/Makefile#L109-L110
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче